1 // Copyright © 2020 Intel Corporation 2 // 3 // SPDX-License-Identifier: Apache-2.0 4 // 5 #![allow(clippy::undocumented_unsafe_blocks)] 6 // When enabling the `mshv` feature, we skip quite some tests and 7 // hence have known dead-code. This annotation silences dead-code 8 // related warnings for our quality workflow to pass. 9 #![allow(dead_code)] 10 11 extern crate test_infra; 12 13 use std::collections::HashMap; 14 use std::io::{BufRead, Read, Seek, Write}; 15 use std::net::TcpListener; 16 use std::os::unix::io::AsRawFd; 17 use std::path::PathBuf; 18 use std::process::{Child, Command, Stdio}; 19 use std::string::String; 20 use std::sync::mpsc::Receiver; 21 use std::sync::{mpsc, Mutex}; 22 use std::time::Duration; 23 use std::{fs, io, thread}; 24 25 use net_util::MacAddr; 26 use test_infra::*; 27 use vmm_sys_util::tempdir::TempDir; 28 use vmm_sys_util::tempfile::TempFile; 29 use wait_timeout::ChildExt; 30 31 // Constant taken from the VMM crate. 32 const MAX_NUM_PCI_SEGMENTS: u16 = 96; 33 34 #[cfg(target_arch = "x86_64")] 35 mod x86_64 { 36 pub const FOCAL_IMAGE_NAME: &str = "focal-server-cloudimg-amd64-custom-20210609-0.raw"; 37 pub const JAMMY_VFIO_IMAGE_NAME: &str = 38 "jammy-server-cloudimg-amd64-custom-vfio-20241012-0.raw"; 39 pub const FOCAL_IMAGE_NAME_QCOW2: &str = "focal-server-cloudimg-amd64-custom-20210609-0.qcow2"; 40 pub const FOCAL_IMAGE_NAME_QCOW2_BACKING_FILE: &str = 41 "focal-server-cloudimg-amd64-custom-20210609-0-backing.qcow2"; 42 pub const FOCAL_IMAGE_NAME_VHD: &str = "focal-server-cloudimg-amd64-custom-20210609-0.vhd"; 43 pub const FOCAL_IMAGE_NAME_VHDX: &str = "focal-server-cloudimg-amd64-custom-20210609-0.vhdx"; 44 pub const JAMMY_IMAGE_NAME: &str = "jammy-server-cloudimg-amd64-custom-20241017-0.raw"; 45 pub const WINDOWS_IMAGE_NAME: &str = "windows-server-2022-amd64-2.raw"; 46 pub const OVMF_NAME: &str = "CLOUDHV.fd"; 47 pub const GREP_SERIAL_IRQ_CMD: &str = "grep -c 'IO-APIC.*ttyS0' /proc/interrupts || true"; 48 } 49 50 #[cfg(target_arch = "x86_64")] 51 use x86_64::*; 52 53 #[cfg(target_arch = "aarch64")] 54 mod aarch64 { 55 pub const FOCAL_IMAGE_NAME: &str = "focal-server-cloudimg-arm64-custom-20210929-0.raw"; 56 pub const FOCAL_IMAGE_UPDATE_KERNEL_NAME: &str = 57 "focal-server-cloudimg-arm64-custom-20210929-0-update-kernel.raw"; 58 pub const FOCAL_IMAGE_NAME_QCOW2: &str = "focal-server-cloudimg-arm64-custom-20210929-0.qcow2"; 59 pub const FOCAL_IMAGE_NAME_QCOW2_BACKING_FILE: &str = 60 "focal-server-cloudimg-arm64-custom-20210929-0-backing.qcow2"; 61 pub const FOCAL_IMAGE_NAME_VHD: &str = "focal-server-cloudimg-arm64-custom-20210929-0.vhd"; 62 pub const FOCAL_IMAGE_NAME_VHDX: &str = "focal-server-cloudimg-arm64-custom-20210929-0.vhdx"; 63 pub const JAMMY_IMAGE_NAME: &str = "jammy-server-cloudimg-arm64-custom-20220329-0.raw"; 64 pub const WINDOWS_IMAGE_NAME: &str = "windows-11-iot-enterprise-aarch64.raw"; 65 pub const OVMF_NAME: &str = "CLOUDHV_EFI.fd"; 66 pub const GREP_SERIAL_IRQ_CMD: &str = "grep -c 'GICv3.*uart-pl011' /proc/interrupts || true"; 67 pub const GREP_PMU_IRQ_CMD: &str = "grep -c 'GICv3.*arm-pmu' /proc/interrupts || true"; 68 } 69 70 #[cfg(target_arch = "aarch64")] 71 use aarch64::*; 72 73 const DIRECT_KERNEL_BOOT_CMDLINE: &str = 74 "root=/dev/vda1 console=hvc0 rw systemd.journald.forward_to_console=1"; 75 76 const CONSOLE_TEST_STRING: &str = "Started OpenBSD Secure Shell server"; 77 78 // This enum exists to make it more convenient to 79 // implement test for both D-Bus and REST APIs. 80 enum TargetApi { 81 // API socket 82 HttpApi(String), 83 // well known service name, object path 84 DBusApi(String, String), 85 } 86 87 impl TargetApi { 88 fn new_http_api(tmp_dir: &TempDir) -> Self { 89 Self::HttpApi(temp_api_path(tmp_dir)) 90 } 91 92 fn new_dbus_api(tmp_dir: &TempDir) -> Self { 93 // `tmp_dir` is in the form of "/tmp/chXXXXXX" 94 // and we take the `chXXXXXX` part as a unique identifier for the guest 95 let id = tmp_dir.as_path().file_name().unwrap().to_str().unwrap(); 96 97 Self::DBusApi( 98 format!("org.cloudhypervisor.{id}"), 99 format!("/org/cloudhypervisor/{id}"), 100 ) 101 } 102 103 fn guest_args(&self) -> Vec<String> { 104 match self { 105 TargetApi::HttpApi(api_socket) => { 106 vec![format!("--api-socket={}", api_socket.as_str())] 107 } 108 TargetApi::DBusApi(service_name, object_path) => { 109 vec![ 110 format!("--dbus-service-name={}", service_name.as_str()), 111 format!("--dbus-object-path={}", object_path.as_str()), 112 ] 113 } 114 } 115 } 116 117 fn remote_args(&self) -> Vec<String> { 118 // `guest_args` and `remote_args` are consistent with each other 119 self.guest_args() 120 } 121 122 fn remote_command(&self, command: &str, arg: Option<&str>) -> bool { 123 let mut cmd = Command::new(clh_command("ch-remote")); 124 cmd.args(self.remote_args()); 125 cmd.arg(command); 126 127 if let Some(arg) = arg { 128 cmd.arg(arg); 129 } 130 131 let output = cmd.output().unwrap(); 132 if output.status.success() { 133 true 134 } else { 135 eprintln!("Error running ch-remote command: {:?}", &cmd); 136 let stderr = String::from_utf8_lossy(&output.stderr); 137 eprintln!("stderr: {stderr}"); 138 false 139 } 140 } 141 } 142 143 // Start cloud-hypervisor with no VM parameters, only the API server running. 144 // From the API: Create a VM, boot it and check that it looks as expected. 145 fn _test_api_create_boot(target_api: TargetApi, guest: Guest) { 146 let mut child = GuestCommand::new(&guest) 147 .args(target_api.guest_args()) 148 .capture_output() 149 .spawn() 150 .unwrap(); 151 152 thread::sleep(std::time::Duration::new(1, 0)); 153 154 // Verify API server is running 155 assert!(target_api.remote_command("ping", None)); 156 157 // Create the VM first 158 let cpu_count: u8 = 4; 159 let request_body = guest.api_create_body( 160 cpu_count, 161 direct_kernel_boot_path().to_str().unwrap(), 162 DIRECT_KERNEL_BOOT_CMDLINE, 163 ); 164 165 let temp_config_path = guest.tmp_dir.as_path().join("config"); 166 std::fs::write(&temp_config_path, request_body).unwrap(); 167 let create_config = temp_config_path.as_os_str().to_str().unwrap(); 168 169 assert!(target_api.remote_command("create", Some(create_config),)); 170 171 // Then boot it 172 assert!(target_api.remote_command("boot", None)); 173 thread::sleep(std::time::Duration::new(20, 0)); 174 175 let r = std::panic::catch_unwind(|| { 176 // Check that the VM booted as expected 177 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 178 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 179 }); 180 181 kill_child(&mut child); 182 let output = child.wait_with_output().unwrap(); 183 184 handle_child_output(r, &output); 185 } 186 187 // Start cloud-hypervisor with no VM parameters, only the API server running. 188 // From the API: Create a VM, boot it and check it can be shutdown and then 189 // booted again 190 fn _test_api_shutdown(target_api: TargetApi, guest: Guest) { 191 let mut child = GuestCommand::new(&guest) 192 .args(target_api.guest_args()) 193 .capture_output() 194 .spawn() 195 .unwrap(); 196 197 thread::sleep(std::time::Duration::new(1, 0)); 198 199 // Verify API server is running 200 assert!(target_api.remote_command("ping", None)); 201 202 // Create the VM first 203 let cpu_count: u8 = 4; 204 let request_body = guest.api_create_body( 205 cpu_count, 206 direct_kernel_boot_path().to_str().unwrap(), 207 DIRECT_KERNEL_BOOT_CMDLINE, 208 ); 209 210 let temp_config_path = guest.tmp_dir.as_path().join("config"); 211 std::fs::write(&temp_config_path, request_body).unwrap(); 212 let create_config = temp_config_path.as_os_str().to_str().unwrap(); 213 214 let r = std::panic::catch_unwind(|| { 215 assert!(target_api.remote_command("create", Some(create_config))); 216 217 // Then boot it 218 assert!(target_api.remote_command("boot", None)); 219 220 guest.wait_vm_boot(None).unwrap(); 221 222 // Check that the VM booted as expected 223 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 224 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 225 226 // Sync and shutdown without powering off to prevent filesystem 227 // corruption. 228 guest.ssh_command("sync").unwrap(); 229 guest.ssh_command("sudo shutdown -H now").unwrap(); 230 231 // Wait for the guest to be fully shutdown 232 thread::sleep(std::time::Duration::new(20, 0)); 233 234 // Then shut it down 235 assert!(target_api.remote_command("shutdown", None)); 236 237 // Then boot it again 238 assert!(target_api.remote_command("boot", None)); 239 240 guest.wait_vm_boot(None).unwrap(); 241 242 // Check that the VM booted as expected 243 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 244 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 245 }); 246 247 kill_child(&mut child); 248 let output = child.wait_with_output().unwrap(); 249 250 handle_child_output(r, &output); 251 } 252 253 // Start cloud-hypervisor with no VM parameters, only the API server running. 254 // From the API: Create a VM, boot it and check it can be deleted and then recreated 255 // booted again. 256 fn _test_api_delete(target_api: TargetApi, guest: Guest) { 257 let mut child = GuestCommand::new(&guest) 258 .args(target_api.guest_args()) 259 .capture_output() 260 .spawn() 261 .unwrap(); 262 263 thread::sleep(std::time::Duration::new(1, 0)); 264 265 // Verify API server is running 266 assert!(target_api.remote_command("ping", None)); 267 268 // Create the VM first 269 let cpu_count: u8 = 4; 270 let request_body = guest.api_create_body( 271 cpu_count, 272 direct_kernel_boot_path().to_str().unwrap(), 273 DIRECT_KERNEL_BOOT_CMDLINE, 274 ); 275 let temp_config_path = guest.tmp_dir.as_path().join("config"); 276 std::fs::write(&temp_config_path, request_body).unwrap(); 277 let create_config = temp_config_path.as_os_str().to_str().unwrap(); 278 279 let r = std::panic::catch_unwind(|| { 280 assert!(target_api.remote_command("create", Some(create_config))); 281 282 // Then boot it 283 assert!(target_api.remote_command("boot", None)); 284 285 guest.wait_vm_boot(None).unwrap(); 286 287 // Check that the VM booted as expected 288 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 289 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 290 291 // Sync and shutdown without powering off to prevent filesystem 292 // corruption. 293 guest.ssh_command("sync").unwrap(); 294 guest.ssh_command("sudo shutdown -H now").unwrap(); 295 296 // Wait for the guest to be fully shutdown 297 thread::sleep(std::time::Duration::new(20, 0)); 298 299 // Then delete it 300 assert!(target_api.remote_command("delete", None)); 301 302 assert!(target_api.remote_command("create", Some(create_config))); 303 304 // Then boot it again 305 assert!(target_api.remote_command("boot", None)); 306 307 guest.wait_vm_boot(None).unwrap(); 308 309 // Check that the VM booted as expected 310 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 311 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 312 }); 313 314 kill_child(&mut child); 315 let output = child.wait_with_output().unwrap(); 316 317 handle_child_output(r, &output); 318 } 319 320 // Start cloud-hypervisor with no VM parameters, only the API server running. 321 // From the API: Create a VM, boot it and check that it looks as expected. 322 // Then we pause the VM, check that it's no longer available. 323 // Finally we resume the VM and check that it's available. 324 fn _test_api_pause_resume(target_api: TargetApi, guest: Guest) { 325 let mut child = GuestCommand::new(&guest) 326 .args(target_api.guest_args()) 327 .capture_output() 328 .spawn() 329 .unwrap(); 330 331 thread::sleep(std::time::Duration::new(1, 0)); 332 333 // Verify API server is running 334 assert!(target_api.remote_command("ping", None)); 335 336 // Create the VM first 337 let cpu_count: u8 = 4; 338 let request_body = guest.api_create_body( 339 cpu_count, 340 direct_kernel_boot_path().to_str().unwrap(), 341 DIRECT_KERNEL_BOOT_CMDLINE, 342 ); 343 344 let temp_config_path = guest.tmp_dir.as_path().join("config"); 345 std::fs::write(&temp_config_path, request_body).unwrap(); 346 let create_config = temp_config_path.as_os_str().to_str().unwrap(); 347 348 assert!(target_api.remote_command("create", Some(create_config))); 349 350 // Then boot it 351 assert!(target_api.remote_command("boot", None)); 352 thread::sleep(std::time::Duration::new(20, 0)); 353 354 let r = std::panic::catch_unwind(|| { 355 // Check that the VM booted as expected 356 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 357 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 358 359 // We now pause the VM 360 assert!(target_api.remote_command("pause", None)); 361 362 // Check pausing again fails 363 assert!(!target_api.remote_command("pause", None)); 364 365 thread::sleep(std::time::Duration::new(2, 0)); 366 367 // SSH into the VM should fail 368 ssh_command_ip( 369 "grep -c processor /proc/cpuinfo", 370 &guest.network.guest_ip, 371 2, 372 5, 373 ) 374 .unwrap_err(); 375 376 // Resume the VM 377 assert!(target_api.remote_command("resume", None)); 378 379 // Check resuming again fails 380 assert!(!target_api.remote_command("resume", None)); 381 382 thread::sleep(std::time::Duration::new(2, 0)); 383 384 // Now we should be able to SSH back in and get the right number of CPUs 385 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 386 }); 387 388 kill_child(&mut child); 389 let output = child.wait_with_output().unwrap(); 390 391 handle_child_output(r, &output); 392 } 393 394 fn _test_pty_interaction(pty_path: PathBuf) { 395 let mut cf = std::fs::OpenOptions::new() 396 .write(true) 397 .read(true) 398 .open(pty_path) 399 .unwrap(); 400 401 // Some dumb sleeps but we don't want to write 402 // before the console is up and we don't want 403 // to try and write the next line before the 404 // login process is ready. 405 thread::sleep(std::time::Duration::new(5, 0)); 406 assert_eq!(cf.write(b"cloud\n").unwrap(), 6); 407 thread::sleep(std::time::Duration::new(2, 0)); 408 assert_eq!(cf.write(b"cloud123\n").unwrap(), 9); 409 thread::sleep(std::time::Duration::new(2, 0)); 410 assert_eq!(cf.write(b"echo test_pty_console\n").unwrap(), 22); 411 thread::sleep(std::time::Duration::new(2, 0)); 412 413 // read pty and ensure they have a login shell 414 // some fairly hacky workarounds to avoid looping 415 // forever in case the channel is blocked getting output 416 let ptyc = pty_read(cf); 417 let mut empty = 0; 418 let mut prev = String::new(); 419 loop { 420 thread::sleep(std::time::Duration::new(2, 0)); 421 match ptyc.try_recv() { 422 Ok(line) => { 423 empty = 0; 424 prev = prev + &line; 425 if prev.contains("test_pty_console") { 426 break; 427 } 428 } 429 Err(mpsc::TryRecvError::Empty) => { 430 empty += 1; 431 assert!(empty <= 5, "No login on pty"); 432 } 433 _ => { 434 panic!("No login on pty") 435 } 436 } 437 } 438 } 439 440 fn prepare_virtiofsd(tmp_dir: &TempDir, shared_dir: &str) -> (std::process::Child, String) { 441 let mut workload_path = dirs::home_dir().unwrap(); 442 workload_path.push("workloads"); 443 444 let mut virtiofsd_path = workload_path; 445 virtiofsd_path.push("virtiofsd"); 446 let virtiofsd_path = String::from(virtiofsd_path.to_str().unwrap()); 447 448 let virtiofsd_socket_path = 449 String::from(tmp_dir.as_path().join("virtiofs.sock").to_str().unwrap()); 450 451 // Start the daemon 452 let child = Command::new(virtiofsd_path.as_str()) 453 .args(["--shared-dir", shared_dir]) 454 .args(["--socket-path", virtiofsd_socket_path.as_str()]) 455 .args(["--cache", "never"]) 456 .spawn() 457 .unwrap(); 458 459 thread::sleep(std::time::Duration::new(10, 0)); 460 461 (child, virtiofsd_socket_path) 462 } 463 464 fn prepare_vubd( 465 tmp_dir: &TempDir, 466 blk_img: &str, 467 num_queues: usize, 468 rdonly: bool, 469 direct: bool, 470 ) -> (std::process::Child, String) { 471 let mut workload_path = dirs::home_dir().unwrap(); 472 workload_path.push("workloads"); 473 474 let mut blk_file_path = workload_path; 475 blk_file_path.push(blk_img); 476 let blk_file_path = String::from(blk_file_path.to_str().unwrap()); 477 478 let vubd_socket_path = String::from(tmp_dir.as_path().join("vub.sock").to_str().unwrap()); 479 480 // Start the daemon 481 let child = Command::new(clh_command("vhost_user_block")) 482 .args([ 483 "--block-backend", 484 format!( 485 "path={blk_file_path},socket={vubd_socket_path},num_queues={num_queues},readonly={rdonly},direct={direct}" 486 ) 487 .as_str(), 488 ]) 489 .spawn() 490 .unwrap(); 491 492 thread::sleep(std::time::Duration::new(10, 0)); 493 494 (child, vubd_socket_path) 495 } 496 497 fn temp_vsock_path(tmp_dir: &TempDir) -> String { 498 String::from(tmp_dir.as_path().join("vsock").to_str().unwrap()) 499 } 500 501 fn temp_api_path(tmp_dir: &TempDir) -> String { 502 String::from( 503 tmp_dir 504 .as_path() 505 .join("cloud-hypervisor.sock") 506 .to_str() 507 .unwrap(), 508 ) 509 } 510 511 fn temp_event_monitor_path(tmp_dir: &TempDir) -> String { 512 String::from(tmp_dir.as_path().join("event.json").to_str().unwrap()) 513 } 514 515 // Creates the directory and returns the path. 516 fn temp_snapshot_dir_path(tmp_dir: &TempDir) -> String { 517 let snapshot_dir = String::from(tmp_dir.as_path().join("snapshot").to_str().unwrap()); 518 std::fs::create_dir(&snapshot_dir).unwrap(); 519 snapshot_dir 520 } 521 522 fn temp_vmcore_file_path(tmp_dir: &TempDir) -> String { 523 let vmcore_file = String::from(tmp_dir.as_path().join("vmcore").to_str().unwrap()); 524 vmcore_file 525 } 526 527 // Creates the path for direct kernel boot and return the path. 528 // For x86_64, this function returns the vmlinux kernel path. 529 // For AArch64, this function returns the PE kernel path. 530 fn direct_kernel_boot_path() -> PathBuf { 531 let mut workload_path = dirs::home_dir().unwrap(); 532 workload_path.push("workloads"); 533 534 let mut kernel_path = workload_path; 535 #[cfg(target_arch = "x86_64")] 536 kernel_path.push("vmlinux"); 537 #[cfg(target_arch = "aarch64")] 538 kernel_path.push("Image"); 539 540 kernel_path 541 } 542 543 fn edk2_path() -> PathBuf { 544 let mut workload_path = dirs::home_dir().unwrap(); 545 workload_path.push("workloads"); 546 let mut edk2_path = workload_path; 547 edk2_path.push(OVMF_NAME); 548 549 edk2_path 550 } 551 552 fn cloud_hypervisor_release_path() -> String { 553 let mut workload_path = dirs::home_dir().unwrap(); 554 workload_path.push("workloads"); 555 556 let mut ch_release_path = workload_path; 557 #[cfg(target_arch = "x86_64")] 558 ch_release_path.push("cloud-hypervisor-static"); 559 #[cfg(target_arch = "aarch64")] 560 ch_release_path.push("cloud-hypervisor-static-aarch64"); 561 562 ch_release_path.into_os_string().into_string().unwrap() 563 } 564 565 fn prepare_vhost_user_net_daemon( 566 tmp_dir: &TempDir, 567 ip: &str, 568 tap: Option<&str>, 569 mtu: Option<u16>, 570 num_queues: usize, 571 client_mode: bool, 572 ) -> (std::process::Command, String) { 573 let vunet_socket_path = String::from(tmp_dir.as_path().join("vunet.sock").to_str().unwrap()); 574 575 // Start the daemon 576 let mut net_params = format!( 577 "ip={ip},mask=255.255.255.0,socket={vunet_socket_path},num_queues={num_queues},queue_size=1024,client={client_mode}" 578 ); 579 580 if let Some(tap) = tap { 581 net_params.push_str(format!(",tap={tap}").as_str()); 582 } 583 584 if let Some(mtu) = mtu { 585 net_params.push_str(format!(",mtu={mtu}").as_str()); 586 } 587 588 let mut command = Command::new(clh_command("vhost_user_net")); 589 command.args(["--net-backend", net_params.as_str()]); 590 591 (command, vunet_socket_path) 592 } 593 594 fn prepare_swtpm_daemon(tmp_dir: &TempDir) -> (std::process::Command, String) { 595 let swtpm_tpm_dir = String::from(tmp_dir.as_path().join("swtpm").to_str().unwrap()); 596 let swtpm_socket_path = String::from( 597 tmp_dir 598 .as_path() 599 .join("swtpm") 600 .join("swtpm.sock") 601 .to_str() 602 .unwrap(), 603 ); 604 std::fs::create_dir(&swtpm_tpm_dir).unwrap(); 605 606 let mut swtpm_command = Command::new("swtpm"); 607 let swtpm_args = [ 608 "socket", 609 "--tpmstate", 610 &format!("dir={swtpm_tpm_dir}"), 611 "--ctrl", 612 &format!("type=unixio,path={swtpm_socket_path}"), 613 "--flags", 614 "startup-clear", 615 "--tpm2", 616 ]; 617 swtpm_command.args(swtpm_args); 618 619 (swtpm_command, swtpm_socket_path) 620 } 621 622 fn remote_command(api_socket: &str, command: &str, arg: Option<&str>) -> bool { 623 let mut cmd = Command::new(clh_command("ch-remote")); 624 cmd.args([&format!("--api-socket={api_socket}"), command]); 625 626 if let Some(arg) = arg { 627 cmd.arg(arg); 628 } 629 let output = cmd.output().unwrap(); 630 if output.status.success() { 631 true 632 } else { 633 eprintln!("Error running ch-remote command: {:?}", &cmd); 634 let stderr = String::from_utf8_lossy(&output.stderr); 635 eprintln!("stderr: {stderr}"); 636 false 637 } 638 } 639 640 fn remote_command_w_output(api_socket: &str, command: &str, arg: Option<&str>) -> (bool, Vec<u8>) { 641 let mut cmd = Command::new(clh_command("ch-remote")); 642 cmd.args([&format!("--api-socket={api_socket}"), command]); 643 644 if let Some(arg) = arg { 645 cmd.arg(arg); 646 } 647 648 let output = cmd.output().expect("Failed to launch ch-remote"); 649 650 (output.status.success(), output.stdout) 651 } 652 653 fn resize_command( 654 api_socket: &str, 655 desired_vcpus: Option<u8>, 656 desired_ram: Option<usize>, 657 desired_balloon: Option<usize>, 658 event_file: Option<&str>, 659 ) -> bool { 660 let mut cmd = Command::new(clh_command("ch-remote")); 661 cmd.args([&format!("--api-socket={api_socket}"), "resize"]); 662 663 if let Some(desired_vcpus) = desired_vcpus { 664 cmd.arg(format!("--cpus={desired_vcpus}")); 665 } 666 667 if let Some(desired_ram) = desired_ram { 668 cmd.arg(format!("--memory={desired_ram}")); 669 } 670 671 if let Some(desired_balloon) = desired_balloon { 672 cmd.arg(format!("--balloon={desired_balloon}")); 673 } 674 675 let ret = cmd.status().expect("Failed to launch ch-remote").success(); 676 677 if let Some(event_path) = event_file { 678 let latest_events = [ 679 &MetaEvent { 680 event: "resizing".to_string(), 681 device_id: None, 682 }, 683 &MetaEvent { 684 event: "resized".to_string(), 685 device_id: None, 686 }, 687 ]; 688 // See: #5938 689 thread::sleep(std::time::Duration::new(1, 0)); 690 assert!(check_latest_events_exact(&latest_events, event_path)); 691 } 692 693 ret 694 } 695 696 fn resize_zone_command(api_socket: &str, id: &str, desired_size: &str) -> bool { 697 let mut cmd = Command::new(clh_command("ch-remote")); 698 cmd.args([ 699 &format!("--api-socket={api_socket}"), 700 "resize-zone", 701 &format!("--id={id}"), 702 &format!("--size={desired_size}"), 703 ]); 704 705 cmd.status().expect("Failed to launch ch-remote").success() 706 } 707 708 // setup OVS-DPDK bridge and ports 709 fn setup_ovs_dpdk() { 710 // setup OVS-DPDK 711 assert!(exec_host_command_status("service openvswitch-switch start").success()); 712 assert!(exec_host_command_status("ovs-vsctl init").success()); 713 assert!( 714 exec_host_command_status("ovs-vsctl set Open_vSwitch . other_config:dpdk-init=true") 715 .success() 716 ); 717 assert!(exec_host_command_status("service openvswitch-switch restart").success()); 718 719 // Create OVS-DPDK bridge and ports 720 assert!(exec_host_command_status( 721 "ovs-vsctl add-br ovsbr0 -- set bridge ovsbr0 datapath_type=netdev", 722 ) 723 .success()); 724 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user1 -- set Interface vhost-user1 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient1").success()); 725 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user2 -- set Interface vhost-user2 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient2").success()); 726 assert!(exec_host_command_status("ip link set up dev ovsbr0").success()); 727 assert!(exec_host_command_status("service openvswitch-switch restart").success()); 728 } 729 fn cleanup_ovs_dpdk() { 730 assert!(exec_host_command_status("ovs-vsctl del-br ovsbr0").success()); 731 exec_host_command_status("rm -f ovs-vsctl /tmp/dpdkvhostclient1 /tmp/dpdkvhostclient2"); 732 } 733 // Setup two guests and ensure they are connected through ovs-dpdk 734 fn setup_ovs_dpdk_guests( 735 guest1: &Guest, 736 guest2: &Guest, 737 api_socket: &str, 738 release_binary: bool, 739 ) -> (Child, Child) { 740 setup_ovs_dpdk(); 741 742 let clh_path = if !release_binary { 743 clh_command("cloud-hypervisor") 744 } else { 745 cloud_hypervisor_release_path() 746 }; 747 748 let mut child1 = GuestCommand::new_with_binary_path(guest1, &clh_path) 749 .args(["--cpus", "boot=2"]) 750 .args(["--memory", "size=0,shared=on"]) 751 .args(["--memory-zone", "id=mem0,size=1G,shared=on,host_numa_node=0"]) 752 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 753 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 754 .default_disks() 755 .args(["--net", guest1.default_net_string().as_str(), "vhost_user=true,socket=/tmp/dpdkvhostclient1,num_queues=2,queue_size=256,vhost_mode=server"]) 756 .capture_output() 757 .spawn() 758 .unwrap(); 759 760 #[cfg(target_arch = "x86_64")] 761 let guest_net_iface = "ens5"; 762 #[cfg(target_arch = "aarch64")] 763 let guest_net_iface = "enp0s5"; 764 765 let r = std::panic::catch_unwind(|| { 766 guest1.wait_vm_boot(None).unwrap(); 767 768 guest1 769 .ssh_command(&format!( 770 "sudo ip addr add 172.100.0.1/24 dev {guest_net_iface}" 771 )) 772 .unwrap(); 773 guest1 774 .ssh_command(&format!("sudo ip link set up dev {guest_net_iface}")) 775 .unwrap(); 776 777 let guest_ip = guest1.network.guest_ip.clone(); 778 thread::spawn(move || { 779 ssh_command_ip( 780 "nc -l 12345", 781 &guest_ip, 782 DEFAULT_SSH_RETRIES, 783 DEFAULT_SSH_TIMEOUT, 784 ) 785 .unwrap(); 786 }); 787 }); 788 if r.is_err() { 789 cleanup_ovs_dpdk(); 790 791 let _ = child1.kill(); 792 let output = child1.wait_with_output().unwrap(); 793 handle_child_output(r, &output); 794 panic!("Test should already be failed/panicked"); // To explicitly mark this block never return 795 } 796 797 let mut child2 = GuestCommand::new_with_binary_path(guest2, &clh_path) 798 .args(["--api-socket", api_socket]) 799 .args(["--cpus", "boot=2"]) 800 .args(["--memory", "size=0,shared=on"]) 801 .args(["--memory-zone", "id=mem0,size=1G,shared=on,host_numa_node=0"]) 802 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 803 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 804 .default_disks() 805 .args(["--net", guest2.default_net_string().as_str(), "vhost_user=true,socket=/tmp/dpdkvhostclient2,num_queues=2,queue_size=256,vhost_mode=server"]) 806 .capture_output() 807 .spawn() 808 .unwrap(); 809 810 let r = std::panic::catch_unwind(|| { 811 guest2.wait_vm_boot(None).unwrap(); 812 813 guest2 814 .ssh_command(&format!( 815 "sudo ip addr add 172.100.0.2/24 dev {guest_net_iface}" 816 )) 817 .unwrap(); 818 guest2 819 .ssh_command(&format!("sudo ip link set up dev {guest_net_iface}")) 820 .unwrap(); 821 822 // Check the connection works properly between the two VMs 823 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 824 }); 825 if r.is_err() { 826 cleanup_ovs_dpdk(); 827 828 let _ = child1.kill(); 829 let _ = child2.kill(); 830 let output = child2.wait_with_output().unwrap(); 831 handle_child_output(r, &output); 832 panic!("Test should already be failed/panicked"); // To explicitly mark this block never return 833 } 834 835 (child1, child2) 836 } 837 838 enum FwType { 839 Ovmf, 840 RustHypervisorFirmware, 841 } 842 843 fn fw_path(_fw_type: FwType) -> String { 844 let mut workload_path = dirs::home_dir().unwrap(); 845 workload_path.push("workloads"); 846 847 let mut fw_path = workload_path; 848 #[cfg(target_arch = "aarch64")] 849 fw_path.push("CLOUDHV_EFI.fd"); 850 #[cfg(target_arch = "x86_64")] 851 { 852 match _fw_type { 853 FwType::Ovmf => fw_path.push(OVMF_NAME), 854 FwType::RustHypervisorFirmware => fw_path.push("hypervisor-fw"), 855 } 856 } 857 858 fw_path.to_str().unwrap().to_string() 859 } 860 861 #[derive(Debug)] 862 struct MetaEvent { 863 event: String, 864 device_id: Option<String>, 865 } 866 867 impl MetaEvent { 868 pub fn match_with_json_event(&self, v: &serde_json::Value) -> bool { 869 let mut matched = false; 870 if v["event"].as_str().unwrap() == self.event { 871 if let Some(device_id) = &self.device_id { 872 if v["properties"]["id"].as_str().unwrap() == device_id { 873 matched = true 874 } 875 } else { 876 matched = true; 877 } 878 } 879 matched 880 } 881 } 882 883 // Parse the event_monitor file based on the format that each event 884 // is followed by a double newline 885 fn parse_event_file(event_file: &str) -> Vec<serde_json::Value> { 886 let content = fs::read(event_file).unwrap(); 887 let mut ret = Vec::new(); 888 for entry in String::from_utf8_lossy(&content) 889 .trim() 890 .split("\n\n") 891 .collect::<Vec<&str>>() 892 { 893 ret.push(serde_json::from_str(entry).unwrap()); 894 } 895 896 ret 897 } 898 899 // Return true if all events from the input 'expected_events' are matched sequentially 900 // with events from the 'event_file' 901 fn check_sequential_events(expected_events: &[&MetaEvent], event_file: &str) -> bool { 902 let json_events = parse_event_file(event_file); 903 let len = expected_events.len(); 904 let mut idx = 0; 905 for e in &json_events { 906 if idx == len { 907 break; 908 } 909 if expected_events[idx].match_with_json_event(e) { 910 idx += 1; 911 } 912 } 913 914 let ret = idx == len; 915 916 if !ret { 917 eprintln!( 918 "\n\n==== Start 'check_sequential_events' failed ==== \ 919 \n\nexpected_events={:?}\nactual_events={:?} \ 920 \n\n==== End 'check_sequential_events' failed ====", 921 expected_events, json_events, 922 ); 923 } 924 925 ret 926 } 927 928 // Return true if all events from the input 'expected_events' are matched exactly 929 // with events from the 'event_file' 930 fn check_sequential_events_exact(expected_events: &[&MetaEvent], event_file: &str) -> bool { 931 let json_events = parse_event_file(event_file); 932 assert!(expected_events.len() <= json_events.len()); 933 let json_events = &json_events[..expected_events.len()]; 934 935 for (idx, e) in json_events.iter().enumerate() { 936 if !expected_events[idx].match_with_json_event(e) { 937 eprintln!( 938 "\n\n==== Start 'check_sequential_events_exact' failed ==== \ 939 \n\nexpected_events={:?}\nactual_events={:?} \ 940 \n\n==== End 'check_sequential_events_exact' failed ====", 941 expected_events, json_events, 942 ); 943 944 return false; 945 } 946 } 947 948 true 949 } 950 951 // Return true if events from the input 'latest_events' are matched exactly 952 // with the most recent events from the 'event_file' 953 fn check_latest_events_exact(latest_events: &[&MetaEvent], event_file: &str) -> bool { 954 let json_events = parse_event_file(event_file); 955 assert!(latest_events.len() <= json_events.len()); 956 let json_events = &json_events[(json_events.len() - latest_events.len())..]; 957 958 for (idx, e) in json_events.iter().enumerate() { 959 if !latest_events[idx].match_with_json_event(e) { 960 eprintln!( 961 "\n\n==== Start 'check_latest_events_exact' failed ==== \ 962 \n\nexpected_events={:?}\nactual_events={:?} \ 963 \n\n==== End 'check_latest_events_exact' failed ====", 964 latest_events, json_events, 965 ); 966 967 return false; 968 } 969 } 970 971 true 972 } 973 974 fn test_cpu_topology(threads_per_core: u8, cores_per_package: u8, packages: u8, use_fw: bool) { 975 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 976 let guest = Guest::new(Box::new(focal)); 977 let total_vcpus = threads_per_core * cores_per_package * packages; 978 let direct_kernel_boot_path = direct_kernel_boot_path(); 979 let mut kernel_path = direct_kernel_boot_path.to_str().unwrap(); 980 let fw_path = fw_path(FwType::RustHypervisorFirmware); 981 if use_fw { 982 kernel_path = fw_path.as_str(); 983 } 984 985 let mut child = GuestCommand::new(&guest) 986 .args([ 987 "--cpus", 988 &format!( 989 "boot={total_vcpus},topology={threads_per_core}:{cores_per_package}:1:{packages}" 990 ), 991 ]) 992 .args(["--memory", "size=512M"]) 993 .args(["--kernel", kernel_path]) 994 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 995 .default_disks() 996 .default_net() 997 .capture_output() 998 .spawn() 999 .unwrap(); 1000 1001 let r = std::panic::catch_unwind(|| { 1002 guest.wait_vm_boot(None).unwrap(); 1003 assert_eq!( 1004 guest.get_cpu_count().unwrap_or_default(), 1005 u32::from(total_vcpus) 1006 ); 1007 assert_eq!( 1008 guest 1009 .ssh_command("lscpu | grep \"per core\" | cut -f 2 -d \":\" | sed \"s# *##\"") 1010 .unwrap() 1011 .trim() 1012 .parse::<u8>() 1013 .unwrap_or(0), 1014 threads_per_core 1015 ); 1016 1017 assert_eq!( 1018 guest 1019 .ssh_command("lscpu | grep \"per socket\" | cut -f 2 -d \":\" | sed \"s# *##\"") 1020 .unwrap() 1021 .trim() 1022 .parse::<u8>() 1023 .unwrap_or(0), 1024 cores_per_package 1025 ); 1026 1027 assert_eq!( 1028 guest 1029 .ssh_command("lscpu | grep \"Socket\" | cut -f 2 -d \":\" | sed \"s# *##\"") 1030 .unwrap() 1031 .trim() 1032 .parse::<u8>() 1033 .unwrap_or(0), 1034 packages 1035 ); 1036 1037 #[cfg(target_arch = "x86_64")] 1038 { 1039 let mut cpu_id = 0; 1040 for package_id in 0..packages { 1041 for core_id in 0..cores_per_package { 1042 for _ in 0..threads_per_core { 1043 assert_eq!( 1044 guest 1045 .ssh_command(&format!("cat /sys/devices/system/cpu/cpu{cpu_id}/topology/physical_package_id")) 1046 .unwrap() 1047 .trim() 1048 .parse::<u8>() 1049 .unwrap_or(0), 1050 package_id 1051 ); 1052 1053 assert_eq!( 1054 guest 1055 .ssh_command(&format!( 1056 "cat /sys/devices/system/cpu/cpu{cpu_id}/topology/core_id" 1057 )) 1058 .unwrap() 1059 .trim() 1060 .parse::<u8>() 1061 .unwrap_or(0), 1062 core_id 1063 ); 1064 1065 cpu_id += 1; 1066 } 1067 } 1068 } 1069 } 1070 }); 1071 1072 kill_child(&mut child); 1073 let output = child.wait_with_output().unwrap(); 1074 1075 handle_child_output(r, &output); 1076 } 1077 1078 #[allow(unused_variables)] 1079 fn _test_guest_numa_nodes(acpi: bool) { 1080 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1081 let guest = Guest::new(Box::new(focal)); 1082 let api_socket = temp_api_path(&guest.tmp_dir); 1083 #[cfg(target_arch = "x86_64")] 1084 let kernel_path = direct_kernel_boot_path(); 1085 #[cfg(target_arch = "aarch64")] 1086 let kernel_path = if acpi { 1087 edk2_path() 1088 } else { 1089 direct_kernel_boot_path() 1090 }; 1091 1092 let mut child = GuestCommand::new(&guest) 1093 .args(["--cpus", "boot=6,max=12"]) 1094 .args(["--memory", "size=0,hotplug_method=virtio-mem"]) 1095 .args([ 1096 "--memory-zone", 1097 "id=mem0,size=1G,hotplug_size=3G", 1098 "id=mem1,size=2G,hotplug_size=3G", 1099 "id=mem2,size=3G,hotplug_size=3G", 1100 ]) 1101 .args([ 1102 "--numa", 1103 "guest_numa_id=0,cpus=[0-2,9],distances=[1@15,2@20],memory_zones=mem0", 1104 "guest_numa_id=1,cpus=[3-4,6-8],distances=[0@20,2@25],memory_zones=mem1", 1105 "guest_numa_id=2,cpus=[5,10-11],distances=[0@25,1@30],memory_zones=mem2", 1106 ]) 1107 .args(["--kernel", kernel_path.to_str().unwrap()]) 1108 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1109 .args(["--api-socket", &api_socket]) 1110 .capture_output() 1111 .default_disks() 1112 .default_net() 1113 .spawn() 1114 .unwrap(); 1115 1116 let r = std::panic::catch_unwind(|| { 1117 guest.wait_vm_boot(None).unwrap(); 1118 1119 guest.check_numa_common( 1120 Some(&[960_000, 1_920_000, 2_880_000]), 1121 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 1122 Some(&["10 15 20", "20 10 25", "25 30 10"]), 1123 ); 1124 1125 // AArch64 currently does not support hotplug, and therefore we only 1126 // test hotplug-related function on x86_64 here. 1127 #[cfg(target_arch = "x86_64")] 1128 { 1129 guest.enable_memory_hotplug(); 1130 1131 // Resize every memory zone and check each associated NUMA node 1132 // has been assigned the right amount of memory. 1133 resize_zone_command(&api_socket, "mem0", "4G"); 1134 resize_zone_command(&api_socket, "mem1", "4G"); 1135 resize_zone_command(&api_socket, "mem2", "4G"); 1136 // Resize to the maximum amount of CPUs and check each NUMA 1137 // node has been assigned the right CPUs set. 1138 resize_command(&api_socket, Some(12), None, None, None); 1139 thread::sleep(std::time::Duration::new(5, 0)); 1140 1141 guest.check_numa_common( 1142 Some(&[3_840_000, 3_840_000, 3_840_000]), 1143 Some(&[vec![0, 1, 2, 9], vec![3, 4, 6, 7, 8], vec![5, 10, 11]]), 1144 None, 1145 ); 1146 } 1147 }); 1148 1149 kill_child(&mut child); 1150 let output = child.wait_with_output().unwrap(); 1151 1152 handle_child_output(r, &output); 1153 } 1154 1155 #[allow(unused_variables)] 1156 fn _test_power_button(acpi: bool) { 1157 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1158 let guest = Guest::new(Box::new(focal)); 1159 let mut cmd = GuestCommand::new(&guest); 1160 let api_socket = temp_api_path(&guest.tmp_dir); 1161 1162 #[cfg(target_arch = "x86_64")] 1163 let kernel_path = direct_kernel_boot_path(); 1164 #[cfg(target_arch = "aarch64")] 1165 let kernel_path = if acpi { 1166 edk2_path() 1167 } else { 1168 direct_kernel_boot_path() 1169 }; 1170 1171 cmd.args(["--cpus", "boot=1"]) 1172 .args(["--memory", "size=512M"]) 1173 .args(["--kernel", kernel_path.to_str().unwrap()]) 1174 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1175 .capture_output() 1176 .default_disks() 1177 .default_net() 1178 .args(["--api-socket", &api_socket]); 1179 1180 let child = cmd.spawn().unwrap(); 1181 1182 let r = std::panic::catch_unwind(|| { 1183 guest.wait_vm_boot(None).unwrap(); 1184 assert!(remote_command(&api_socket, "power-button", None)); 1185 }); 1186 1187 let output = child.wait_with_output().unwrap(); 1188 assert!(output.status.success()); 1189 handle_child_output(r, &output); 1190 } 1191 1192 type PrepareNetDaemon = dyn Fn( 1193 &TempDir, 1194 &str, 1195 Option<&str>, 1196 Option<u16>, 1197 usize, 1198 bool, 1199 ) -> (std::process::Command, String); 1200 1201 fn test_vhost_user_net( 1202 tap: Option<&str>, 1203 num_queues: usize, 1204 prepare_daemon: &PrepareNetDaemon, 1205 generate_host_mac: bool, 1206 client_mode_daemon: bool, 1207 ) { 1208 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1209 let guest = Guest::new(Box::new(focal)); 1210 let api_socket = temp_api_path(&guest.tmp_dir); 1211 1212 let kernel_path = direct_kernel_boot_path(); 1213 1214 let host_mac = if generate_host_mac { 1215 Some(MacAddr::local_random()) 1216 } else { 1217 None 1218 }; 1219 1220 let mtu = Some(3000); 1221 1222 let (mut daemon_command, vunet_socket_path) = prepare_daemon( 1223 &guest.tmp_dir, 1224 &guest.network.host_ip, 1225 tap, 1226 mtu, 1227 num_queues, 1228 client_mode_daemon, 1229 ); 1230 1231 let net_params = format!( 1232 "vhost_user=true,mac={},socket={},num_queues={},queue_size=1024{},vhost_mode={},mtu=3000", 1233 guest.network.guest_mac, 1234 vunet_socket_path, 1235 num_queues, 1236 if let Some(host_mac) = host_mac { 1237 format!(",host_mac={host_mac}") 1238 } else { 1239 "".to_owned() 1240 }, 1241 if client_mode_daemon { 1242 "server" 1243 } else { 1244 "client" 1245 }, 1246 ); 1247 1248 let mut ch_command = GuestCommand::new(&guest); 1249 ch_command 1250 .args(["--cpus", format!("boot={}", num_queues / 2).as_str()]) 1251 .args(["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 1252 .args(["--kernel", kernel_path.to_str().unwrap()]) 1253 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1254 .default_disks() 1255 .args(["--net", net_params.as_str()]) 1256 .args(["--api-socket", &api_socket]) 1257 .capture_output(); 1258 1259 let mut daemon_child: std::process::Child; 1260 let mut child: std::process::Child; 1261 1262 if client_mode_daemon { 1263 child = ch_command.spawn().unwrap(); 1264 // Make sure the VMM is waiting for the backend to connect 1265 thread::sleep(std::time::Duration::new(10, 0)); 1266 daemon_child = daemon_command.spawn().unwrap(); 1267 } else { 1268 daemon_child = daemon_command.spawn().unwrap(); 1269 // Make sure the backend is waiting for the VMM to connect 1270 thread::sleep(std::time::Duration::new(10, 0)); 1271 child = ch_command.spawn().unwrap(); 1272 } 1273 1274 let r = std::panic::catch_unwind(|| { 1275 guest.wait_vm_boot(None).unwrap(); 1276 1277 if let Some(tap_name) = tap { 1278 let tap_count = exec_host_command_output(&format!("ip link | grep -c {tap_name}")); 1279 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 1280 } 1281 1282 if let Some(host_mac) = tap { 1283 let mac_count = exec_host_command_output(&format!("ip link | grep -c {host_mac}")); 1284 assert_eq!(String::from_utf8_lossy(&mac_count.stdout).trim(), "1"); 1285 } 1286 1287 #[cfg(target_arch = "aarch64")] 1288 let iface = "enp0s4"; 1289 #[cfg(target_arch = "x86_64")] 1290 let iface = "ens4"; 1291 1292 assert_eq!( 1293 guest 1294 .ssh_command(format!("cat /sys/class/net/{iface}/mtu").as_str()) 1295 .unwrap() 1296 .trim(), 1297 "3000" 1298 ); 1299 1300 // 1 network interface + default localhost ==> 2 interfaces 1301 // It's important to note that this test is fully exercising the 1302 // vhost-user-net implementation and the associated backend since 1303 // it does not define any --net network interface. That means all 1304 // the ssh communication in that test happens through the network 1305 // interface backed by vhost-user-net. 1306 assert_eq!( 1307 guest 1308 .ssh_command("ip -o link | wc -l") 1309 .unwrap() 1310 .trim() 1311 .parse::<u32>() 1312 .unwrap_or_default(), 1313 2 1314 ); 1315 1316 // The following pci devices will appear on guest with PCI-MSI 1317 // interrupt vectors assigned. 1318 // 1 virtio-console with 3 vectors: config, Rx, Tx 1319 // 1 virtio-blk with 2 vectors: config, Request 1320 // 1 virtio-blk with 2 vectors: config, Request 1321 // 1 virtio-rng with 2 vectors: config, Request 1322 // Since virtio-net has 2 queue pairs, its vectors is as follows: 1323 // 1 virtio-net with 5 vectors: config, Rx (2), Tx (2) 1324 // Based on the above, the total vectors should 14. 1325 #[cfg(target_arch = "x86_64")] 1326 let grep_cmd = "grep -c PCI-MSI /proc/interrupts"; 1327 #[cfg(target_arch = "aarch64")] 1328 let grep_cmd = "grep -c ITS-PCI-MSIX /proc/interrupts"; 1329 assert_eq!( 1330 guest 1331 .ssh_command(grep_cmd) 1332 .unwrap() 1333 .trim() 1334 .parse::<u32>() 1335 .unwrap_or_default(), 1336 10 + (num_queues as u32) 1337 ); 1338 1339 // ACPI feature is needed. 1340 #[cfg(target_arch = "x86_64")] 1341 { 1342 guest.enable_memory_hotplug(); 1343 1344 // Add RAM to the VM 1345 let desired_ram = 1024 << 20; 1346 resize_command(&api_socket, None, Some(desired_ram), None, None); 1347 1348 thread::sleep(std::time::Duration::new(10, 0)); 1349 1350 // Here by simply checking the size (through ssh), we validate 1351 // the connection is still working, which means vhost-user-net 1352 // keeps working after the resize. 1353 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 1354 } 1355 }); 1356 1357 kill_child(&mut child); 1358 let output = child.wait_with_output().unwrap(); 1359 1360 thread::sleep(std::time::Duration::new(5, 0)); 1361 let _ = daemon_child.kill(); 1362 let _ = daemon_child.wait(); 1363 1364 handle_child_output(r, &output); 1365 } 1366 1367 type PrepareBlkDaemon = dyn Fn(&TempDir, &str, usize, bool, bool) -> (std::process::Child, String); 1368 1369 fn test_vhost_user_blk( 1370 num_queues: usize, 1371 readonly: bool, 1372 direct: bool, 1373 prepare_vhost_user_blk_daemon: Option<&PrepareBlkDaemon>, 1374 ) { 1375 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1376 let guest = Guest::new(Box::new(focal)); 1377 let api_socket = temp_api_path(&guest.tmp_dir); 1378 1379 let kernel_path = direct_kernel_boot_path(); 1380 1381 let (blk_params, daemon_child) = { 1382 let prepare_daemon = prepare_vhost_user_blk_daemon.unwrap(); 1383 // Start the daemon 1384 let (daemon_child, vubd_socket_path) = 1385 prepare_daemon(&guest.tmp_dir, "blk.img", num_queues, readonly, direct); 1386 1387 ( 1388 format!( 1389 "vhost_user=true,socket={vubd_socket_path},num_queues={num_queues},queue_size=128", 1390 ), 1391 Some(daemon_child), 1392 ) 1393 }; 1394 1395 let mut child = GuestCommand::new(&guest) 1396 .args(["--cpus", format!("boot={num_queues}").as_str()]) 1397 .args(["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 1398 .args(["--kernel", kernel_path.to_str().unwrap()]) 1399 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1400 .args([ 1401 "--disk", 1402 format!( 1403 "path={}", 1404 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 1405 ) 1406 .as_str(), 1407 format!( 1408 "path={}", 1409 guest.disk_config.disk(DiskType::CloudInit).unwrap() 1410 ) 1411 .as_str(), 1412 blk_params.as_str(), 1413 ]) 1414 .default_net() 1415 .args(["--api-socket", &api_socket]) 1416 .capture_output() 1417 .spawn() 1418 .unwrap(); 1419 1420 let r = std::panic::catch_unwind(|| { 1421 guest.wait_vm_boot(None).unwrap(); 1422 1423 // Check both if /dev/vdc exists and if the block size is 16M. 1424 assert_eq!( 1425 guest 1426 .ssh_command("lsblk | grep vdc | grep -c 16M") 1427 .unwrap() 1428 .trim() 1429 .parse::<u32>() 1430 .unwrap_or_default(), 1431 1 1432 ); 1433 1434 // Check if this block is RO or RW. 1435 assert_eq!( 1436 guest 1437 .ssh_command("lsblk | grep vdc | awk '{print $5}'") 1438 .unwrap() 1439 .trim() 1440 .parse::<u32>() 1441 .unwrap_or_default(), 1442 readonly as u32 1443 ); 1444 1445 // Check if the number of queues in /sys/block/vdc/mq matches the 1446 // expected num_queues. 1447 assert_eq!( 1448 guest 1449 .ssh_command("ls -ll /sys/block/vdc/mq | grep ^d | wc -l") 1450 .unwrap() 1451 .trim() 1452 .parse::<u32>() 1453 .unwrap_or_default(), 1454 num_queues as u32 1455 ); 1456 1457 // Mount the device 1458 let mount_ro_rw_flag = if readonly { "ro,noload" } else { "rw" }; 1459 guest.ssh_command("mkdir mount_image").unwrap(); 1460 guest 1461 .ssh_command( 1462 format!("sudo mount -o {mount_ro_rw_flag} -t ext4 /dev/vdc mount_image/").as_str(), 1463 ) 1464 .unwrap(); 1465 1466 // Check the content of the block device. The file "foo" should 1467 // contain "bar". 1468 assert_eq!( 1469 guest.ssh_command("cat mount_image/foo").unwrap().trim(), 1470 "bar" 1471 ); 1472 1473 // ACPI feature is needed. 1474 #[cfg(target_arch = "x86_64")] 1475 { 1476 guest.enable_memory_hotplug(); 1477 1478 // Add RAM to the VM 1479 let desired_ram = 1024 << 20; 1480 resize_command(&api_socket, None, Some(desired_ram), None, None); 1481 1482 thread::sleep(std::time::Duration::new(10, 0)); 1483 1484 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 1485 1486 // Check again the content of the block device after the resize 1487 // has been performed. 1488 assert_eq!( 1489 guest.ssh_command("cat mount_image/foo").unwrap().trim(), 1490 "bar" 1491 ); 1492 } 1493 1494 // Unmount the device 1495 guest.ssh_command("sudo umount /dev/vdc").unwrap(); 1496 guest.ssh_command("rm -r mount_image").unwrap(); 1497 }); 1498 1499 kill_child(&mut child); 1500 let output = child.wait_with_output().unwrap(); 1501 1502 if let Some(mut daemon_child) = daemon_child { 1503 thread::sleep(std::time::Duration::new(5, 0)); 1504 let _ = daemon_child.kill(); 1505 let _ = daemon_child.wait(); 1506 } 1507 1508 handle_child_output(r, &output); 1509 } 1510 1511 fn test_boot_from_vhost_user_blk( 1512 num_queues: usize, 1513 readonly: bool, 1514 direct: bool, 1515 prepare_vhost_user_blk_daemon: Option<&PrepareBlkDaemon>, 1516 ) { 1517 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1518 let guest = Guest::new(Box::new(focal)); 1519 1520 let kernel_path = direct_kernel_boot_path(); 1521 1522 let disk_path = guest.disk_config.disk(DiskType::OperatingSystem).unwrap(); 1523 1524 let (blk_boot_params, daemon_child) = { 1525 let prepare_daemon = prepare_vhost_user_blk_daemon.unwrap(); 1526 // Start the daemon 1527 let (daemon_child, vubd_socket_path) = prepare_daemon( 1528 &guest.tmp_dir, 1529 disk_path.as_str(), 1530 num_queues, 1531 readonly, 1532 direct, 1533 ); 1534 1535 ( 1536 format!( 1537 "vhost_user=true,socket={vubd_socket_path},num_queues={num_queues},queue_size=128", 1538 ), 1539 Some(daemon_child), 1540 ) 1541 }; 1542 1543 let mut child = GuestCommand::new(&guest) 1544 .args(["--cpus", format!("boot={num_queues}").as_str()]) 1545 .args(["--memory", "size=512M,shared=on"]) 1546 .args(["--kernel", kernel_path.to_str().unwrap()]) 1547 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1548 .args([ 1549 "--disk", 1550 blk_boot_params.as_str(), 1551 format!( 1552 "path={}", 1553 guest.disk_config.disk(DiskType::CloudInit).unwrap() 1554 ) 1555 .as_str(), 1556 ]) 1557 .default_net() 1558 .capture_output() 1559 .spawn() 1560 .unwrap(); 1561 1562 let r = std::panic::catch_unwind(|| { 1563 guest.wait_vm_boot(None).unwrap(); 1564 1565 // Just check the VM booted correctly. 1566 assert_eq!(guest.get_cpu_count().unwrap_or_default(), num_queues as u32); 1567 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 1568 }); 1569 kill_child(&mut child); 1570 let output = child.wait_with_output().unwrap(); 1571 1572 if let Some(mut daemon_child) = daemon_child { 1573 thread::sleep(std::time::Duration::new(5, 0)); 1574 let _ = daemon_child.kill(); 1575 let _ = daemon_child.wait(); 1576 } 1577 1578 handle_child_output(r, &output); 1579 } 1580 1581 fn _test_virtio_fs( 1582 prepare_daemon: &dyn Fn(&TempDir, &str) -> (std::process::Child, String), 1583 hotplug: bool, 1584 pci_segment: Option<u16>, 1585 ) { 1586 #[cfg(target_arch = "aarch64")] 1587 let focal_image = if hotplug { 1588 FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string() 1589 } else { 1590 FOCAL_IMAGE_NAME.to_string() 1591 }; 1592 #[cfg(target_arch = "x86_64")] 1593 let focal_image = FOCAL_IMAGE_NAME.to_string(); 1594 let focal = UbuntuDiskConfig::new(focal_image); 1595 let guest = Guest::new(Box::new(focal)); 1596 let api_socket = temp_api_path(&guest.tmp_dir); 1597 1598 let mut workload_path = dirs::home_dir().unwrap(); 1599 workload_path.push("workloads"); 1600 1601 let mut shared_dir = workload_path; 1602 shared_dir.push("shared_dir"); 1603 1604 #[cfg(target_arch = "x86_64")] 1605 let kernel_path = direct_kernel_boot_path(); 1606 #[cfg(target_arch = "aarch64")] 1607 let kernel_path = if hotplug { 1608 edk2_path() 1609 } else { 1610 direct_kernel_boot_path() 1611 }; 1612 1613 let (mut daemon_child, virtiofsd_socket_path) = 1614 prepare_daemon(&guest.tmp_dir, shared_dir.to_str().unwrap()); 1615 1616 let mut guest_command = GuestCommand::new(&guest); 1617 guest_command 1618 .args(["--cpus", "boot=1"]) 1619 .args(["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 1620 .args(["--kernel", kernel_path.to_str().unwrap()]) 1621 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1622 .default_disks() 1623 .default_net() 1624 .args(["--api-socket", &api_socket]); 1625 if pci_segment.is_some() { 1626 guest_command.args([ 1627 "--platform", 1628 &format!("num_pci_segments={MAX_NUM_PCI_SEGMENTS}"), 1629 ]); 1630 } 1631 1632 let fs_params = format!( 1633 "id=myfs0,tag=myfs,socket={},num_queues=1,queue_size=1024{}", 1634 virtiofsd_socket_path, 1635 if let Some(pci_segment) = pci_segment { 1636 format!(",pci_segment={pci_segment}") 1637 } else { 1638 "".to_owned() 1639 } 1640 ); 1641 1642 if !hotplug { 1643 guest_command.args(["--fs", fs_params.as_str()]); 1644 } 1645 1646 let mut child = guest_command.capture_output().spawn().unwrap(); 1647 1648 let r = std::panic::catch_unwind(|| { 1649 guest.wait_vm_boot(None).unwrap(); 1650 1651 if hotplug { 1652 // Add fs to the VM 1653 let (cmd_success, cmd_output) = 1654 remote_command_w_output(&api_socket, "add-fs", Some(&fs_params)); 1655 assert!(cmd_success); 1656 1657 if let Some(pci_segment) = pci_segment { 1658 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 1659 "{{\"id\":\"myfs0\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" 1660 ))); 1661 } else { 1662 assert!(String::from_utf8_lossy(&cmd_output) 1663 .contains("{\"id\":\"myfs0\",\"bdf\":\"0000:00:06.0\"}")); 1664 } 1665 1666 thread::sleep(std::time::Duration::new(10, 0)); 1667 } 1668 1669 // Mount shared directory through virtio_fs filesystem 1670 guest 1671 .ssh_command("mkdir -p mount_dir && sudo mount -t virtiofs myfs mount_dir/") 1672 .unwrap(); 1673 1674 // Check file1 exists and its content is "foo" 1675 assert_eq!( 1676 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1677 "foo" 1678 ); 1679 // Check file2 does not exist 1680 guest 1681 .ssh_command("[ ! -f 'mount_dir/file2' ] || true") 1682 .unwrap(); 1683 1684 // Check file3 exists and its content is "bar" 1685 assert_eq!( 1686 guest.ssh_command("cat mount_dir/file3").unwrap().trim(), 1687 "bar" 1688 ); 1689 1690 // ACPI feature is needed. 1691 #[cfg(target_arch = "x86_64")] 1692 { 1693 guest.enable_memory_hotplug(); 1694 1695 // Add RAM to the VM 1696 let desired_ram = 1024 << 20; 1697 resize_command(&api_socket, None, Some(desired_ram), None, None); 1698 1699 thread::sleep(std::time::Duration::new(30, 0)); 1700 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 1701 1702 // After the resize, check again that file1 exists and its 1703 // content is "foo". 1704 assert_eq!( 1705 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1706 "foo" 1707 ); 1708 } 1709 1710 if hotplug { 1711 // Remove from VM 1712 guest.ssh_command("sudo umount mount_dir").unwrap(); 1713 assert!(remote_command(&api_socket, "remove-device", Some("myfs0"))); 1714 } 1715 }); 1716 1717 let (r, hotplug_daemon_child) = if r.is_ok() && hotplug { 1718 thread::sleep(std::time::Duration::new(10, 0)); 1719 let (daemon_child, virtiofsd_socket_path) = 1720 prepare_daemon(&guest.tmp_dir, shared_dir.to_str().unwrap()); 1721 1722 let r = std::panic::catch_unwind(|| { 1723 thread::sleep(std::time::Duration::new(10, 0)); 1724 let fs_params = format!( 1725 "id=myfs0,tag=myfs,socket={},num_queues=1,queue_size=1024{}", 1726 virtiofsd_socket_path, 1727 if let Some(pci_segment) = pci_segment { 1728 format!(",pci_segment={pci_segment}") 1729 } else { 1730 "".to_owned() 1731 } 1732 ); 1733 1734 // Add back and check it works 1735 let (cmd_success, cmd_output) = 1736 remote_command_w_output(&api_socket, "add-fs", Some(&fs_params)); 1737 assert!(cmd_success); 1738 if let Some(pci_segment) = pci_segment { 1739 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 1740 "{{\"id\":\"myfs0\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" 1741 ))); 1742 } else { 1743 assert!(String::from_utf8_lossy(&cmd_output) 1744 .contains("{\"id\":\"myfs0\",\"bdf\":\"0000:00:06.0\"}")); 1745 } 1746 1747 thread::sleep(std::time::Duration::new(10, 0)); 1748 // Mount shared directory through virtio_fs filesystem 1749 guest 1750 .ssh_command("mkdir -p mount_dir && sudo mount -t virtiofs myfs mount_dir/") 1751 .unwrap(); 1752 1753 // Check file1 exists and its content is "foo" 1754 assert_eq!( 1755 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1756 "foo" 1757 ); 1758 }); 1759 1760 (r, Some(daemon_child)) 1761 } else { 1762 (r, None) 1763 }; 1764 1765 kill_child(&mut child); 1766 let output = child.wait_with_output().unwrap(); 1767 1768 let _ = daemon_child.kill(); 1769 let _ = daemon_child.wait(); 1770 1771 if let Some(mut daemon_child) = hotplug_daemon_child { 1772 let _ = daemon_child.kill(); 1773 let _ = daemon_child.wait(); 1774 } 1775 1776 handle_child_output(r, &output); 1777 } 1778 1779 fn test_virtio_pmem(discard_writes: bool, specify_size: bool) { 1780 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1781 let guest = Guest::new(Box::new(focal)); 1782 1783 let kernel_path = direct_kernel_boot_path(); 1784 1785 let pmem_temp_file = TempFile::new().unwrap(); 1786 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 1787 1788 std::process::Command::new("mkfs.ext4") 1789 .arg(pmem_temp_file.as_path()) 1790 .output() 1791 .expect("Expect creating disk image to succeed"); 1792 1793 let mut child = GuestCommand::new(&guest) 1794 .args(["--cpus", "boot=1"]) 1795 .args(["--memory", "size=512M"]) 1796 .args(["--kernel", kernel_path.to_str().unwrap()]) 1797 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1798 .default_disks() 1799 .default_net() 1800 .args([ 1801 "--pmem", 1802 format!( 1803 "file={}{}{}", 1804 pmem_temp_file.as_path().to_str().unwrap(), 1805 if specify_size { ",size=128M" } else { "" }, 1806 if discard_writes { 1807 ",discard_writes=on" 1808 } else { 1809 "" 1810 } 1811 ) 1812 .as_str(), 1813 ]) 1814 .capture_output() 1815 .spawn() 1816 .unwrap(); 1817 1818 let r = std::panic::catch_unwind(|| { 1819 guest.wait_vm_boot(None).unwrap(); 1820 1821 // Check for the presence of /dev/pmem0 1822 assert_eq!( 1823 guest.ssh_command("ls /dev/pmem0").unwrap().trim(), 1824 "/dev/pmem0" 1825 ); 1826 1827 // Check changes persist after reboot 1828 assert_eq!(guest.ssh_command("sudo mount /dev/pmem0 /mnt").unwrap(), ""); 1829 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), "lost+found\n"); 1830 guest 1831 .ssh_command("echo test123 | sudo tee /mnt/test") 1832 .unwrap(); 1833 assert_eq!(guest.ssh_command("sudo umount /mnt").unwrap(), ""); 1834 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), ""); 1835 1836 guest.reboot_linux(0, None); 1837 assert_eq!(guest.ssh_command("sudo mount /dev/pmem0 /mnt").unwrap(), ""); 1838 assert_eq!( 1839 guest 1840 .ssh_command("sudo cat /mnt/test || true") 1841 .unwrap() 1842 .trim(), 1843 if discard_writes { "" } else { "test123" } 1844 ); 1845 }); 1846 1847 kill_child(&mut child); 1848 let output = child.wait_with_output().unwrap(); 1849 1850 handle_child_output(r, &output); 1851 } 1852 1853 fn get_fd_count(pid: u32) -> usize { 1854 fs::read_dir(format!("/proc/{pid}/fd")).unwrap().count() 1855 } 1856 1857 fn _test_virtio_vsock(hotplug: bool) { 1858 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1859 let guest = Guest::new(Box::new(focal)); 1860 1861 #[cfg(target_arch = "x86_64")] 1862 let kernel_path = direct_kernel_boot_path(); 1863 #[cfg(target_arch = "aarch64")] 1864 let kernel_path = if hotplug { 1865 edk2_path() 1866 } else { 1867 direct_kernel_boot_path() 1868 }; 1869 1870 let socket = temp_vsock_path(&guest.tmp_dir); 1871 let api_socket = temp_api_path(&guest.tmp_dir); 1872 1873 let mut cmd = GuestCommand::new(&guest); 1874 cmd.args(["--api-socket", &api_socket]); 1875 cmd.args(["--cpus", "boot=1"]); 1876 cmd.args(["--memory", "size=512M"]); 1877 cmd.args(["--kernel", kernel_path.to_str().unwrap()]); 1878 cmd.args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]); 1879 cmd.default_disks(); 1880 cmd.default_net(); 1881 1882 if !hotplug { 1883 cmd.args(["--vsock", format!("cid=3,socket={socket}").as_str()]); 1884 } 1885 1886 let mut child = cmd.capture_output().spawn().unwrap(); 1887 1888 let r = std::panic::catch_unwind(|| { 1889 guest.wait_vm_boot(None).unwrap(); 1890 1891 if hotplug { 1892 let (cmd_success, cmd_output) = remote_command_w_output( 1893 &api_socket, 1894 "add-vsock", 1895 Some(format!("cid=3,socket={socket},id=test0").as_str()), 1896 ); 1897 assert!(cmd_success); 1898 assert!(String::from_utf8_lossy(&cmd_output) 1899 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 1900 thread::sleep(std::time::Duration::new(10, 0)); 1901 // Check adding a second one fails 1902 assert!(!remote_command( 1903 &api_socket, 1904 "add-vsock", 1905 Some("cid=1234,socket=/tmp/fail") 1906 )); 1907 } 1908 1909 // Validate vsock works as expected. 1910 guest.check_vsock(socket.as_str()); 1911 guest.reboot_linux(0, None); 1912 // Validate vsock still works after a reboot. 1913 guest.check_vsock(socket.as_str()); 1914 1915 if hotplug { 1916 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 1917 } 1918 }); 1919 1920 kill_child(&mut child); 1921 let output = child.wait_with_output().unwrap(); 1922 1923 handle_child_output(r, &output); 1924 } 1925 1926 fn get_ksm_pages_shared() -> u32 { 1927 fs::read_to_string("/sys/kernel/mm/ksm/pages_shared") 1928 .unwrap() 1929 .trim() 1930 .parse::<u32>() 1931 .unwrap() 1932 } 1933 1934 fn test_memory_mergeable(mergeable: bool) { 1935 let memory_param = if mergeable { 1936 "mergeable=on" 1937 } else { 1938 "mergeable=off" 1939 }; 1940 1941 // We are assuming the rest of the system in our CI is not using mergeable memory 1942 let ksm_ps_init = get_ksm_pages_shared(); 1943 assert!(ksm_ps_init == 0); 1944 1945 let focal1 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1946 let guest1 = Guest::new(Box::new(focal1)); 1947 let mut child1 = GuestCommand::new(&guest1) 1948 .args(["--cpus", "boot=1"]) 1949 .args(["--memory", format!("size=512M,{memory_param}").as_str()]) 1950 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 1951 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1952 .default_disks() 1953 .args(["--net", guest1.default_net_string().as_str()]) 1954 .args(["--serial", "tty", "--console", "off"]) 1955 .capture_output() 1956 .spawn() 1957 .unwrap(); 1958 1959 let r = std::panic::catch_unwind(|| { 1960 guest1.wait_vm_boot(None).unwrap(); 1961 }); 1962 if r.is_err() { 1963 kill_child(&mut child1); 1964 let output = child1.wait_with_output().unwrap(); 1965 handle_child_output(r, &output); 1966 panic!("Test should already be failed/panicked"); // To explicitly mark this block never return 1967 } 1968 1969 let ksm_ps_guest1 = get_ksm_pages_shared(); 1970 1971 let focal2 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1972 let guest2 = Guest::new(Box::new(focal2)); 1973 let mut child2 = GuestCommand::new(&guest2) 1974 .args(["--cpus", "boot=1"]) 1975 .args(["--memory", format!("size=512M,{memory_param}").as_str()]) 1976 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 1977 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1978 .default_disks() 1979 .args(["--net", guest2.default_net_string().as_str()]) 1980 .args(["--serial", "tty", "--console", "off"]) 1981 .capture_output() 1982 .spawn() 1983 .unwrap(); 1984 1985 let r = std::panic::catch_unwind(|| { 1986 guest2.wait_vm_boot(None).unwrap(); 1987 let ksm_ps_guest2 = get_ksm_pages_shared(); 1988 1989 if mergeable { 1990 println!( 1991 "ksm pages_shared after vm1 booted '{ksm_ps_guest1}', ksm pages_shared after vm2 booted '{ksm_ps_guest2}'" 1992 ); 1993 // We are expecting the number of shared pages to increase as the number of VM increases 1994 assert!(ksm_ps_guest1 < ksm_ps_guest2); 1995 } else { 1996 assert!(ksm_ps_guest1 == 0); 1997 assert!(ksm_ps_guest2 == 0); 1998 } 1999 }); 2000 2001 kill_child(&mut child1); 2002 kill_child(&mut child2); 2003 2004 let output = child1.wait_with_output().unwrap(); 2005 child2.wait().unwrap(); 2006 2007 handle_child_output(r, &output); 2008 } 2009 2010 fn _get_vmm_overhead(pid: u32, guest_memory_size: u32) -> HashMap<String, u32> { 2011 let smaps = fs::File::open(format!("/proc/{pid}/smaps")).unwrap(); 2012 let reader = io::BufReader::new(smaps); 2013 2014 let mut skip_map: bool = false; 2015 let mut region_name: String = "".to_string(); 2016 let mut region_maps = HashMap::new(); 2017 for line in reader.lines() { 2018 let l = line.unwrap(); 2019 2020 if l.contains('-') { 2021 let values: Vec<&str> = l.split_whitespace().collect(); 2022 region_name = values.last().unwrap().trim().to_string(); 2023 if region_name == "0" { 2024 region_name = "anonymous".to_string() 2025 } 2026 } 2027 2028 // Each section begins with something that looks like: 2029 // Size: 2184 kB 2030 if l.starts_with("Size:") { 2031 let values: Vec<&str> = l.split_whitespace().collect(); 2032 let map_size = values[1].parse::<u32>().unwrap(); 2033 // We skip the assigned guest RAM map, its RSS is only 2034 // dependent on the guest actual memory usage. 2035 // Everything else can be added to the VMM overhead. 2036 skip_map = map_size >= guest_memory_size; 2037 continue; 2038 } 2039 2040 // If this is a map we're taking into account, then we only 2041 // count the RSS. The sum of all counted RSS is the VMM overhead. 2042 if !skip_map && l.starts_with("Rss:") { 2043 let values: Vec<&str> = l.split_whitespace().collect(); 2044 let value = values[1].trim().parse::<u32>().unwrap(); 2045 *region_maps.entry(region_name.clone()).or_insert(0) += value; 2046 } 2047 } 2048 2049 region_maps 2050 } 2051 2052 fn get_vmm_overhead(pid: u32, guest_memory_size: u32) -> u32 { 2053 let mut total = 0; 2054 2055 for (region_name, value) in &_get_vmm_overhead(pid, guest_memory_size) { 2056 eprintln!("{region_name}: {value}"); 2057 total += value; 2058 } 2059 2060 total 2061 } 2062 2063 fn process_rss_kib(pid: u32) -> usize { 2064 let command = format!("ps -q {pid} -o rss="); 2065 let rss = exec_host_command_output(&command); 2066 String::from_utf8_lossy(&rss.stdout).trim().parse().unwrap() 2067 } 2068 2069 // 10MB is our maximum accepted overhead. 2070 const MAXIMUM_VMM_OVERHEAD_KB: u32 = 10 * 1024; 2071 2072 #[derive(PartialEq, Eq, PartialOrd)] 2073 struct Counters { 2074 rx_bytes: u64, 2075 rx_frames: u64, 2076 tx_bytes: u64, 2077 tx_frames: u64, 2078 read_bytes: u64, 2079 write_bytes: u64, 2080 read_ops: u64, 2081 write_ops: u64, 2082 } 2083 2084 fn get_counters(api_socket: &str) -> Counters { 2085 // Get counters 2086 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "counters", None); 2087 assert!(cmd_success); 2088 2089 let counters: HashMap<&str, HashMap<&str, u64>> = 2090 serde_json::from_slice(&cmd_output).unwrap_or_default(); 2091 2092 let rx_bytes = *counters.get("_net2").unwrap().get("rx_bytes").unwrap(); 2093 let rx_frames = *counters.get("_net2").unwrap().get("rx_frames").unwrap(); 2094 let tx_bytes = *counters.get("_net2").unwrap().get("tx_bytes").unwrap(); 2095 let tx_frames = *counters.get("_net2").unwrap().get("tx_frames").unwrap(); 2096 2097 let read_bytes = *counters.get("_disk0").unwrap().get("read_bytes").unwrap(); 2098 let write_bytes = *counters.get("_disk0").unwrap().get("write_bytes").unwrap(); 2099 let read_ops = *counters.get("_disk0").unwrap().get("read_ops").unwrap(); 2100 let write_ops = *counters.get("_disk0").unwrap().get("write_ops").unwrap(); 2101 2102 Counters { 2103 rx_bytes, 2104 rx_frames, 2105 tx_bytes, 2106 tx_frames, 2107 read_bytes, 2108 write_bytes, 2109 read_ops, 2110 write_ops, 2111 } 2112 } 2113 2114 fn pty_read(mut pty: std::fs::File) -> Receiver<String> { 2115 let (tx, rx) = mpsc::channel::<String>(); 2116 thread::spawn(move || loop { 2117 thread::sleep(std::time::Duration::new(1, 0)); 2118 let mut buf = [0; 512]; 2119 match pty.read(&mut buf) { 2120 Ok(_bytes) => { 2121 let output = std::str::from_utf8(&buf).unwrap().to_string(); 2122 match tx.send(output) { 2123 Ok(_) => (), 2124 Err(_) => break, 2125 } 2126 } 2127 Err(_) => break, 2128 } 2129 }); 2130 rx 2131 } 2132 2133 fn get_pty_path(api_socket: &str, pty_type: &str) -> PathBuf { 2134 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "info", None); 2135 assert!(cmd_success); 2136 let info: serde_json::Value = serde_json::from_slice(&cmd_output).unwrap_or_default(); 2137 assert_eq!("Pty", info["config"][pty_type]["mode"]); 2138 PathBuf::from( 2139 info["config"][pty_type]["file"] 2140 .as_str() 2141 .expect("Missing pty path"), 2142 ) 2143 } 2144 2145 // VFIO test network setup. 2146 // We reserve a different IP class for it: 172.18.0.0/24. 2147 #[cfg(target_arch = "x86_64")] 2148 fn setup_vfio_network_interfaces() { 2149 // 'vfio-br0' 2150 assert!(exec_host_command_status("sudo ip link add name vfio-br0 type bridge").success()); 2151 assert!(exec_host_command_status("sudo ip link set vfio-br0 up").success()); 2152 assert!(exec_host_command_status("sudo ip addr add 172.18.0.1/24 dev vfio-br0").success()); 2153 // 'vfio-tap0' 2154 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap0 mode tap").success()); 2155 assert!(exec_host_command_status("sudo ip link set vfio-tap0 master vfio-br0").success()); 2156 assert!(exec_host_command_status("sudo ip link set vfio-tap0 up").success()); 2157 // 'vfio-tap1' 2158 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap1 mode tap").success()); 2159 assert!(exec_host_command_status("sudo ip link set vfio-tap1 master vfio-br0").success()); 2160 assert!(exec_host_command_status("sudo ip link set vfio-tap1 up").success()); 2161 // 'vfio-tap2' 2162 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap2 mode tap").success()); 2163 assert!(exec_host_command_status("sudo ip link set vfio-tap2 master vfio-br0").success()); 2164 assert!(exec_host_command_status("sudo ip link set vfio-tap2 up").success()); 2165 // 'vfio-tap3' 2166 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap3 mode tap").success()); 2167 assert!(exec_host_command_status("sudo ip link set vfio-tap3 master vfio-br0").success()); 2168 assert!(exec_host_command_status("sudo ip link set vfio-tap3 up").success()); 2169 } 2170 2171 // Tear VFIO test network down 2172 #[cfg(target_arch = "x86_64")] 2173 fn cleanup_vfio_network_interfaces() { 2174 assert!(exec_host_command_status("sudo ip link del vfio-br0").success()); 2175 assert!(exec_host_command_status("sudo ip link del vfio-tap0").success()); 2176 assert!(exec_host_command_status("sudo ip link del vfio-tap1").success()); 2177 assert!(exec_host_command_status("sudo ip link del vfio-tap2").success()); 2178 assert!(exec_host_command_status("sudo ip link del vfio-tap3").success()); 2179 } 2180 2181 fn balloon_size(api_socket: &str) -> u64 { 2182 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "info", None); 2183 assert!(cmd_success); 2184 2185 let info: serde_json::Value = serde_json::from_slice(&cmd_output).unwrap_or_default(); 2186 let total_mem = &info["config"]["memory"]["size"] 2187 .to_string() 2188 .parse::<u64>() 2189 .unwrap(); 2190 let actual_mem = &info["memory_actual_size"] 2191 .to_string() 2192 .parse::<u64>() 2193 .unwrap(); 2194 total_mem - actual_mem 2195 } 2196 2197 fn vm_state(api_socket: &str) -> String { 2198 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "info", None); 2199 assert!(cmd_success); 2200 2201 let info: serde_json::Value = serde_json::from_slice(&cmd_output).unwrap_or_default(); 2202 let state = &info["state"].as_str().unwrap(); 2203 2204 state.to_string() 2205 } 2206 2207 // This test validates that it can find the virtio-iommu device at first. 2208 // It also verifies that both disks and the network card are attached to 2209 // the virtual IOMMU by looking at /sys/kernel/iommu_groups directory. 2210 // The last interesting part of this test is that it exercises the network 2211 // interface attached to the virtual IOMMU since this is the one used to 2212 // send all commands through SSH. 2213 fn _test_virtio_iommu(acpi: bool) { 2214 // Virtio-iommu support is ready in recent kernel (v5.14). But the kernel in 2215 // Focal image is still old. 2216 // So if ACPI is enabled on AArch64, we use a modified Focal image in which 2217 // the kernel binary has been updated. 2218 #[cfg(target_arch = "aarch64")] 2219 let focal_image = FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string(); 2220 #[cfg(target_arch = "x86_64")] 2221 let focal_image = FOCAL_IMAGE_NAME.to_string(); 2222 let focal = UbuntuDiskConfig::new(focal_image); 2223 let guest = Guest::new(Box::new(focal)); 2224 2225 #[cfg(target_arch = "x86_64")] 2226 let kernel_path = direct_kernel_boot_path(); 2227 #[cfg(target_arch = "aarch64")] 2228 let kernel_path = if acpi { 2229 edk2_path() 2230 } else { 2231 direct_kernel_boot_path() 2232 }; 2233 2234 let mut child = GuestCommand::new(&guest) 2235 .args(["--cpus", "boot=1"]) 2236 .args(["--memory", "size=512M"]) 2237 .args(["--kernel", kernel_path.to_str().unwrap()]) 2238 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2239 .args([ 2240 "--disk", 2241 format!( 2242 "path={},iommu=on", 2243 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 2244 ) 2245 .as_str(), 2246 format!( 2247 "path={},iommu=on", 2248 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2249 ) 2250 .as_str(), 2251 ]) 2252 .args(["--net", guest.default_net_string_w_iommu().as_str()]) 2253 .capture_output() 2254 .spawn() 2255 .unwrap(); 2256 2257 let r = std::panic::catch_unwind(|| { 2258 guest.wait_vm_boot(None).unwrap(); 2259 2260 // Verify the virtio-iommu device is present. 2261 assert!(guest 2262 .does_device_vendor_pair_match("0x1057", "0x1af4") 2263 .unwrap_or_default()); 2264 2265 // On AArch64, if the guest system boots from FDT, the behavior of IOMMU is a bit 2266 // different with ACPI. 2267 // All devices on the PCI bus will be attached to the virtual IOMMU, except the 2268 // virtio-iommu device itself. So these devices will all be added to IOMMU groups, 2269 // and appear under folder '/sys/kernel/iommu_groups/'. 2270 // The result is, in the case of FDT, IOMMU group '0' contains "0000:00:01.0" 2271 // which is the console. The first disk "0000:00:02.0" is in group '1'. 2272 // While on ACPI, console device is not attached to IOMMU. So the IOMMU group '0' 2273 // contains "0000:00:02.0" which is the first disk. 2274 // 2275 // Verify the iommu group of the first disk. 2276 let iommu_group = !acpi as i32; 2277 assert_eq!( 2278 guest 2279 .ssh_command(format!("ls /sys/kernel/iommu_groups/{iommu_group}/devices").as_str()) 2280 .unwrap() 2281 .trim(), 2282 "0000:00:02.0" 2283 ); 2284 2285 // Verify the iommu group of the second disk. 2286 let iommu_group = if acpi { 1 } else { 2 }; 2287 assert_eq!( 2288 guest 2289 .ssh_command(format!("ls /sys/kernel/iommu_groups/{iommu_group}/devices").as_str()) 2290 .unwrap() 2291 .trim(), 2292 "0000:00:03.0" 2293 ); 2294 2295 // Verify the iommu group of the network card. 2296 let iommu_group = if acpi { 2 } else { 3 }; 2297 assert_eq!( 2298 guest 2299 .ssh_command(format!("ls /sys/kernel/iommu_groups/{iommu_group}/devices").as_str()) 2300 .unwrap() 2301 .trim(), 2302 "0000:00:04.0" 2303 ); 2304 }); 2305 2306 kill_child(&mut child); 2307 let output = child.wait_with_output().unwrap(); 2308 2309 handle_child_output(r, &output); 2310 } 2311 2312 fn get_reboot_count(guest: &Guest) -> u32 { 2313 guest 2314 .ssh_command("sudo last | grep -c reboot") 2315 .unwrap() 2316 .trim() 2317 .parse::<u32>() 2318 .unwrap_or_default() 2319 } 2320 2321 fn enable_guest_watchdog(guest: &Guest, watchdog_sec: u32) { 2322 // Check for PCI device 2323 assert!(guest 2324 .does_device_vendor_pair_match("0x1063", "0x1af4") 2325 .unwrap_or_default()); 2326 2327 // Enable systemd watchdog 2328 guest 2329 .ssh_command(&format!( 2330 "echo RuntimeWatchdogSec={watchdog_sec}s | sudo tee -a /etc/systemd/system.conf" 2331 )) 2332 .unwrap(); 2333 2334 guest.ssh_command("sudo systemctl daemon-reexec").unwrap(); 2335 } 2336 2337 fn make_guest_panic(guest: &Guest) { 2338 // Check for pvpanic device 2339 assert!(guest 2340 .does_device_vendor_pair_match("0x0011", "0x1b36") 2341 .unwrap_or_default()); 2342 2343 // Trigger guest a panic 2344 guest.ssh_command("screen -dmS reboot sh -c \"sleep 5; echo s | tee /proc/sysrq-trigger; echo c | sudo tee /proc/sysrq-trigger\"").unwrap(); 2345 } 2346 2347 mod common_parallel { 2348 use std::fs::OpenOptions; 2349 use std::io::SeekFrom; 2350 2351 use crate::*; 2352 2353 #[test] 2354 #[cfg(target_arch = "x86_64")] 2355 fn test_focal_hypervisor_fw() { 2356 test_simple_launch(fw_path(FwType::RustHypervisorFirmware), FOCAL_IMAGE_NAME) 2357 } 2358 2359 #[test] 2360 #[cfg(target_arch = "x86_64")] 2361 fn test_focal_ovmf() { 2362 test_simple_launch(fw_path(FwType::Ovmf), FOCAL_IMAGE_NAME) 2363 } 2364 2365 #[cfg(target_arch = "x86_64")] 2366 fn test_simple_launch(fw_path: String, disk_path: &str) { 2367 let disk_config = Box::new(UbuntuDiskConfig::new(disk_path.to_string())); 2368 let guest = Guest::new(disk_config); 2369 let event_path = temp_event_monitor_path(&guest.tmp_dir); 2370 2371 let mut child = GuestCommand::new(&guest) 2372 .args(["--cpus", "boot=1"]) 2373 .args(["--memory", "size=512M"]) 2374 .args(["--kernel", fw_path.as_str()]) 2375 .default_disks() 2376 .default_net() 2377 .args(["--serial", "tty", "--console", "off"]) 2378 .args(["--event-monitor", format!("path={event_path}").as_str()]) 2379 .capture_output() 2380 .spawn() 2381 .unwrap(); 2382 2383 let r = std::panic::catch_unwind(|| { 2384 guest.wait_vm_boot(Some(120)).unwrap(); 2385 2386 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 2387 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 2388 assert_eq!(guest.get_pci_bridge_class().unwrap_or_default(), "0x060000"); 2389 2390 let expected_sequential_events = [ 2391 &MetaEvent { 2392 event: "starting".to_string(), 2393 device_id: None, 2394 }, 2395 &MetaEvent { 2396 event: "booting".to_string(), 2397 device_id: None, 2398 }, 2399 &MetaEvent { 2400 event: "booted".to_string(), 2401 device_id: None, 2402 }, 2403 &MetaEvent { 2404 event: "activated".to_string(), 2405 device_id: Some("_disk0".to_string()), 2406 }, 2407 &MetaEvent { 2408 event: "reset".to_string(), 2409 device_id: Some("_disk0".to_string()), 2410 }, 2411 ]; 2412 assert!(check_sequential_events( 2413 &expected_sequential_events, 2414 &event_path 2415 )); 2416 2417 // It's been observed on the Bionic image that udev and snapd 2418 // services can cause some delay in the VM's shutdown. Disabling 2419 // them improves the reliability of this test. 2420 let _ = guest.ssh_command("sudo systemctl disable udev"); 2421 let _ = guest.ssh_command("sudo systemctl stop udev"); 2422 let _ = guest.ssh_command("sudo systemctl disable snapd"); 2423 let _ = guest.ssh_command("sudo systemctl stop snapd"); 2424 2425 guest.ssh_command("sudo poweroff").unwrap(); 2426 thread::sleep(std::time::Duration::new(20, 0)); 2427 let latest_events = [ 2428 &MetaEvent { 2429 event: "shutdown".to_string(), 2430 device_id: None, 2431 }, 2432 &MetaEvent { 2433 event: "deleted".to_string(), 2434 device_id: None, 2435 }, 2436 &MetaEvent { 2437 event: "shutdown".to_string(), 2438 device_id: None, 2439 }, 2440 ]; 2441 assert!(check_latest_events_exact(&latest_events, &event_path)); 2442 }); 2443 2444 kill_child(&mut child); 2445 let output = child.wait_with_output().unwrap(); 2446 2447 handle_child_output(r, &output); 2448 } 2449 2450 #[test] 2451 fn test_multi_cpu() { 2452 let jammy_image = JAMMY_IMAGE_NAME.to_string(); 2453 let jammy = UbuntuDiskConfig::new(jammy_image); 2454 let guest = Guest::new(Box::new(jammy)); 2455 2456 let mut cmd = GuestCommand::new(&guest); 2457 cmd.args(["--cpus", "boot=2,max=4"]) 2458 .args(["--memory", "size=512M"]) 2459 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2460 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2461 .capture_output() 2462 .default_disks() 2463 .default_net(); 2464 2465 let mut child = cmd.spawn().unwrap(); 2466 2467 let r = std::panic::catch_unwind(|| { 2468 guest.wait_vm_boot(Some(120)).unwrap(); 2469 2470 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 2471 2472 assert_eq!( 2473 guest 2474 .ssh_command( 2475 r#"sudo dmesg | grep "smp: Brought up" | sed "s/\[\ *[0-9.]*\] //""# 2476 ) 2477 .unwrap() 2478 .trim(), 2479 "smp: Brought up 1 node, 2 CPUs" 2480 ); 2481 }); 2482 2483 kill_child(&mut child); 2484 let output = child.wait_with_output().unwrap(); 2485 2486 handle_child_output(r, &output); 2487 } 2488 2489 #[test] 2490 fn test_cpu_topology_421() { 2491 test_cpu_topology(4, 2, 1, false); 2492 } 2493 2494 #[test] 2495 fn test_cpu_topology_142() { 2496 test_cpu_topology(1, 4, 2, false); 2497 } 2498 2499 #[test] 2500 fn test_cpu_topology_262() { 2501 test_cpu_topology(2, 6, 2, false); 2502 } 2503 2504 #[test] 2505 #[cfg(target_arch = "x86_64")] 2506 #[cfg(not(feature = "mshv"))] 2507 fn test_cpu_physical_bits() { 2508 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2509 let guest = Guest::new(Box::new(focal)); 2510 let max_phys_bits: u8 = 36; 2511 let mut child = GuestCommand::new(&guest) 2512 .args(["--cpus", &format!("max_phys_bits={max_phys_bits}")]) 2513 .args(["--memory", "size=512M"]) 2514 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2515 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2516 .default_disks() 2517 .default_net() 2518 .capture_output() 2519 .spawn() 2520 .unwrap(); 2521 2522 let r = std::panic::catch_unwind(|| { 2523 guest.wait_vm_boot(None).unwrap(); 2524 2525 assert!( 2526 guest 2527 .ssh_command("lscpu | grep \"Address sizes:\" | cut -f 2 -d \":\" | sed \"s# *##\" | cut -f 1 -d \" \"") 2528 .unwrap() 2529 .trim() 2530 .parse::<u8>() 2531 .unwrap_or(max_phys_bits + 1) <= max_phys_bits, 2532 ); 2533 }); 2534 2535 kill_child(&mut child); 2536 let output = child.wait_with_output().unwrap(); 2537 2538 handle_child_output(r, &output); 2539 } 2540 2541 #[test] 2542 fn test_cpu_affinity() { 2543 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2544 let guest = Guest::new(Box::new(focal)); 2545 2546 // We need the host to have at least 4 CPUs if we want to be able 2547 // to run this test. 2548 let host_cpus_count = exec_host_command_output("nproc"); 2549 assert!( 2550 String::from_utf8_lossy(&host_cpus_count.stdout) 2551 .trim() 2552 .parse::<u16>() 2553 .unwrap_or(0) 2554 >= 4 2555 ); 2556 2557 let mut child = GuestCommand::new(&guest) 2558 .args(["--cpus", "boot=2,affinity=[0@[0,2],1@[1,3]]"]) 2559 .args(["--memory", "size=512M"]) 2560 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2561 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2562 .default_disks() 2563 .default_net() 2564 .capture_output() 2565 .spawn() 2566 .unwrap(); 2567 2568 let r = std::panic::catch_unwind(|| { 2569 guest.wait_vm_boot(None).unwrap(); 2570 let pid = child.id(); 2571 let taskset_vcpu0 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep vcpu0 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); 2572 assert_eq!(String::from_utf8_lossy(&taskset_vcpu0.stdout).trim(), "0,2"); 2573 let taskset_vcpu1 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep vcpu1 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); 2574 assert_eq!(String::from_utf8_lossy(&taskset_vcpu1.stdout).trim(), "1,3"); 2575 }); 2576 2577 kill_child(&mut child); 2578 let output = child.wait_with_output().unwrap(); 2579 handle_child_output(r, &output); 2580 } 2581 2582 #[test] 2583 fn test_virtio_queue_affinity() { 2584 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2585 let guest = Guest::new(Box::new(focal)); 2586 2587 // We need the host to have at least 4 CPUs if we want to be able 2588 // to run this test. 2589 let host_cpus_count = exec_host_command_output("nproc"); 2590 assert!( 2591 String::from_utf8_lossy(&host_cpus_count.stdout) 2592 .trim() 2593 .parse::<u16>() 2594 .unwrap_or(0) 2595 >= 4 2596 ); 2597 2598 let mut child = GuestCommand::new(&guest) 2599 .args(["--cpus", "boot=4"]) 2600 .args(["--memory", "size=512M"]) 2601 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2602 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2603 .args([ 2604 "--disk", 2605 format!( 2606 "path={}", 2607 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 2608 ) 2609 .as_str(), 2610 format!( 2611 "path={},num_queues=4,queue_affinity=[0@[0,2],1@[1,3],2@[1],3@[3]]", 2612 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2613 ) 2614 .as_str(), 2615 ]) 2616 .default_net() 2617 .capture_output() 2618 .spawn() 2619 .unwrap(); 2620 2621 let r = std::panic::catch_unwind(|| { 2622 guest.wait_vm_boot(None).unwrap(); 2623 let pid = child.id(); 2624 let taskset_q0 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep disk1_q0 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); 2625 assert_eq!(String::from_utf8_lossy(&taskset_q0.stdout).trim(), "0,2"); 2626 let taskset_q1 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep disk1_q1 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); 2627 assert_eq!(String::from_utf8_lossy(&taskset_q1.stdout).trim(), "1,3"); 2628 let taskset_q2 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep disk1_q2 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); 2629 assert_eq!(String::from_utf8_lossy(&taskset_q2.stdout).trim(), "1"); 2630 let taskset_q3 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep disk1_q3 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str()); 2631 assert_eq!(String::from_utf8_lossy(&taskset_q3.stdout).trim(), "3"); 2632 }); 2633 2634 kill_child(&mut child); 2635 let output = child.wait_with_output().unwrap(); 2636 handle_child_output(r, &output); 2637 } 2638 2639 #[test] 2640 #[cfg(not(feature = "mshv"))] 2641 fn test_large_vm() { 2642 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2643 let guest = Guest::new(Box::new(focal)); 2644 let mut cmd = GuestCommand::new(&guest); 2645 cmd.args(["--cpus", "boot=48"]) 2646 .args(["--memory", "size=5120M"]) 2647 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2648 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2649 .args(["--serial", "tty"]) 2650 .args(["--console", "off"]) 2651 .capture_output() 2652 .default_disks() 2653 .default_net(); 2654 2655 let mut child = cmd.spawn().unwrap(); 2656 2657 guest.wait_vm_boot(None).unwrap(); 2658 2659 let r = std::panic::catch_unwind(|| { 2660 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 48); 2661 assert_eq!( 2662 guest 2663 .ssh_command("lscpu | grep \"On-line\" | cut -f 2 -d \":\" | sed \"s# *##\"") 2664 .unwrap() 2665 .trim(), 2666 "0-47" 2667 ); 2668 2669 assert!(guest.get_total_memory().unwrap_or_default() > 5_000_000); 2670 }); 2671 2672 kill_child(&mut child); 2673 let output = child.wait_with_output().unwrap(); 2674 2675 handle_child_output(r, &output); 2676 } 2677 2678 #[test] 2679 #[cfg(not(feature = "mshv"))] 2680 fn test_huge_memory() { 2681 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2682 let guest = Guest::new(Box::new(focal)); 2683 let mut cmd = GuestCommand::new(&guest); 2684 cmd.args(["--cpus", "boot=1"]) 2685 .args(["--memory", "size=128G"]) 2686 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2687 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2688 .capture_output() 2689 .default_disks() 2690 .default_net(); 2691 2692 let mut child = cmd.spawn().unwrap(); 2693 2694 guest.wait_vm_boot(Some(120)).unwrap(); 2695 2696 let r = std::panic::catch_unwind(|| { 2697 assert!(guest.get_total_memory().unwrap_or_default() > 128_000_000); 2698 }); 2699 2700 kill_child(&mut child); 2701 let output = child.wait_with_output().unwrap(); 2702 2703 handle_child_output(r, &output); 2704 } 2705 2706 #[test] 2707 fn test_power_button() { 2708 _test_power_button(false); 2709 } 2710 2711 #[test] 2712 #[cfg(not(feature = "mshv"))] 2713 fn test_user_defined_memory_regions() { 2714 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2715 let guest = Guest::new(Box::new(focal)); 2716 let api_socket = temp_api_path(&guest.tmp_dir); 2717 2718 let kernel_path = direct_kernel_boot_path(); 2719 2720 let mut child = GuestCommand::new(&guest) 2721 .args(["--cpus", "boot=1"]) 2722 .args(["--memory", "size=0,hotplug_method=virtio-mem"]) 2723 .args([ 2724 "--memory-zone", 2725 "id=mem0,size=1G,hotplug_size=2G", 2726 "id=mem1,size=1G,shared=on", 2727 "id=mem2,size=1G,host_numa_node=0,hotplug_size=2G", 2728 ]) 2729 .args(["--kernel", kernel_path.to_str().unwrap()]) 2730 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2731 .args(["--api-socket", &api_socket]) 2732 .capture_output() 2733 .default_disks() 2734 .default_net() 2735 .spawn() 2736 .unwrap(); 2737 2738 let r = std::panic::catch_unwind(|| { 2739 guest.wait_vm_boot(None).unwrap(); 2740 2741 assert!(guest.get_total_memory().unwrap_or_default() > 2_880_000); 2742 2743 guest.enable_memory_hotplug(); 2744 2745 resize_zone_command(&api_socket, "mem0", "3G"); 2746 thread::sleep(std::time::Duration::new(5, 0)); 2747 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2748 resize_zone_command(&api_socket, "mem2", "3G"); 2749 thread::sleep(std::time::Duration::new(5, 0)); 2750 assert!(guest.get_total_memory().unwrap_or_default() > 6_720_000); 2751 resize_zone_command(&api_socket, "mem0", "2G"); 2752 thread::sleep(std::time::Duration::new(5, 0)); 2753 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 2754 resize_zone_command(&api_socket, "mem2", "2G"); 2755 thread::sleep(std::time::Duration::new(5, 0)); 2756 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2757 2758 guest.reboot_linux(0, None); 2759 2760 // Check the amount of RAM after reboot 2761 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2762 assert!(guest.get_total_memory().unwrap_or_default() < 5_760_000); 2763 2764 // Check if we can still resize down to the initial 'boot'size 2765 resize_zone_command(&api_socket, "mem0", "1G"); 2766 thread::sleep(std::time::Duration::new(5, 0)); 2767 assert!(guest.get_total_memory().unwrap_or_default() < 4_800_000); 2768 resize_zone_command(&api_socket, "mem2", "1G"); 2769 thread::sleep(std::time::Duration::new(5, 0)); 2770 assert!(guest.get_total_memory().unwrap_or_default() < 3_840_000); 2771 }); 2772 2773 kill_child(&mut child); 2774 let output = child.wait_with_output().unwrap(); 2775 2776 handle_child_output(r, &output); 2777 } 2778 2779 #[test] 2780 #[cfg(not(feature = "mshv"))] 2781 fn test_guest_numa_nodes() { 2782 _test_guest_numa_nodes(false); 2783 } 2784 2785 #[test] 2786 #[cfg(target_arch = "x86_64")] 2787 fn test_iommu_segments() { 2788 let focal_image = FOCAL_IMAGE_NAME.to_string(); 2789 let focal = UbuntuDiskConfig::new(focal_image); 2790 let guest = Guest::new(Box::new(focal)); 2791 2792 // Prepare another disk file for the virtio-disk device 2793 let test_disk_path = String::from( 2794 guest 2795 .tmp_dir 2796 .as_path() 2797 .join("test-disk.raw") 2798 .to_str() 2799 .unwrap(), 2800 ); 2801 assert!( 2802 exec_host_command_status(format!("truncate {test_disk_path} -s 4M").as_str()).success() 2803 ); 2804 assert!(exec_host_command_status(format!("mkfs.ext4 {test_disk_path}").as_str()).success()); 2805 2806 let api_socket = temp_api_path(&guest.tmp_dir); 2807 let mut cmd = GuestCommand::new(&guest); 2808 2809 cmd.args(["--cpus", "boot=1"]) 2810 .args(["--api-socket", &api_socket]) 2811 .args(["--memory", "size=512M"]) 2812 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2813 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2814 .args([ 2815 "--platform", 2816 &format!("num_pci_segments={MAX_NUM_PCI_SEGMENTS},iommu_segments=[1]"), 2817 ]) 2818 .default_disks() 2819 .capture_output() 2820 .default_net(); 2821 2822 let mut child = cmd.spawn().unwrap(); 2823 2824 guest.wait_vm_boot(None).unwrap(); 2825 2826 let r = std::panic::catch_unwind(|| { 2827 let (cmd_success, cmd_output) = remote_command_w_output( 2828 &api_socket, 2829 "add-disk", 2830 Some( 2831 format!( 2832 "path={},id=test0,pci_segment=1,iommu=on", 2833 test_disk_path.as_str() 2834 ) 2835 .as_str(), 2836 ), 2837 ); 2838 assert!(cmd_success); 2839 assert!(String::from_utf8_lossy(&cmd_output) 2840 .contains("{\"id\":\"test0\",\"bdf\":\"0001:00:01.0\"}")); 2841 2842 // Check IOMMU setup 2843 assert!(guest 2844 .does_device_vendor_pair_match("0x1057", "0x1af4") 2845 .unwrap_or_default()); 2846 assert_eq!( 2847 guest 2848 .ssh_command("ls /sys/kernel/iommu_groups/0/devices") 2849 .unwrap() 2850 .trim(), 2851 "0001:00:01.0" 2852 ); 2853 }); 2854 2855 kill_child(&mut child); 2856 let output = child.wait_with_output().unwrap(); 2857 2858 handle_child_output(r, &output); 2859 } 2860 2861 #[test] 2862 fn test_pci_msi() { 2863 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2864 let guest = Guest::new(Box::new(focal)); 2865 let mut cmd = GuestCommand::new(&guest); 2866 cmd.args(["--cpus", "boot=1"]) 2867 .args(["--memory", "size=512M"]) 2868 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2869 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2870 .capture_output() 2871 .default_disks() 2872 .default_net(); 2873 2874 let mut child = cmd.spawn().unwrap(); 2875 2876 guest.wait_vm_boot(None).unwrap(); 2877 2878 #[cfg(target_arch = "x86_64")] 2879 let grep_cmd = "grep -c PCI-MSI /proc/interrupts"; 2880 #[cfg(target_arch = "aarch64")] 2881 let grep_cmd = "grep -c ITS-PCI-MSIX /proc/interrupts"; 2882 2883 let r = std::panic::catch_unwind(|| { 2884 assert_eq!( 2885 guest 2886 .ssh_command(grep_cmd) 2887 .unwrap() 2888 .trim() 2889 .parse::<u32>() 2890 .unwrap_or_default(), 2891 12 2892 ); 2893 }); 2894 2895 kill_child(&mut child); 2896 let output = child.wait_with_output().unwrap(); 2897 2898 handle_child_output(r, &output); 2899 } 2900 2901 #[test] 2902 fn test_virtio_net_ctrl_queue() { 2903 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2904 let guest = Guest::new(Box::new(focal)); 2905 let mut cmd = GuestCommand::new(&guest); 2906 cmd.args(["--cpus", "boot=1"]) 2907 .args(["--memory", "size=512M"]) 2908 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2909 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2910 .args(["--net", guest.default_net_string_w_mtu(3000).as_str()]) 2911 .capture_output() 2912 .default_disks(); 2913 2914 let mut child = cmd.spawn().unwrap(); 2915 2916 guest.wait_vm_boot(None).unwrap(); 2917 2918 #[cfg(target_arch = "aarch64")] 2919 let iface = "enp0s4"; 2920 #[cfg(target_arch = "x86_64")] 2921 let iface = "ens4"; 2922 2923 let r = std::panic::catch_unwind(|| { 2924 assert_eq!( 2925 guest 2926 .ssh_command( 2927 format!("sudo ethtool -K {iface} rx-gro-hw off && echo success").as_str() 2928 ) 2929 .unwrap() 2930 .trim(), 2931 "success" 2932 ); 2933 assert_eq!( 2934 guest 2935 .ssh_command(format!("cat /sys/class/net/{iface}/mtu").as_str()) 2936 .unwrap() 2937 .trim(), 2938 "3000" 2939 ); 2940 }); 2941 2942 kill_child(&mut child); 2943 let output = child.wait_with_output().unwrap(); 2944 2945 handle_child_output(r, &output); 2946 } 2947 2948 #[test] 2949 fn test_pci_multiple_segments() { 2950 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2951 let guest = Guest::new(Box::new(focal)); 2952 2953 // Prepare another disk file for the virtio-disk device 2954 let test_disk_path = String::from( 2955 guest 2956 .tmp_dir 2957 .as_path() 2958 .join("test-disk.raw") 2959 .to_str() 2960 .unwrap(), 2961 ); 2962 assert!( 2963 exec_host_command_status(format!("truncate {test_disk_path} -s 4M").as_str()).success() 2964 ); 2965 assert!(exec_host_command_status(format!("mkfs.ext4 {test_disk_path}").as_str()).success()); 2966 2967 let mut cmd = GuestCommand::new(&guest); 2968 cmd.args(["--cpus", "boot=1"]) 2969 .args(["--memory", "size=512M"]) 2970 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2971 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2972 .args([ 2973 "--platform", 2974 &format!("num_pci_segments={MAX_NUM_PCI_SEGMENTS}"), 2975 ]) 2976 .args([ 2977 "--disk", 2978 format!( 2979 "path={}", 2980 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 2981 ) 2982 .as_str(), 2983 format!( 2984 "path={}", 2985 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2986 ) 2987 .as_str(), 2988 format!("path={test_disk_path},pci_segment=15").as_str(), 2989 ]) 2990 .capture_output() 2991 .default_net(); 2992 2993 let mut child = cmd.spawn().unwrap(); 2994 2995 guest.wait_vm_boot(None).unwrap(); 2996 2997 let grep_cmd = "lspci | grep \"Host bridge\" | wc -l"; 2998 2999 let r = std::panic::catch_unwind(|| { 3000 // There should be MAX_NUM_PCI_SEGMENTS PCI host bridges in the guest. 3001 assert_eq!( 3002 guest 3003 .ssh_command(grep_cmd) 3004 .unwrap() 3005 .trim() 3006 .parse::<u16>() 3007 .unwrap_or_default(), 3008 MAX_NUM_PCI_SEGMENTS 3009 ); 3010 3011 // Check both if /dev/vdc exists and if the block size is 4M. 3012 assert_eq!( 3013 guest 3014 .ssh_command("lsblk | grep vdc | grep -c 4M") 3015 .unwrap() 3016 .trim() 3017 .parse::<u32>() 3018 .unwrap_or_default(), 3019 1 3020 ); 3021 3022 // Mount the device. 3023 guest.ssh_command("mkdir mount_image").unwrap(); 3024 guest 3025 .ssh_command("sudo mount -o rw -t ext4 /dev/vdc mount_image/") 3026 .unwrap(); 3027 // Grant all users with write permission. 3028 guest.ssh_command("sudo chmod a+w mount_image/").unwrap(); 3029 3030 // Write something to the device. 3031 guest 3032 .ssh_command("sudo echo \"bar\" >> mount_image/foo") 3033 .unwrap(); 3034 3035 // Check the content of the block device. The file "foo" should 3036 // contain "bar". 3037 assert_eq!( 3038 guest 3039 .ssh_command("sudo cat mount_image/foo") 3040 .unwrap() 3041 .trim(), 3042 "bar" 3043 ); 3044 }); 3045 3046 kill_child(&mut child); 3047 let output = child.wait_with_output().unwrap(); 3048 3049 handle_child_output(r, &output); 3050 } 3051 3052 #[test] 3053 fn test_pci_multiple_segments_numa_node() { 3054 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3055 let guest = Guest::new(Box::new(focal)); 3056 let api_socket = temp_api_path(&guest.tmp_dir); 3057 #[cfg(target_arch = "x86_64")] 3058 let kernel_path = direct_kernel_boot_path(); 3059 #[cfg(target_arch = "aarch64")] 3060 let kernel_path = edk2_path(); 3061 3062 // Prepare another disk file for the virtio-disk device 3063 let test_disk_path = String::from( 3064 guest 3065 .tmp_dir 3066 .as_path() 3067 .join("test-disk.raw") 3068 .to_str() 3069 .unwrap(), 3070 ); 3071 assert!( 3072 exec_host_command_status(format!("truncate {test_disk_path} -s 4M").as_str()).success() 3073 ); 3074 assert!(exec_host_command_status(format!("mkfs.ext4 {test_disk_path}").as_str()).success()); 3075 const TEST_DISK_NODE: u16 = 1; 3076 3077 let mut child = GuestCommand::new(&guest) 3078 .args(["--platform", "num_pci_segments=2"]) 3079 .args(["--cpus", "boot=2"]) 3080 .args(["--memory", "size=0"]) 3081 .args(["--memory-zone", "id=mem0,size=256M", "id=mem1,size=256M"]) 3082 .args([ 3083 "--numa", 3084 "guest_numa_id=0,cpus=[0],distances=[1@20],memory_zones=mem0,pci_segments=[0]", 3085 "guest_numa_id=1,cpus=[1],distances=[0@20],memory_zones=mem1,pci_segments=[1]", 3086 ]) 3087 .args(["--kernel", kernel_path.to_str().unwrap()]) 3088 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3089 .args(["--api-socket", &api_socket]) 3090 .capture_output() 3091 .args([ 3092 "--disk", 3093 format!( 3094 "path={}", 3095 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 3096 ) 3097 .as_str(), 3098 format!( 3099 "path={}", 3100 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3101 ) 3102 .as_str(), 3103 format!("path={test_disk_path},pci_segment={TEST_DISK_NODE}").as_str(), 3104 ]) 3105 .default_net() 3106 .spawn() 3107 .unwrap(); 3108 3109 let cmd = "cat /sys/block/vdc/device/../numa_node"; 3110 3111 let r = std::panic::catch_unwind(|| { 3112 guest.wait_vm_boot(None).unwrap(); 3113 3114 assert_eq!( 3115 guest 3116 .ssh_command(cmd) 3117 .unwrap() 3118 .trim() 3119 .parse::<u16>() 3120 .unwrap_or_default(), 3121 TEST_DISK_NODE 3122 ); 3123 }); 3124 3125 kill_child(&mut child); 3126 let output = child.wait_with_output().unwrap(); 3127 3128 handle_child_output(r, &output); 3129 } 3130 3131 #[test] 3132 fn test_direct_kernel_boot() { 3133 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3134 let guest = Guest::new(Box::new(focal)); 3135 3136 let kernel_path = direct_kernel_boot_path(); 3137 3138 let mut child = GuestCommand::new(&guest) 3139 .args(["--cpus", "boot=1"]) 3140 .args(["--memory", "size=512M"]) 3141 .args(["--kernel", kernel_path.to_str().unwrap()]) 3142 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3143 .default_disks() 3144 .default_net() 3145 .capture_output() 3146 .spawn() 3147 .unwrap(); 3148 3149 let r = std::panic::catch_unwind(|| { 3150 guest.wait_vm_boot(None).unwrap(); 3151 3152 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 3153 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 3154 3155 let grep_cmd = if cfg!(target_arch = "x86_64") { 3156 "grep -c PCI-MSI /proc/interrupts" 3157 } else { 3158 "grep -c ITS-PCI-MSIX /proc/interrupts" 3159 }; 3160 assert_eq!( 3161 guest 3162 .ssh_command(grep_cmd) 3163 .unwrap() 3164 .trim() 3165 .parse::<u32>() 3166 .unwrap_or_default(), 3167 12 3168 ); 3169 }); 3170 3171 kill_child(&mut child); 3172 let output = child.wait_with_output().unwrap(); 3173 3174 handle_child_output(r, &output); 3175 } 3176 3177 #[test] 3178 #[cfg(target_arch = "x86_64")] 3179 fn test_direct_kernel_boot_bzimage() { 3180 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3181 let guest = Guest::new(Box::new(focal)); 3182 3183 let mut kernel_path = direct_kernel_boot_path(); 3184 // Replace the default kernel with the bzImage. 3185 kernel_path.pop(); 3186 kernel_path.push("bzImage"); 3187 3188 let mut child = GuestCommand::new(&guest) 3189 .args(["--cpus", "boot=1"]) 3190 .args(["--memory", "size=512M"]) 3191 .args(["--kernel", kernel_path.to_str().unwrap()]) 3192 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3193 .default_disks() 3194 .default_net() 3195 .capture_output() 3196 .spawn() 3197 .unwrap(); 3198 3199 let r = std::panic::catch_unwind(|| { 3200 guest.wait_vm_boot(None).unwrap(); 3201 3202 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 3203 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 3204 3205 let grep_cmd = if cfg!(target_arch = "x86_64") { 3206 "grep -c PCI-MSI /proc/interrupts" 3207 } else { 3208 "grep -c ITS-PCI-MSIX /proc/interrupts" 3209 }; 3210 assert_eq!( 3211 guest 3212 .ssh_command(grep_cmd) 3213 .unwrap() 3214 .trim() 3215 .parse::<u32>() 3216 .unwrap_or_default(), 3217 12 3218 ); 3219 }); 3220 3221 kill_child(&mut child); 3222 let output = child.wait_with_output().unwrap(); 3223 3224 handle_child_output(r, &output); 3225 } 3226 3227 fn _test_virtio_block(image_name: &str, disable_io_uring: bool, disable_aio: bool) { 3228 let focal = UbuntuDiskConfig::new(image_name.to_string()); 3229 let guest = Guest::new(Box::new(focal)); 3230 3231 let mut workload_path = dirs::home_dir().unwrap(); 3232 workload_path.push("workloads"); 3233 3234 let mut blk_file_path = workload_path; 3235 blk_file_path.push("blk.img"); 3236 3237 let kernel_path = direct_kernel_boot_path(); 3238 3239 let mut cloud_child = GuestCommand::new(&guest) 3240 .args(["--cpus", "boot=4"]) 3241 .args(["--memory", "size=512M,shared=on"]) 3242 .args(["--kernel", kernel_path.to_str().unwrap()]) 3243 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3244 .args([ 3245 "--disk", 3246 format!( 3247 "path={}", 3248 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 3249 ) 3250 .as_str(), 3251 format!( 3252 "path={}", 3253 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3254 ) 3255 .as_str(), 3256 format!( 3257 "path={},readonly=on,direct=on,num_queues=4,_disable_io_uring={},_disable_aio={}", 3258 blk_file_path.to_str().unwrap(), 3259 disable_io_uring, 3260 disable_aio, 3261 ) 3262 .as_str(), 3263 ]) 3264 .default_net() 3265 .capture_output() 3266 .spawn() 3267 .unwrap(); 3268 3269 let r = std::panic::catch_unwind(|| { 3270 guest.wait_vm_boot(None).unwrap(); 3271 3272 // Check both if /dev/vdc exists and if the block size is 16M. 3273 assert_eq!( 3274 guest 3275 .ssh_command("lsblk | grep vdc | grep -c 16M") 3276 .unwrap() 3277 .trim() 3278 .parse::<u32>() 3279 .unwrap_or_default(), 3280 1 3281 ); 3282 3283 // Check both if /dev/vdc exists and if this block is RO. 3284 assert_eq!( 3285 guest 3286 .ssh_command("lsblk | grep vdc | awk '{print $5}'") 3287 .unwrap() 3288 .trim() 3289 .parse::<u32>() 3290 .unwrap_or_default(), 3291 1 3292 ); 3293 3294 // Check if the number of queues is 4. 3295 assert_eq!( 3296 guest 3297 .ssh_command("ls -ll /sys/block/vdc/mq | grep ^d | wc -l") 3298 .unwrap() 3299 .trim() 3300 .parse::<u32>() 3301 .unwrap_or_default(), 3302 4 3303 ); 3304 }); 3305 3306 let _ = cloud_child.kill(); 3307 let output = cloud_child.wait_with_output().unwrap(); 3308 3309 handle_child_output(r, &output); 3310 } 3311 3312 #[test] 3313 fn test_virtio_block_io_uring() { 3314 _test_virtio_block(FOCAL_IMAGE_NAME, false, true) 3315 } 3316 3317 #[test] 3318 fn test_virtio_block_aio() { 3319 _test_virtio_block(FOCAL_IMAGE_NAME, true, false) 3320 } 3321 3322 #[test] 3323 fn test_virtio_block_sync() { 3324 _test_virtio_block(FOCAL_IMAGE_NAME, true, true) 3325 } 3326 3327 #[test] 3328 fn test_virtio_block_qcow2() { 3329 _test_virtio_block(FOCAL_IMAGE_NAME_QCOW2, false, false) 3330 } 3331 3332 #[test] 3333 fn test_virtio_block_qcow2_backing_file() { 3334 _test_virtio_block(FOCAL_IMAGE_NAME_QCOW2_BACKING_FILE, false, false) 3335 } 3336 3337 #[test] 3338 fn test_virtio_block_vhd() { 3339 let mut workload_path = dirs::home_dir().unwrap(); 3340 workload_path.push("workloads"); 3341 3342 let mut raw_file_path = workload_path.clone(); 3343 let mut vhd_file_path = workload_path; 3344 raw_file_path.push(FOCAL_IMAGE_NAME); 3345 vhd_file_path.push(FOCAL_IMAGE_NAME_VHD); 3346 3347 // Generate VHD file from RAW file 3348 std::process::Command::new("qemu-img") 3349 .arg("convert") 3350 .arg("-p") 3351 .args(["-f", "raw"]) 3352 .args(["-O", "vpc"]) 3353 .args(["-o", "subformat=fixed"]) 3354 .arg(raw_file_path.to_str().unwrap()) 3355 .arg(vhd_file_path.to_str().unwrap()) 3356 .output() 3357 .expect("Expect generating VHD image from RAW image"); 3358 3359 _test_virtio_block(FOCAL_IMAGE_NAME_VHD, false, false) 3360 } 3361 3362 #[test] 3363 fn test_virtio_block_vhdx() { 3364 let mut workload_path = dirs::home_dir().unwrap(); 3365 workload_path.push("workloads"); 3366 3367 let mut raw_file_path = workload_path.clone(); 3368 let mut vhdx_file_path = workload_path; 3369 raw_file_path.push(FOCAL_IMAGE_NAME); 3370 vhdx_file_path.push(FOCAL_IMAGE_NAME_VHDX); 3371 3372 // Generate dynamic VHDX file from RAW file 3373 std::process::Command::new("qemu-img") 3374 .arg("convert") 3375 .arg("-p") 3376 .args(["-f", "raw"]) 3377 .args(["-O", "vhdx"]) 3378 .arg(raw_file_path.to_str().unwrap()) 3379 .arg(vhdx_file_path.to_str().unwrap()) 3380 .output() 3381 .expect("Expect generating dynamic VHDx image from RAW image"); 3382 3383 _test_virtio_block(FOCAL_IMAGE_NAME_VHDX, false, false) 3384 } 3385 3386 #[test] 3387 fn test_virtio_block_dynamic_vhdx_expand() { 3388 const VIRTUAL_DISK_SIZE: u64 = 100 << 20; 3389 const EMPTY_VHDX_FILE_SIZE: u64 = 8 << 20; 3390 const FULL_VHDX_FILE_SIZE: u64 = 112 << 20; 3391 const DYNAMIC_VHDX_NAME: &str = "dynamic.vhdx"; 3392 3393 let mut workload_path = dirs::home_dir().unwrap(); 3394 workload_path.push("workloads"); 3395 3396 let mut vhdx_file_path = workload_path; 3397 vhdx_file_path.push(DYNAMIC_VHDX_NAME); 3398 let vhdx_path = vhdx_file_path.to_str().unwrap(); 3399 3400 // Generate a 100 MiB dynamic VHDX file 3401 std::process::Command::new("qemu-img") 3402 .arg("create") 3403 .args(["-f", "vhdx"]) 3404 .arg(vhdx_path) 3405 .arg(VIRTUAL_DISK_SIZE.to_string()) 3406 .output() 3407 .expect("Expect generating dynamic VHDx image from RAW image"); 3408 3409 // Check if the size matches with empty VHDx file size 3410 assert_eq!(vhdx_image_size(vhdx_path), EMPTY_VHDX_FILE_SIZE); 3411 3412 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3413 let guest = Guest::new(Box::new(focal)); 3414 let kernel_path = direct_kernel_boot_path(); 3415 3416 let mut cloud_child = GuestCommand::new(&guest) 3417 .args(["--cpus", "boot=1"]) 3418 .args(["--memory", "size=512M"]) 3419 .args(["--kernel", kernel_path.to_str().unwrap()]) 3420 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3421 .args([ 3422 "--disk", 3423 format!( 3424 "path={}", 3425 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 3426 ) 3427 .as_str(), 3428 format!( 3429 "path={}", 3430 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3431 ) 3432 .as_str(), 3433 format!("path={vhdx_path}").as_str(), 3434 ]) 3435 .default_net() 3436 .capture_output() 3437 .spawn() 3438 .unwrap(); 3439 3440 let r = std::panic::catch_unwind(|| { 3441 guest.wait_vm_boot(None).unwrap(); 3442 3443 // Check both if /dev/vdc exists and if the block size is 100 MiB. 3444 assert_eq!( 3445 guest 3446 .ssh_command("lsblk | grep vdc | grep -c 100M") 3447 .unwrap() 3448 .trim() 3449 .parse::<u32>() 3450 .unwrap_or_default(), 3451 1 3452 ); 3453 3454 // Write 100 MB of data to the VHDx disk 3455 guest 3456 .ssh_command("sudo dd if=/dev/urandom of=/dev/vdc bs=1M count=100") 3457 .unwrap(); 3458 }); 3459 3460 // Check if the size matches with expected expanded VHDx file size 3461 assert_eq!(vhdx_image_size(vhdx_path), FULL_VHDX_FILE_SIZE); 3462 3463 kill_child(&mut cloud_child); 3464 let output = cloud_child.wait_with_output().unwrap(); 3465 3466 handle_child_output(r, &output); 3467 } 3468 3469 fn vhdx_image_size(disk_name: &str) -> u64 { 3470 std::fs::File::open(disk_name) 3471 .unwrap() 3472 .seek(SeekFrom::End(0)) 3473 .unwrap() 3474 } 3475 3476 #[test] 3477 fn test_virtio_block_direct_and_firmware() { 3478 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3479 let guest = Guest::new(Box::new(focal)); 3480 3481 // The OS disk must be copied to a location that is not backed by 3482 // tmpfs, otherwise the syscall openat(2) with O_DIRECT simply fails 3483 // with EINVAL because tmpfs doesn't support this flag. 3484 let mut workloads_path = dirs::home_dir().unwrap(); 3485 workloads_path.push("workloads"); 3486 let os_dir = TempDir::new_in(workloads_path.as_path()).unwrap(); 3487 let mut os_path = os_dir.as_path().to_path_buf(); 3488 os_path.push("osdisk.img"); 3489 rate_limited_copy( 3490 guest.disk_config.disk(DiskType::OperatingSystem).unwrap(), 3491 os_path.as_path(), 3492 ) 3493 .expect("copying of OS disk failed"); 3494 3495 let mut child = GuestCommand::new(&guest) 3496 .args(["--cpus", "boot=1"]) 3497 .args(["--memory", "size=512M"]) 3498 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 3499 .args([ 3500 "--disk", 3501 format!("path={},direct=on", os_path.as_path().to_str().unwrap()).as_str(), 3502 format!( 3503 "path={}", 3504 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3505 ) 3506 .as_str(), 3507 ]) 3508 .default_net() 3509 .capture_output() 3510 .spawn() 3511 .unwrap(); 3512 3513 let r = std::panic::catch_unwind(|| { 3514 guest.wait_vm_boot(Some(120)).unwrap(); 3515 }); 3516 3517 kill_child(&mut child); 3518 let output = child.wait_with_output().unwrap(); 3519 3520 handle_child_output(r, &output); 3521 } 3522 3523 #[test] 3524 fn test_vhost_user_net_default() { 3525 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, false, false) 3526 } 3527 3528 #[test] 3529 fn test_vhost_user_net_named_tap() { 3530 test_vhost_user_net( 3531 Some("mytap0"), 3532 2, 3533 &prepare_vhost_user_net_daemon, 3534 false, 3535 false, 3536 ) 3537 } 3538 3539 #[test] 3540 fn test_vhost_user_net_existing_tap() { 3541 test_vhost_user_net( 3542 Some("vunet-tap0"), 3543 2, 3544 &prepare_vhost_user_net_daemon, 3545 false, 3546 false, 3547 ) 3548 } 3549 3550 #[test] 3551 fn test_vhost_user_net_multiple_queues() { 3552 test_vhost_user_net(None, 4, &prepare_vhost_user_net_daemon, false, false) 3553 } 3554 3555 #[test] 3556 fn test_vhost_user_net_tap_multiple_queues() { 3557 test_vhost_user_net( 3558 Some("vunet-tap1"), 3559 4, 3560 &prepare_vhost_user_net_daemon, 3561 false, 3562 false, 3563 ) 3564 } 3565 3566 #[test] 3567 fn test_vhost_user_net_host_mac() { 3568 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, true, false) 3569 } 3570 3571 #[test] 3572 fn test_vhost_user_net_client_mode() { 3573 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, false, true) 3574 } 3575 3576 #[test] 3577 #[cfg(not(target_arch = "aarch64"))] 3578 fn test_vhost_user_blk_default() { 3579 test_vhost_user_blk(2, false, false, Some(&prepare_vubd)) 3580 } 3581 3582 #[test] 3583 #[cfg(not(target_arch = "aarch64"))] 3584 fn test_vhost_user_blk_readonly() { 3585 test_vhost_user_blk(1, true, false, Some(&prepare_vubd)) 3586 } 3587 3588 #[test] 3589 #[cfg(not(target_arch = "aarch64"))] 3590 fn test_vhost_user_blk_direct() { 3591 test_vhost_user_blk(1, false, true, Some(&prepare_vubd)) 3592 } 3593 3594 #[test] 3595 fn test_boot_from_vhost_user_blk_default() { 3596 test_boot_from_vhost_user_blk(1, false, false, Some(&prepare_vubd)) 3597 } 3598 3599 #[test] 3600 #[cfg(target_arch = "x86_64")] 3601 fn test_split_irqchip() { 3602 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3603 let guest = Guest::new(Box::new(focal)); 3604 3605 let mut child = GuestCommand::new(&guest) 3606 .args(["--cpus", "boot=1"]) 3607 .args(["--memory", "size=512M"]) 3608 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3609 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3610 .default_disks() 3611 .default_net() 3612 .capture_output() 3613 .spawn() 3614 .unwrap(); 3615 3616 let r = std::panic::catch_unwind(|| { 3617 guest.wait_vm_boot(None).unwrap(); 3618 3619 assert_eq!( 3620 guest 3621 .ssh_command("grep -c IO-APIC.*timer /proc/interrupts || true") 3622 .unwrap() 3623 .trim() 3624 .parse::<u32>() 3625 .unwrap_or(1), 3626 0 3627 ); 3628 assert_eq!( 3629 guest 3630 .ssh_command("grep -c IO-APIC.*cascade /proc/interrupts || true") 3631 .unwrap() 3632 .trim() 3633 .parse::<u32>() 3634 .unwrap_or(1), 3635 0 3636 ); 3637 }); 3638 3639 kill_child(&mut child); 3640 let output = child.wait_with_output().unwrap(); 3641 3642 handle_child_output(r, &output); 3643 } 3644 3645 #[test] 3646 #[cfg(target_arch = "x86_64")] 3647 fn test_dmi_serial_number() { 3648 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3649 let guest = Guest::new(Box::new(focal)); 3650 3651 let mut child = GuestCommand::new(&guest) 3652 .args(["--cpus", "boot=1"]) 3653 .args(["--memory", "size=512M"]) 3654 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3655 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3656 .args(["--platform", "serial_number=a=b;c=d"]) 3657 .default_disks() 3658 .default_net() 3659 .capture_output() 3660 .spawn() 3661 .unwrap(); 3662 3663 let r = std::panic::catch_unwind(|| { 3664 guest.wait_vm_boot(None).unwrap(); 3665 3666 assert_eq!( 3667 guest 3668 .ssh_command("sudo cat /sys/class/dmi/id/product_serial") 3669 .unwrap() 3670 .trim(), 3671 "a=b;c=d" 3672 ); 3673 }); 3674 3675 kill_child(&mut child); 3676 let output = child.wait_with_output().unwrap(); 3677 3678 handle_child_output(r, &output); 3679 } 3680 3681 #[test] 3682 #[cfg(target_arch = "x86_64")] 3683 fn test_dmi_uuid() { 3684 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3685 let guest = Guest::new(Box::new(focal)); 3686 3687 let mut child = GuestCommand::new(&guest) 3688 .args(["--cpus", "boot=1"]) 3689 .args(["--memory", "size=512M"]) 3690 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3691 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3692 .args(["--platform", "uuid=1e8aa28a-435d-4027-87f4-40dceff1fa0a"]) 3693 .default_disks() 3694 .default_net() 3695 .capture_output() 3696 .spawn() 3697 .unwrap(); 3698 3699 let r = std::panic::catch_unwind(|| { 3700 guest.wait_vm_boot(None).unwrap(); 3701 3702 assert_eq!( 3703 guest 3704 .ssh_command("sudo cat /sys/class/dmi/id/product_uuid") 3705 .unwrap() 3706 .trim(), 3707 "1e8aa28a-435d-4027-87f4-40dceff1fa0a" 3708 ); 3709 }); 3710 3711 kill_child(&mut child); 3712 let output = child.wait_with_output().unwrap(); 3713 3714 handle_child_output(r, &output); 3715 } 3716 3717 #[test] 3718 #[cfg(target_arch = "x86_64")] 3719 fn test_dmi_oem_strings() { 3720 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3721 let guest = Guest::new(Box::new(focal)); 3722 3723 let s1 = "io.systemd.credential:xx=yy"; 3724 let s2 = "This is a test string"; 3725 3726 let oem_strings = format!("oem_strings=[{s1},{s2}]"); 3727 3728 let mut child = GuestCommand::new(&guest) 3729 .args(["--cpus", "boot=1"]) 3730 .args(["--memory", "size=512M"]) 3731 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3732 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3733 .args(["--platform", &oem_strings]) 3734 .default_disks() 3735 .default_net() 3736 .capture_output() 3737 .spawn() 3738 .unwrap(); 3739 3740 let r = std::panic::catch_unwind(|| { 3741 guest.wait_vm_boot(None).unwrap(); 3742 3743 assert_eq!( 3744 guest 3745 .ssh_command("sudo dmidecode --oem-string count") 3746 .unwrap() 3747 .trim(), 3748 "2" 3749 ); 3750 3751 assert_eq!( 3752 guest 3753 .ssh_command("sudo dmidecode --oem-string 1") 3754 .unwrap() 3755 .trim(), 3756 s1 3757 ); 3758 3759 assert_eq!( 3760 guest 3761 .ssh_command("sudo dmidecode --oem-string 2") 3762 .unwrap() 3763 .trim(), 3764 s2 3765 ); 3766 }); 3767 3768 kill_child(&mut child); 3769 let output = child.wait_with_output().unwrap(); 3770 3771 handle_child_output(r, &output); 3772 } 3773 3774 #[test] 3775 fn test_virtio_fs() { 3776 _test_virtio_fs(&prepare_virtiofsd, false, None) 3777 } 3778 3779 #[test] 3780 fn test_virtio_fs_hotplug() { 3781 _test_virtio_fs(&prepare_virtiofsd, true, None) 3782 } 3783 3784 #[test] 3785 #[cfg(not(feature = "mshv"))] 3786 fn test_virtio_fs_multi_segment_hotplug() { 3787 _test_virtio_fs(&prepare_virtiofsd, true, Some(15)) 3788 } 3789 3790 #[test] 3791 #[cfg(not(feature = "mshv"))] 3792 fn test_virtio_fs_multi_segment() { 3793 _test_virtio_fs(&prepare_virtiofsd, false, Some(15)) 3794 } 3795 3796 #[test] 3797 fn test_virtio_pmem_discard_writes() { 3798 test_virtio_pmem(true, false) 3799 } 3800 3801 #[test] 3802 fn test_virtio_pmem_with_size() { 3803 test_virtio_pmem(true, true) 3804 } 3805 3806 #[test] 3807 fn test_boot_from_virtio_pmem() { 3808 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3809 let guest = Guest::new(Box::new(focal)); 3810 3811 let kernel_path = direct_kernel_boot_path(); 3812 3813 let mut child = GuestCommand::new(&guest) 3814 .args(["--cpus", "boot=1"]) 3815 .args(["--memory", "size=512M"]) 3816 .args(["--kernel", kernel_path.to_str().unwrap()]) 3817 .args([ 3818 "--disk", 3819 format!( 3820 "path={}", 3821 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3822 ) 3823 .as_str(), 3824 ]) 3825 .default_net() 3826 .args([ 3827 "--pmem", 3828 format!( 3829 "file={},size={}", 3830 guest.disk_config.disk(DiskType::OperatingSystem).unwrap(), 3831 fs::metadata(guest.disk_config.disk(DiskType::OperatingSystem).unwrap()) 3832 .unwrap() 3833 .len() 3834 ) 3835 .as_str(), 3836 ]) 3837 .args([ 3838 "--cmdline", 3839 DIRECT_KERNEL_BOOT_CMDLINE 3840 .replace("vda1", "pmem0p1") 3841 .as_str(), 3842 ]) 3843 .capture_output() 3844 .spawn() 3845 .unwrap(); 3846 3847 let r = std::panic::catch_unwind(|| { 3848 guest.wait_vm_boot(None).unwrap(); 3849 3850 // Simple checks to validate the VM booted properly 3851 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 3852 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 3853 }); 3854 3855 kill_child(&mut child); 3856 let output = child.wait_with_output().unwrap(); 3857 3858 handle_child_output(r, &output); 3859 } 3860 3861 #[test] 3862 fn test_multiple_network_interfaces() { 3863 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3864 let guest = Guest::new(Box::new(focal)); 3865 3866 let kernel_path = direct_kernel_boot_path(); 3867 3868 let mut child = GuestCommand::new(&guest) 3869 .args(["--cpus", "boot=1"]) 3870 .args(["--memory", "size=512M"]) 3871 .args(["--kernel", kernel_path.to_str().unwrap()]) 3872 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3873 .default_disks() 3874 .args([ 3875 "--net", 3876 guest.default_net_string().as_str(), 3877 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 3878 "tap=mytap1,mac=fe:1f:9e:e1:60:f2,ip=192.168.4.1,mask=255.255.255.0", 3879 ]) 3880 .capture_output() 3881 .spawn() 3882 .unwrap(); 3883 3884 let r = std::panic::catch_unwind(|| { 3885 guest.wait_vm_boot(None).unwrap(); 3886 3887 let tap_count = exec_host_command_output("ip link | grep -c mytap1"); 3888 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 3889 3890 // 3 network interfaces + default localhost ==> 4 interfaces 3891 assert_eq!( 3892 guest 3893 .ssh_command("ip -o link | wc -l") 3894 .unwrap() 3895 .trim() 3896 .parse::<u32>() 3897 .unwrap_or_default(), 3898 4 3899 ); 3900 }); 3901 3902 kill_child(&mut child); 3903 let output = child.wait_with_output().unwrap(); 3904 3905 handle_child_output(r, &output); 3906 } 3907 3908 #[test] 3909 #[cfg(target_arch = "aarch64")] 3910 fn test_pmu_on() { 3911 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3912 let guest = Guest::new(Box::new(focal)); 3913 let mut child = GuestCommand::new(&guest) 3914 .args(["--cpus", "boot=1"]) 3915 .args(["--memory", "size=512M"]) 3916 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3917 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3918 .default_disks() 3919 .default_net() 3920 .capture_output() 3921 .spawn() 3922 .unwrap(); 3923 3924 let r = std::panic::catch_unwind(|| { 3925 guest.wait_vm_boot(None).unwrap(); 3926 3927 // Test that PMU exists. 3928 assert_eq!( 3929 guest 3930 .ssh_command(GREP_PMU_IRQ_CMD) 3931 .unwrap() 3932 .trim() 3933 .parse::<u32>() 3934 .unwrap_or_default(), 3935 1 3936 ); 3937 }); 3938 3939 kill_child(&mut child); 3940 let output = child.wait_with_output().unwrap(); 3941 3942 handle_child_output(r, &output); 3943 } 3944 3945 #[test] 3946 fn test_serial_off() { 3947 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3948 let guest = Guest::new(Box::new(focal)); 3949 let mut child = GuestCommand::new(&guest) 3950 .args(["--cpus", "boot=1"]) 3951 .args(["--memory", "size=512M"]) 3952 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3953 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3954 .default_disks() 3955 .default_net() 3956 .args(["--serial", "off"]) 3957 .capture_output() 3958 .spawn() 3959 .unwrap(); 3960 3961 let r = std::panic::catch_unwind(|| { 3962 guest.wait_vm_boot(None).unwrap(); 3963 3964 // Test that there is no ttyS0 3965 assert_eq!( 3966 guest 3967 .ssh_command(GREP_SERIAL_IRQ_CMD) 3968 .unwrap() 3969 .trim() 3970 .parse::<u32>() 3971 .unwrap_or(1), 3972 0 3973 ); 3974 }); 3975 3976 kill_child(&mut child); 3977 let output = child.wait_with_output().unwrap(); 3978 3979 handle_child_output(r, &output); 3980 } 3981 3982 #[test] 3983 fn test_serial_null() { 3984 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3985 let guest = Guest::new(Box::new(focal)); 3986 let mut cmd = GuestCommand::new(&guest); 3987 #[cfg(target_arch = "x86_64")] 3988 let console_str: &str = "console=ttyS0"; 3989 #[cfg(target_arch = "aarch64")] 3990 let console_str: &str = "console=ttyAMA0"; 3991 3992 cmd.args(["--cpus", "boot=1"]) 3993 .args(["--memory", "size=512M"]) 3994 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3995 .args([ 3996 "--cmdline", 3997 DIRECT_KERNEL_BOOT_CMDLINE 3998 .replace("console=hvc0 ", console_str) 3999 .as_str(), 4000 ]) 4001 .default_disks() 4002 .default_net() 4003 .args(["--serial", "null"]) 4004 .args(["--console", "off"]) 4005 .capture_output(); 4006 4007 let mut child = cmd.spawn().unwrap(); 4008 4009 let r = std::panic::catch_unwind(|| { 4010 guest.wait_vm_boot(None).unwrap(); 4011 4012 // Test that there is a ttyS0 4013 assert_eq!( 4014 guest 4015 .ssh_command(GREP_SERIAL_IRQ_CMD) 4016 .unwrap() 4017 .trim() 4018 .parse::<u32>() 4019 .unwrap_or_default(), 4020 1 4021 ); 4022 }); 4023 4024 kill_child(&mut child); 4025 let output = child.wait_with_output().unwrap(); 4026 handle_child_output(r, &output); 4027 4028 let r = std::panic::catch_unwind(|| { 4029 assert!(!String::from_utf8_lossy(&output.stdout).contains(CONSOLE_TEST_STRING)); 4030 }); 4031 4032 handle_child_output(r, &output); 4033 } 4034 4035 #[test] 4036 fn test_serial_tty() { 4037 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4038 let guest = Guest::new(Box::new(focal)); 4039 4040 let kernel_path = direct_kernel_boot_path(); 4041 4042 #[cfg(target_arch = "x86_64")] 4043 let console_str: &str = "console=ttyS0"; 4044 #[cfg(target_arch = "aarch64")] 4045 let console_str: &str = "console=ttyAMA0"; 4046 4047 let mut child = GuestCommand::new(&guest) 4048 .args(["--cpus", "boot=1"]) 4049 .args(["--memory", "size=512M"]) 4050 .args(["--kernel", kernel_path.to_str().unwrap()]) 4051 .args([ 4052 "--cmdline", 4053 DIRECT_KERNEL_BOOT_CMDLINE 4054 .replace("console=hvc0 ", console_str) 4055 .as_str(), 4056 ]) 4057 .default_disks() 4058 .default_net() 4059 .args(["--serial", "tty"]) 4060 .args(["--console", "off"]) 4061 .capture_output() 4062 .spawn() 4063 .unwrap(); 4064 4065 let r = std::panic::catch_unwind(|| { 4066 guest.wait_vm_boot(None).unwrap(); 4067 4068 // Test that there is a ttyS0 4069 assert_eq!( 4070 guest 4071 .ssh_command(GREP_SERIAL_IRQ_CMD) 4072 .unwrap() 4073 .trim() 4074 .parse::<u32>() 4075 .unwrap_or_default(), 4076 1 4077 ); 4078 }); 4079 4080 // This sleep is needed to wait for the login prompt 4081 thread::sleep(std::time::Duration::new(2, 0)); 4082 4083 kill_child(&mut child); 4084 let output = child.wait_with_output().unwrap(); 4085 handle_child_output(r, &output); 4086 4087 let r = std::panic::catch_unwind(|| { 4088 assert!(String::from_utf8_lossy(&output.stdout).contains(CONSOLE_TEST_STRING)); 4089 }); 4090 4091 handle_child_output(r, &output); 4092 } 4093 4094 #[test] 4095 fn test_serial_file() { 4096 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4097 let guest = Guest::new(Box::new(focal)); 4098 4099 let serial_path = guest.tmp_dir.as_path().join("serial-output"); 4100 #[cfg(target_arch = "x86_64")] 4101 let console_str: &str = "console=ttyS0"; 4102 #[cfg(target_arch = "aarch64")] 4103 let console_str: &str = "console=ttyAMA0"; 4104 4105 let mut child = GuestCommand::new(&guest) 4106 .args(["--cpus", "boot=1"]) 4107 .args(["--memory", "size=512M"]) 4108 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 4109 .args([ 4110 "--cmdline", 4111 DIRECT_KERNEL_BOOT_CMDLINE 4112 .replace("console=hvc0 ", console_str) 4113 .as_str(), 4114 ]) 4115 .default_disks() 4116 .default_net() 4117 .args([ 4118 "--serial", 4119 format!("file={}", serial_path.to_str().unwrap()).as_str(), 4120 ]) 4121 .capture_output() 4122 .spawn() 4123 .unwrap(); 4124 4125 let r = std::panic::catch_unwind(|| { 4126 guest.wait_vm_boot(None).unwrap(); 4127 4128 // Test that there is a ttyS0 4129 assert_eq!( 4130 guest 4131 .ssh_command(GREP_SERIAL_IRQ_CMD) 4132 .unwrap() 4133 .trim() 4134 .parse::<u32>() 4135 .unwrap_or_default(), 4136 1 4137 ); 4138 4139 guest.ssh_command("sudo shutdown -h now").unwrap(); 4140 }); 4141 4142 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 4143 kill_child(&mut child); 4144 let output = child.wait_with_output().unwrap(); 4145 handle_child_output(r, &output); 4146 4147 let r = std::panic::catch_unwind(|| { 4148 // Check that the cloud-hypervisor binary actually terminated 4149 assert!(output.status.success()); 4150 4151 // Do this check after shutdown of the VM as an easy way to ensure 4152 // all writes are flushed to disk 4153 let mut f = std::fs::File::open(serial_path).unwrap(); 4154 let mut buf = String::new(); 4155 f.read_to_string(&mut buf).unwrap(); 4156 assert!(buf.contains(CONSOLE_TEST_STRING)); 4157 }); 4158 4159 handle_child_output(r, &output); 4160 } 4161 4162 #[test] 4163 fn test_pty_interaction() { 4164 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4165 let guest = Guest::new(Box::new(focal)); 4166 let api_socket = temp_api_path(&guest.tmp_dir); 4167 let serial_option = if cfg!(target_arch = "x86_64") { 4168 " console=ttyS0" 4169 } else { 4170 " console=ttyAMA0" 4171 }; 4172 let cmdline = DIRECT_KERNEL_BOOT_CMDLINE.to_owned() + serial_option; 4173 4174 let mut child = GuestCommand::new(&guest) 4175 .args(["--cpus", "boot=1"]) 4176 .args(["--memory", "size=512M"]) 4177 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 4178 .args(["--cmdline", &cmdline]) 4179 .default_disks() 4180 .default_net() 4181 .args(["--serial", "null"]) 4182 .args(["--console", "pty"]) 4183 .args(["--api-socket", &api_socket]) 4184 .spawn() 4185 .unwrap(); 4186 4187 let r = std::panic::catch_unwind(|| { 4188 guest.wait_vm_boot(None).unwrap(); 4189 // Get pty fd for console 4190 let console_path = get_pty_path(&api_socket, "console"); 4191 _test_pty_interaction(console_path); 4192 4193 guest.ssh_command("sudo shutdown -h now").unwrap(); 4194 }); 4195 4196 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 4197 let _ = child.kill(); 4198 let output = child.wait_with_output().unwrap(); 4199 handle_child_output(r, &output); 4200 4201 let r = std::panic::catch_unwind(|| { 4202 // Check that the cloud-hypervisor binary actually terminated 4203 assert!(output.status.success()) 4204 }); 4205 handle_child_output(r, &output); 4206 } 4207 4208 #[test] 4209 fn test_serial_socket_interaction() { 4210 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4211 let guest = Guest::new(Box::new(focal)); 4212 let serial_socket = guest.tmp_dir.as_path().join("serial.socket"); 4213 let serial_socket_pty = guest.tmp_dir.as_path().join("serial.pty"); 4214 let serial_option = if cfg!(target_arch = "x86_64") { 4215 " console=ttyS0" 4216 } else { 4217 " console=ttyAMA0" 4218 }; 4219 let cmdline = DIRECT_KERNEL_BOOT_CMDLINE.to_owned() + serial_option; 4220 4221 let mut child = GuestCommand::new(&guest) 4222 .args(["--cpus", "boot=1"]) 4223 .args(["--memory", "size=512M"]) 4224 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 4225 .args(["--cmdline", &cmdline]) 4226 .default_disks() 4227 .default_net() 4228 .args(["--console", "null"]) 4229 .args([ 4230 "--serial", 4231 format!("socket={}", serial_socket.to_str().unwrap()).as_str(), 4232 ]) 4233 .spawn() 4234 .unwrap(); 4235 4236 let _ = std::panic::catch_unwind(|| { 4237 guest.wait_vm_boot(None).unwrap(); 4238 }); 4239 4240 let mut socat_command = Command::new("socat"); 4241 let socat_args = [ 4242 &format!("pty,link={},raw", serial_socket_pty.display()), 4243 &format!("UNIX-CONNECT:{}", serial_socket.display()), 4244 ]; 4245 socat_command.args(socat_args); 4246 4247 let mut socat_child = socat_command.spawn().unwrap(); 4248 thread::sleep(std::time::Duration::new(1, 0)); 4249 4250 let _ = std::panic::catch_unwind(|| { 4251 _test_pty_interaction(serial_socket_pty); 4252 }); 4253 4254 let _ = socat_child.kill(); 4255 let _ = socat_child.wait(); 4256 4257 let r = std::panic::catch_unwind(|| { 4258 guest.ssh_command("sudo shutdown -h now").unwrap(); 4259 }); 4260 4261 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 4262 kill_child(&mut child); 4263 let output = child.wait_with_output().unwrap(); 4264 handle_child_output(r, &output); 4265 4266 let r = std::panic::catch_unwind(|| { 4267 // Check that the cloud-hypervisor binary actually terminated 4268 if !output.status.success() { 4269 panic!( 4270 "Cloud Hypervisor process failed to terminate gracefully: {:?}", 4271 output.status 4272 ); 4273 } 4274 }); 4275 handle_child_output(r, &output); 4276 } 4277 4278 #[test] 4279 fn test_virtio_console() { 4280 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4281 let guest = Guest::new(Box::new(focal)); 4282 4283 let kernel_path = direct_kernel_boot_path(); 4284 4285 let mut child = GuestCommand::new(&guest) 4286 .args(["--cpus", "boot=1"]) 4287 .args(["--memory", "size=512M"]) 4288 .args(["--kernel", kernel_path.to_str().unwrap()]) 4289 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4290 .default_disks() 4291 .default_net() 4292 .args(["--console", "tty"]) 4293 .args(["--serial", "null"]) 4294 .capture_output() 4295 .spawn() 4296 .unwrap(); 4297 4298 let text = String::from("On a branch floating down river a cricket, singing."); 4299 let cmd = format!("echo {text} | sudo tee /dev/hvc0"); 4300 4301 let r = std::panic::catch_unwind(|| { 4302 guest.wait_vm_boot(None).unwrap(); 4303 4304 assert!(guest 4305 .does_device_vendor_pair_match("0x1043", "0x1af4") 4306 .unwrap_or_default()); 4307 4308 guest.ssh_command(&cmd).unwrap(); 4309 }); 4310 4311 kill_child(&mut child); 4312 let output = child.wait_with_output().unwrap(); 4313 handle_child_output(r, &output); 4314 4315 let r = std::panic::catch_unwind(|| { 4316 assert!(String::from_utf8_lossy(&output.stdout).contains(&text)); 4317 }); 4318 4319 handle_child_output(r, &output); 4320 } 4321 4322 #[test] 4323 fn test_console_file() { 4324 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4325 let guest = Guest::new(Box::new(focal)); 4326 4327 let console_path = guest.tmp_dir.as_path().join("console-output"); 4328 let mut child = GuestCommand::new(&guest) 4329 .args(["--cpus", "boot=1"]) 4330 .args(["--memory", "size=512M"]) 4331 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 4332 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4333 .default_disks() 4334 .default_net() 4335 .args([ 4336 "--console", 4337 format!("file={}", console_path.to_str().unwrap()).as_str(), 4338 ]) 4339 .capture_output() 4340 .spawn() 4341 .unwrap(); 4342 4343 guest.wait_vm_boot(None).unwrap(); 4344 4345 guest.ssh_command("sudo shutdown -h now").unwrap(); 4346 4347 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 4348 kill_child(&mut child); 4349 let output = child.wait_with_output().unwrap(); 4350 4351 let r = std::panic::catch_unwind(|| { 4352 // Check that the cloud-hypervisor binary actually terminated 4353 assert!(output.status.success()); 4354 4355 // Do this check after shutdown of the VM as an easy way to ensure 4356 // all writes are flushed to disk 4357 let mut f = std::fs::File::open(console_path).unwrap(); 4358 let mut buf = String::new(); 4359 f.read_to_string(&mut buf).unwrap(); 4360 4361 if !buf.contains(CONSOLE_TEST_STRING) { 4362 eprintln!( 4363 "\n\n==== Console file output ====\n\n{buf}\n\n==== End console file output ====" 4364 ); 4365 } 4366 assert!(buf.contains(CONSOLE_TEST_STRING)); 4367 }); 4368 4369 handle_child_output(r, &output); 4370 } 4371 4372 #[test] 4373 #[cfg(target_arch = "x86_64")] 4374 #[cfg(not(feature = "mshv"))] 4375 // The VFIO integration test starts cloud-hypervisor guest with 3 TAP 4376 // backed networking interfaces, bound through a simple bridge on the host. 4377 // So if the nested cloud-hypervisor succeeds in getting a directly 4378 // assigned interface from its cloud-hypervisor host, we should be able to 4379 // ssh into it, and verify that it's running with the right kernel command 4380 // line (We tag the command line from cloud-hypervisor for that purpose). 4381 // The third device is added to validate that hotplug works correctly since 4382 // it is being added to the L2 VM through hotplugging mechanism. 4383 // Also, we pass-through a virtio-blk device to the L2 VM to test the 32-bit 4384 // vfio device support 4385 fn test_vfio() { 4386 setup_vfio_network_interfaces(); 4387 4388 let jammy = UbuntuDiskConfig::new(JAMMY_IMAGE_NAME.to_string()); 4389 let guest = Guest::new_from_ip_range(Box::new(jammy), "172.18", 0); 4390 4391 let mut workload_path = dirs::home_dir().unwrap(); 4392 workload_path.push("workloads"); 4393 4394 let kernel_path = direct_kernel_boot_path(); 4395 4396 let mut vfio_path = workload_path.clone(); 4397 vfio_path.push("vfio"); 4398 4399 let mut cloud_init_vfio_base_path = vfio_path.clone(); 4400 cloud_init_vfio_base_path.push("cloudinit.img"); 4401 4402 // We copy our cloudinit into the vfio mount point, for the nested 4403 // cloud-hypervisor guest to use. 4404 rate_limited_copy( 4405 guest.disk_config.disk(DiskType::CloudInit).unwrap(), 4406 &cloud_init_vfio_base_path, 4407 ) 4408 .expect("copying of cloud-init disk failed"); 4409 4410 let mut vfio_disk_path = workload_path.clone(); 4411 vfio_disk_path.push("vfio.img"); 4412 4413 // Create the vfio disk image 4414 let output = Command::new("mkfs.ext4") 4415 .arg("-d") 4416 .arg(vfio_path.to_str().unwrap()) 4417 .arg(vfio_disk_path.to_str().unwrap()) 4418 .arg("2g") 4419 .output() 4420 .unwrap(); 4421 if !output.status.success() { 4422 eprintln!("{}", String::from_utf8_lossy(&output.stderr)); 4423 panic!("mkfs.ext4 command generated an error"); 4424 } 4425 4426 let mut blk_file_path = workload_path; 4427 blk_file_path.push("blk.img"); 4428 4429 let vfio_tap0 = "vfio-tap0"; 4430 let vfio_tap1 = "vfio-tap1"; 4431 let vfio_tap2 = "vfio-tap2"; 4432 let vfio_tap3 = "vfio-tap3"; 4433 4434 let mut child = GuestCommand::new(&guest) 4435 .args(["--cpus", "boot=4"]) 4436 .args(["--memory", "size=2G,hugepages=on,shared=on"]) 4437 .args(["--kernel", kernel_path.to_str().unwrap()]) 4438 .args([ 4439 "--disk", 4440 format!( 4441 "path={}", 4442 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 4443 ) 4444 .as_str(), 4445 format!( 4446 "path={}", 4447 guest.disk_config.disk(DiskType::CloudInit).unwrap() 4448 ) 4449 .as_str(), 4450 format!("path={}", vfio_disk_path.to_str().unwrap()).as_str(), 4451 format!("path={},iommu=on", blk_file_path.to_str().unwrap()).as_str(), 4452 ]) 4453 .args([ 4454 "--cmdline", 4455 format!( 4456 "{DIRECT_KERNEL_BOOT_CMDLINE} kvm-intel.nested=1 vfio_iommu_type1.allow_unsafe_interrupts" 4457 ) 4458 .as_str(), 4459 ]) 4460 .args([ 4461 "--net", 4462 format!("tap={},mac={}", vfio_tap0, guest.network.guest_mac).as_str(), 4463 format!( 4464 "tap={},mac={},iommu=on", 4465 vfio_tap1, guest.network.l2_guest_mac1 4466 ) 4467 .as_str(), 4468 format!( 4469 "tap={},mac={},iommu=on", 4470 vfio_tap2, guest.network.l2_guest_mac2 4471 ) 4472 .as_str(), 4473 format!( 4474 "tap={},mac={},iommu=on", 4475 vfio_tap3, guest.network.l2_guest_mac3 4476 ) 4477 .as_str(), 4478 ]) 4479 .capture_output() 4480 .spawn() 4481 .unwrap(); 4482 4483 thread::sleep(std::time::Duration::new(30, 0)); 4484 4485 let r = std::panic::catch_unwind(|| { 4486 guest.ssh_command_l1("sudo systemctl start vfio").unwrap(); 4487 thread::sleep(std::time::Duration::new(120, 0)); 4488 4489 // We booted our cloud hypervisor L2 guest with a "VFIOTAG" tag 4490 // added to its kernel command line. 4491 // Let's ssh into it and verify that it's there. If it is it means 4492 // we're in the right guest (The L2 one) because the QEMU L1 guest 4493 // does not have this command line tag. 4494 assert!(check_matched_lines_count( 4495 guest.ssh_command_l2_1("cat /proc/cmdline").unwrap().trim(), 4496 vec!["VFIOTAG"], 4497 1 4498 )); 4499 4500 // Let's also verify from the second virtio-net device passed to 4501 // the L2 VM. 4502 assert!(check_matched_lines_count( 4503 guest.ssh_command_l2_2("cat /proc/cmdline").unwrap().trim(), 4504 vec!["VFIOTAG"], 4505 1 4506 )); 4507 4508 // Check the amount of PCI devices appearing in L2 VM. 4509 assert!(check_lines_count( 4510 guest 4511 .ssh_command_l2_1("ls /sys/bus/pci/devices") 4512 .unwrap() 4513 .trim(), 4514 8 4515 )); 4516 4517 // Check both if /dev/vdc exists and if the block size is 16M in L2 VM 4518 assert!(check_matched_lines_count( 4519 guest.ssh_command_l2_1("lsblk").unwrap().trim(), 4520 vec!["vdc", "16M"], 4521 1 4522 )); 4523 4524 // Hotplug an extra virtio-net device through L2 VM. 4525 guest 4526 .ssh_command_l1( 4527 "echo 0000:00:09.0 | sudo tee /sys/bus/pci/devices/0000:00:09.0/driver/unbind", 4528 ) 4529 .unwrap(); 4530 guest 4531 .ssh_command_l1("echo 0000:00:09.0 | sudo tee /sys/bus/pci/drivers/vfio-pci/bind") 4532 .unwrap(); 4533 let vfio_hotplug_output = guest 4534 .ssh_command_l1( 4535 "sudo /mnt/ch-remote \ 4536 --api-socket=/tmp/ch_api.sock \ 4537 add-device path=/sys/bus/pci/devices/0000:00:09.0,id=vfio123", 4538 ) 4539 .unwrap(); 4540 assert!(check_matched_lines_count( 4541 vfio_hotplug_output.trim(), 4542 vec!["{\"id\":\"vfio123\",\"bdf\":\"0000:00:08.0\"}"], 4543 1 4544 )); 4545 4546 thread::sleep(std::time::Duration::new(10, 0)); 4547 4548 // Let's also verify from the third virtio-net device passed to 4549 // the L2 VM. This third device has been hotplugged through the L2 4550 // VM, so this is our way to validate hotplug works for VFIO PCI. 4551 assert!(check_matched_lines_count( 4552 guest.ssh_command_l2_3("cat /proc/cmdline").unwrap().trim(), 4553 vec!["VFIOTAG"], 4554 1 4555 )); 4556 4557 // Check the amount of PCI devices appearing in L2 VM. 4558 // There should be one more device than before, raising the count 4559 // up to 9 PCI devices. 4560 assert!(check_lines_count( 4561 guest 4562 .ssh_command_l2_1("ls /sys/bus/pci/devices") 4563 .unwrap() 4564 .trim(), 4565 9 4566 )); 4567 4568 // Let's now verify that we can correctly remove the virtio-net 4569 // device through the "remove-device" command responsible for 4570 // unplugging VFIO devices. 4571 guest 4572 .ssh_command_l1( 4573 "sudo /mnt/ch-remote \ 4574 --api-socket=/tmp/ch_api.sock \ 4575 remove-device vfio123", 4576 ) 4577 .unwrap(); 4578 thread::sleep(std::time::Duration::new(10, 0)); 4579 4580 // Check the amount of PCI devices appearing in L2 VM is back down 4581 // to 8 devices. 4582 assert!(check_lines_count( 4583 guest 4584 .ssh_command_l2_1("ls /sys/bus/pci/devices") 4585 .unwrap() 4586 .trim(), 4587 8 4588 )); 4589 4590 // Perform memory hotplug in L2 and validate the memory is showing 4591 // up as expected. In order to check, we will use the virtio-net 4592 // device already passed through L2 as a VFIO device, this will 4593 // verify that VFIO devices are functional with memory hotplug. 4594 assert!(guest.get_total_memory_l2().unwrap_or_default() > 480_000); 4595 guest 4596 .ssh_command_l2_1( 4597 "sudo bash -c 'echo online > /sys/devices/system/memory/auto_online_blocks'", 4598 ) 4599 .unwrap(); 4600 guest 4601 .ssh_command_l1( 4602 "sudo /mnt/ch-remote \ 4603 --api-socket=/tmp/ch_api.sock \ 4604 resize --memory=1073741824", 4605 ) 4606 .unwrap(); 4607 assert!(guest.get_total_memory_l2().unwrap_or_default() > 960_000); 4608 }); 4609 4610 kill_child(&mut child); 4611 let output = child.wait_with_output().unwrap(); 4612 4613 cleanup_vfio_network_interfaces(); 4614 4615 handle_child_output(r, &output); 4616 } 4617 4618 #[test] 4619 fn test_direct_kernel_boot_noacpi() { 4620 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4621 let guest = Guest::new(Box::new(focal)); 4622 4623 let kernel_path = direct_kernel_boot_path(); 4624 4625 let mut child = GuestCommand::new(&guest) 4626 .args(["--cpus", "boot=1"]) 4627 .args(["--memory", "size=512M"]) 4628 .args(["--kernel", kernel_path.to_str().unwrap()]) 4629 .args([ 4630 "--cmdline", 4631 format!("{DIRECT_KERNEL_BOOT_CMDLINE} acpi=off").as_str(), 4632 ]) 4633 .default_disks() 4634 .default_net() 4635 .capture_output() 4636 .spawn() 4637 .unwrap(); 4638 4639 let r = std::panic::catch_unwind(|| { 4640 guest.wait_vm_boot(None).unwrap(); 4641 4642 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 4643 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4644 }); 4645 4646 kill_child(&mut child); 4647 let output = child.wait_with_output().unwrap(); 4648 4649 handle_child_output(r, &output); 4650 } 4651 4652 #[test] 4653 fn test_virtio_vsock() { 4654 _test_virtio_vsock(false) 4655 } 4656 4657 #[test] 4658 fn test_virtio_vsock_hotplug() { 4659 _test_virtio_vsock(true); 4660 } 4661 4662 #[test] 4663 fn test_api_http_shutdown() { 4664 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4665 let guest = Guest::new(Box::new(focal)); 4666 4667 _test_api_shutdown(TargetApi::new_http_api(&guest.tmp_dir), guest) 4668 } 4669 4670 #[test] 4671 fn test_api_http_delete() { 4672 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4673 let guest = Guest::new(Box::new(focal)); 4674 4675 _test_api_delete(TargetApi::new_http_api(&guest.tmp_dir), guest); 4676 } 4677 4678 #[test] 4679 fn test_api_http_pause_resume() { 4680 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4681 let guest = Guest::new(Box::new(focal)); 4682 4683 _test_api_pause_resume(TargetApi::new_http_api(&guest.tmp_dir), guest) 4684 } 4685 4686 #[test] 4687 fn test_api_http_create_boot() { 4688 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4689 let guest = Guest::new(Box::new(focal)); 4690 4691 _test_api_create_boot(TargetApi::new_http_api(&guest.tmp_dir), guest) 4692 } 4693 4694 #[test] 4695 fn test_virtio_iommu() { 4696 _test_virtio_iommu(cfg!(target_arch = "x86_64")) 4697 } 4698 4699 #[test] 4700 // We cannot force the software running in the guest to reprogram the BAR 4701 // with some different addresses, but we have a reliable way of testing it 4702 // with a standard Linux kernel. 4703 // By removing a device from the PCI tree, and then rescanning the tree, 4704 // Linux consistently chooses to reorganize the PCI device BARs to other 4705 // locations in the guest address space. 4706 // This test creates a dedicated PCI network device to be checked as being 4707 // properly probed first, then removing it, and adding it again by doing a 4708 // rescan. 4709 fn test_pci_bar_reprogramming() { 4710 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4711 let guest = Guest::new(Box::new(focal)); 4712 4713 #[cfg(target_arch = "x86_64")] 4714 let kernel_path = direct_kernel_boot_path(); 4715 #[cfg(target_arch = "aarch64")] 4716 let kernel_path = edk2_path(); 4717 4718 let mut child = GuestCommand::new(&guest) 4719 .args(["--cpus", "boot=1"]) 4720 .args(["--memory", "size=512M"]) 4721 .args(["--kernel", kernel_path.to_str().unwrap()]) 4722 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4723 .default_disks() 4724 .args([ 4725 "--net", 4726 guest.default_net_string().as_str(), 4727 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 4728 ]) 4729 .capture_output() 4730 .spawn() 4731 .unwrap(); 4732 4733 let r = std::panic::catch_unwind(|| { 4734 guest.wait_vm_boot(None).unwrap(); 4735 4736 // 2 network interfaces + default localhost ==> 3 interfaces 4737 assert_eq!( 4738 guest 4739 .ssh_command("ip -o link | wc -l") 4740 .unwrap() 4741 .trim() 4742 .parse::<u32>() 4743 .unwrap_or_default(), 4744 3 4745 ); 4746 4747 let init_bar_addr = guest 4748 .ssh_command( 4749 "sudo awk '{print $1; exit}' /sys/bus/pci/devices/0000:00:05.0/resource", 4750 ) 4751 .unwrap(); 4752 4753 // Remove the PCI device 4754 guest 4755 .ssh_command("echo 1 | sudo tee /sys/bus/pci/devices/0000:00:05.0/remove") 4756 .unwrap(); 4757 4758 // Only 1 network interface left + default localhost ==> 2 interfaces 4759 assert_eq!( 4760 guest 4761 .ssh_command("ip -o link | wc -l") 4762 .unwrap() 4763 .trim() 4764 .parse::<u32>() 4765 .unwrap_or_default(), 4766 2 4767 ); 4768 4769 // Remove the PCI device 4770 guest 4771 .ssh_command("echo 1 | sudo tee /sys/bus/pci/rescan") 4772 .unwrap(); 4773 4774 // Back to 2 network interface + default localhost ==> 3 interfaces 4775 assert_eq!( 4776 guest 4777 .ssh_command("ip -o link | wc -l") 4778 .unwrap() 4779 .trim() 4780 .parse::<u32>() 4781 .unwrap_or_default(), 4782 3 4783 ); 4784 4785 let new_bar_addr = guest 4786 .ssh_command( 4787 "sudo awk '{print $1; exit}' /sys/bus/pci/devices/0000:00:05.0/resource", 4788 ) 4789 .unwrap(); 4790 4791 // Let's compare the BAR addresses for our virtio-net device. 4792 // They should be different as we expect the BAR reprogramming 4793 // to have happened. 4794 assert_ne!(init_bar_addr, new_bar_addr); 4795 }); 4796 4797 kill_child(&mut child); 4798 let output = child.wait_with_output().unwrap(); 4799 4800 handle_child_output(r, &output); 4801 } 4802 4803 #[test] 4804 fn test_memory_mergeable_off() { 4805 test_memory_mergeable(false) 4806 } 4807 4808 #[test] 4809 #[cfg(target_arch = "x86_64")] 4810 fn test_cpu_hotplug() { 4811 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4812 let guest = Guest::new(Box::new(focal)); 4813 let api_socket = temp_api_path(&guest.tmp_dir); 4814 4815 let kernel_path = direct_kernel_boot_path(); 4816 4817 let mut child = GuestCommand::new(&guest) 4818 .args(["--cpus", "boot=2,max=4"]) 4819 .args(["--memory", "size=512M"]) 4820 .args(["--kernel", kernel_path.to_str().unwrap()]) 4821 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4822 .default_disks() 4823 .default_net() 4824 .args(["--api-socket", &api_socket]) 4825 .capture_output() 4826 .spawn() 4827 .unwrap(); 4828 4829 let r = std::panic::catch_unwind(|| { 4830 guest.wait_vm_boot(None).unwrap(); 4831 4832 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 4833 4834 // Resize the VM 4835 let desired_vcpus = 4; 4836 resize_command(&api_socket, Some(desired_vcpus), None, None, None); 4837 4838 guest 4839 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 4840 .unwrap(); 4841 guest 4842 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 4843 .unwrap(); 4844 thread::sleep(std::time::Duration::new(10, 0)); 4845 assert_eq!( 4846 guest.get_cpu_count().unwrap_or_default(), 4847 u32::from(desired_vcpus) 4848 ); 4849 4850 guest.reboot_linux(0, None); 4851 4852 assert_eq!( 4853 guest.get_cpu_count().unwrap_or_default(), 4854 u32::from(desired_vcpus) 4855 ); 4856 4857 // Resize the VM 4858 let desired_vcpus = 2; 4859 resize_command(&api_socket, Some(desired_vcpus), None, None, None); 4860 4861 thread::sleep(std::time::Duration::new(10, 0)); 4862 assert_eq!( 4863 guest.get_cpu_count().unwrap_or_default(), 4864 u32::from(desired_vcpus) 4865 ); 4866 4867 // Resize the VM back up to 4 4868 let desired_vcpus = 4; 4869 resize_command(&api_socket, Some(desired_vcpus), None, None, None); 4870 4871 guest 4872 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 4873 .unwrap(); 4874 guest 4875 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 4876 .unwrap(); 4877 thread::sleep(std::time::Duration::new(10, 0)); 4878 assert_eq!( 4879 guest.get_cpu_count().unwrap_or_default(), 4880 u32::from(desired_vcpus) 4881 ); 4882 }); 4883 4884 kill_child(&mut child); 4885 let output = child.wait_with_output().unwrap(); 4886 4887 handle_child_output(r, &output); 4888 } 4889 4890 #[test] 4891 fn test_memory_hotplug() { 4892 #[cfg(target_arch = "aarch64")] 4893 let focal_image = FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string(); 4894 #[cfg(target_arch = "x86_64")] 4895 let focal_image = FOCAL_IMAGE_NAME.to_string(); 4896 let focal = UbuntuDiskConfig::new(focal_image); 4897 let guest = Guest::new(Box::new(focal)); 4898 let api_socket = temp_api_path(&guest.tmp_dir); 4899 4900 #[cfg(target_arch = "aarch64")] 4901 let kernel_path = edk2_path(); 4902 #[cfg(target_arch = "x86_64")] 4903 let kernel_path = direct_kernel_boot_path(); 4904 4905 let mut child = GuestCommand::new(&guest) 4906 .args(["--cpus", "boot=2,max=4"]) 4907 .args(["--memory", "size=512M,hotplug_size=8192M"]) 4908 .args(["--kernel", kernel_path.to_str().unwrap()]) 4909 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4910 .default_disks() 4911 .default_net() 4912 .args(["--balloon", "size=0"]) 4913 .args(["--api-socket", &api_socket]) 4914 .capture_output() 4915 .spawn() 4916 .unwrap(); 4917 4918 let r = std::panic::catch_unwind(|| { 4919 guest.wait_vm_boot(None).unwrap(); 4920 4921 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4922 4923 guest.enable_memory_hotplug(); 4924 4925 // Add RAM to the VM 4926 let desired_ram = 1024 << 20; 4927 resize_command(&api_socket, None, Some(desired_ram), None, None); 4928 4929 thread::sleep(std::time::Duration::new(10, 0)); 4930 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4931 4932 // Use balloon to remove RAM from the VM 4933 let desired_balloon = 512 << 20; 4934 resize_command(&api_socket, None, None, Some(desired_balloon), None); 4935 4936 thread::sleep(std::time::Duration::new(10, 0)); 4937 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4938 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 4939 4940 guest.reboot_linux(0, None); 4941 4942 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 4943 4944 // Use balloon add RAM to the VM 4945 let desired_balloon = 0; 4946 resize_command(&api_socket, None, None, Some(desired_balloon), None); 4947 4948 thread::sleep(std::time::Duration::new(10, 0)); 4949 4950 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4951 4952 guest.enable_memory_hotplug(); 4953 4954 // Add RAM to the VM 4955 let desired_ram = 2048 << 20; 4956 resize_command(&api_socket, None, Some(desired_ram), None, None); 4957 4958 thread::sleep(std::time::Duration::new(10, 0)); 4959 assert!(guest.get_total_memory().unwrap_or_default() > 1_920_000); 4960 4961 // Remove RAM to the VM (only applies after reboot) 4962 let desired_ram = 1024 << 20; 4963 resize_command(&api_socket, None, Some(desired_ram), None, None); 4964 4965 guest.reboot_linux(1, None); 4966 4967 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4968 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 4969 }); 4970 4971 kill_child(&mut child); 4972 let output = child.wait_with_output().unwrap(); 4973 4974 handle_child_output(r, &output); 4975 } 4976 4977 #[test] 4978 #[cfg(not(feature = "mshv"))] 4979 fn test_virtio_mem() { 4980 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4981 let guest = Guest::new(Box::new(focal)); 4982 let api_socket = temp_api_path(&guest.tmp_dir); 4983 4984 let kernel_path = direct_kernel_boot_path(); 4985 4986 let mut child = GuestCommand::new(&guest) 4987 .args(["--cpus", "boot=2,max=4"]) 4988 .args([ 4989 "--memory", 4990 "size=512M,hotplug_method=virtio-mem,hotplug_size=8192M", 4991 ]) 4992 .args(["--kernel", kernel_path.to_str().unwrap()]) 4993 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4994 .default_disks() 4995 .default_net() 4996 .args(["--api-socket", &api_socket]) 4997 .capture_output() 4998 .spawn() 4999 .unwrap(); 5000 5001 let r = std::panic::catch_unwind(|| { 5002 guest.wait_vm_boot(None).unwrap(); 5003 5004 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 5005 5006 guest.enable_memory_hotplug(); 5007 5008 // Add RAM to the VM 5009 let desired_ram = 1024 << 20; 5010 resize_command(&api_socket, None, Some(desired_ram), None, None); 5011 5012 thread::sleep(std::time::Duration::new(10, 0)); 5013 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 5014 5015 // Add RAM to the VM 5016 let desired_ram = 2048 << 20; 5017 resize_command(&api_socket, None, Some(desired_ram), None, None); 5018 5019 thread::sleep(std::time::Duration::new(10, 0)); 5020 assert!(guest.get_total_memory().unwrap_or_default() > 1_920_000); 5021 5022 // Remove RAM from the VM 5023 let desired_ram = 1024 << 20; 5024 resize_command(&api_socket, None, Some(desired_ram), None, None); 5025 5026 thread::sleep(std::time::Duration::new(10, 0)); 5027 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 5028 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 5029 5030 guest.reboot_linux(0, None); 5031 5032 // Check the amount of memory after reboot is 1GiB 5033 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 5034 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 5035 5036 // Check we can still resize to 512MiB 5037 let desired_ram = 512 << 20; 5038 resize_command(&api_socket, None, Some(desired_ram), None, None); 5039 thread::sleep(std::time::Duration::new(10, 0)); 5040 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 5041 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 5042 }); 5043 5044 kill_child(&mut child); 5045 let output = child.wait_with_output().unwrap(); 5046 5047 handle_child_output(r, &output); 5048 } 5049 5050 #[test] 5051 #[cfg(target_arch = "x86_64")] 5052 #[cfg(not(feature = "mshv"))] 5053 // Test both vCPU and memory resizing together 5054 fn test_resize() { 5055 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5056 let guest = Guest::new(Box::new(focal)); 5057 let api_socket = temp_api_path(&guest.tmp_dir); 5058 5059 let kernel_path = direct_kernel_boot_path(); 5060 5061 let mut child = GuestCommand::new(&guest) 5062 .args(["--cpus", "boot=2,max=4"]) 5063 .args(["--memory", "size=512M,hotplug_size=8192M"]) 5064 .args(["--kernel", kernel_path.to_str().unwrap()]) 5065 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5066 .default_disks() 5067 .default_net() 5068 .args(["--api-socket", &api_socket]) 5069 .capture_output() 5070 .spawn() 5071 .unwrap(); 5072 5073 let r = std::panic::catch_unwind(|| { 5074 guest.wait_vm_boot(None).unwrap(); 5075 5076 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 5077 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 5078 5079 guest.enable_memory_hotplug(); 5080 5081 // Resize the VM 5082 let desired_vcpus = 4; 5083 let desired_ram = 1024 << 20; 5084 resize_command( 5085 &api_socket, 5086 Some(desired_vcpus), 5087 Some(desired_ram), 5088 None, 5089 None, 5090 ); 5091 5092 guest 5093 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 5094 .unwrap(); 5095 guest 5096 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 5097 .unwrap(); 5098 thread::sleep(std::time::Duration::new(10, 0)); 5099 assert_eq!( 5100 guest.get_cpu_count().unwrap_or_default(), 5101 u32::from(desired_vcpus) 5102 ); 5103 5104 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 5105 }); 5106 5107 kill_child(&mut child); 5108 let output = child.wait_with_output().unwrap(); 5109 5110 handle_child_output(r, &output); 5111 } 5112 5113 #[test] 5114 fn test_memory_overhead() { 5115 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5116 let guest = Guest::new(Box::new(focal)); 5117 5118 let kernel_path = direct_kernel_boot_path(); 5119 5120 let guest_memory_size_kb = 512 * 1024; 5121 5122 let mut child = GuestCommand::new(&guest) 5123 .args(["--cpus", "boot=1"]) 5124 .args(["--memory", format!("size={guest_memory_size_kb}K").as_str()]) 5125 .args(["--kernel", kernel_path.to_str().unwrap()]) 5126 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5127 .default_net() 5128 .default_disks() 5129 .capture_output() 5130 .spawn() 5131 .unwrap(); 5132 5133 guest.wait_vm_boot(None).unwrap(); 5134 5135 let r = std::panic::catch_unwind(|| { 5136 let overhead = get_vmm_overhead(child.id(), guest_memory_size_kb); 5137 eprintln!("Guest memory overhead: {overhead} vs {MAXIMUM_VMM_OVERHEAD_KB}"); 5138 assert!(overhead <= MAXIMUM_VMM_OVERHEAD_KB); 5139 }); 5140 5141 kill_child(&mut child); 5142 let output = child.wait_with_output().unwrap(); 5143 5144 handle_child_output(r, &output); 5145 } 5146 5147 #[test] 5148 #[cfg(target_arch = "x86_64")] 5149 // This test runs a guest with Landlock enabled and hotplugs a new disk. As 5150 // the path for the hotplug disk is not pre-added to Landlock rules, this 5151 // the test will result in a failure. 5152 fn test_landlock() { 5153 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5154 let guest = Guest::new(Box::new(focal)); 5155 5156 #[cfg(target_arch = "x86_64")] 5157 let kernel_path = direct_kernel_boot_path(); 5158 #[cfg(target_arch = "aarch64")] 5159 let kernel_path = edk2_path(); 5160 5161 let api_socket = temp_api_path(&guest.tmp_dir); 5162 5163 let mut child = GuestCommand::new(&guest) 5164 .args(["--api-socket", &api_socket]) 5165 .args(["--cpus", "boot=1"]) 5166 .args(["--memory", "size=512M"]) 5167 .args(["--kernel", kernel_path.to_str().unwrap()]) 5168 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5169 .args(["--landlock"]) 5170 .default_disks() 5171 .default_net() 5172 .capture_output() 5173 .spawn() 5174 .unwrap(); 5175 5176 let r = std::panic::catch_unwind(|| { 5177 guest.wait_vm_boot(None).unwrap(); 5178 5179 // Check /dev/vdc is not there 5180 assert_eq!( 5181 guest 5182 .ssh_command("lsblk | grep -c vdc.*16M || true") 5183 .unwrap() 5184 .trim() 5185 .parse::<u32>() 5186 .unwrap_or(1), 5187 0 5188 ); 5189 5190 // Now let's add the extra disk. 5191 let mut blk_file_path = dirs::home_dir().unwrap(); 5192 blk_file_path.push("workloads"); 5193 blk_file_path.push("blk.img"); 5194 // As the path to the hotplug disk is not pre-added, this remote 5195 // command will fail. 5196 assert!(!remote_command( 5197 &api_socket, 5198 "add-disk", 5199 Some(format!("path={},id=test0", blk_file_path.to_str().unwrap()).as_str()), 5200 )); 5201 }); 5202 5203 let _ = child.kill(); 5204 let output = child.wait_with_output().unwrap(); 5205 5206 handle_child_output(r, &output); 5207 } 5208 5209 fn _test_disk_hotplug(landlock_enabled: bool) { 5210 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5211 let guest = Guest::new(Box::new(focal)); 5212 5213 #[cfg(target_arch = "x86_64")] 5214 let kernel_path = direct_kernel_boot_path(); 5215 #[cfg(target_arch = "aarch64")] 5216 let kernel_path = edk2_path(); 5217 5218 let api_socket = temp_api_path(&guest.tmp_dir); 5219 5220 let mut blk_file_path = dirs::home_dir().unwrap(); 5221 blk_file_path.push("workloads"); 5222 blk_file_path.push("blk.img"); 5223 5224 let mut cmd = GuestCommand::new(&guest); 5225 if landlock_enabled { 5226 cmd.args(["--landlock"]).args([ 5227 "--landlock-rules", 5228 format!("path={:?},access=rw", blk_file_path).as_str(), 5229 ]); 5230 } 5231 5232 cmd.args(["--api-socket", &api_socket]) 5233 .args(["--cpus", "boot=1"]) 5234 .args(["--memory", "size=512M"]) 5235 .args(["--kernel", kernel_path.to_str().unwrap()]) 5236 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5237 .default_disks() 5238 .default_net() 5239 .capture_output(); 5240 5241 let mut child = cmd.spawn().unwrap(); 5242 5243 let r = std::panic::catch_unwind(|| { 5244 guest.wait_vm_boot(None).unwrap(); 5245 5246 // Check /dev/vdc is not there 5247 assert_eq!( 5248 guest 5249 .ssh_command("lsblk | grep -c vdc.*16M || true") 5250 .unwrap() 5251 .trim() 5252 .parse::<u32>() 5253 .unwrap_or(1), 5254 0 5255 ); 5256 5257 // Now let's add the extra disk. 5258 let (cmd_success, cmd_output) = remote_command_w_output( 5259 &api_socket, 5260 "add-disk", 5261 Some(format!("path={},id=test0", blk_file_path.to_str().unwrap()).as_str()), 5262 ); 5263 assert!(cmd_success); 5264 assert!(String::from_utf8_lossy(&cmd_output) 5265 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 5266 5267 thread::sleep(std::time::Duration::new(10, 0)); 5268 5269 // Check that /dev/vdc exists and the block size is 16M. 5270 assert_eq!( 5271 guest 5272 .ssh_command("lsblk | grep vdc | grep -c 16M") 5273 .unwrap() 5274 .trim() 5275 .parse::<u32>() 5276 .unwrap_or_default(), 5277 1 5278 ); 5279 // And check the block device can be read. 5280 guest 5281 .ssh_command("sudo dd if=/dev/vdc of=/dev/null bs=1M iflag=direct count=16") 5282 .unwrap(); 5283 5284 // Let's remove it the extra disk. 5285 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 5286 thread::sleep(std::time::Duration::new(5, 0)); 5287 // And check /dev/vdc is not there 5288 assert_eq!( 5289 guest 5290 .ssh_command("lsblk | grep -c vdc.*16M || true") 5291 .unwrap() 5292 .trim() 5293 .parse::<u32>() 5294 .unwrap_or(1), 5295 0 5296 ); 5297 5298 // And add it back to validate unplug did work correctly. 5299 let (cmd_success, cmd_output) = remote_command_w_output( 5300 &api_socket, 5301 "add-disk", 5302 Some(format!("path={},id=test0", blk_file_path.to_str().unwrap()).as_str()), 5303 ); 5304 assert!(cmd_success); 5305 assert!(String::from_utf8_lossy(&cmd_output) 5306 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 5307 5308 thread::sleep(std::time::Duration::new(10, 0)); 5309 5310 // Check that /dev/vdc exists and the block size is 16M. 5311 assert_eq!( 5312 guest 5313 .ssh_command("lsblk | grep vdc | grep -c 16M") 5314 .unwrap() 5315 .trim() 5316 .parse::<u32>() 5317 .unwrap_or_default(), 5318 1 5319 ); 5320 // And check the block device can be read. 5321 guest 5322 .ssh_command("sudo dd if=/dev/vdc of=/dev/null bs=1M iflag=direct count=16") 5323 .unwrap(); 5324 5325 // Reboot the VM. 5326 guest.reboot_linux(0, None); 5327 5328 // Check still there after reboot 5329 assert_eq!( 5330 guest 5331 .ssh_command("lsblk | grep vdc | grep -c 16M") 5332 .unwrap() 5333 .trim() 5334 .parse::<u32>() 5335 .unwrap_or_default(), 5336 1 5337 ); 5338 5339 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 5340 5341 thread::sleep(std::time::Duration::new(20, 0)); 5342 5343 // Check device has gone away 5344 assert_eq!( 5345 guest 5346 .ssh_command("lsblk | grep -c vdc.*16M || true") 5347 .unwrap() 5348 .trim() 5349 .parse::<u32>() 5350 .unwrap_or(1), 5351 0 5352 ); 5353 5354 guest.reboot_linux(1, None); 5355 5356 // Check device still absent 5357 assert_eq!( 5358 guest 5359 .ssh_command("lsblk | grep -c vdc.*16M || true") 5360 .unwrap() 5361 .trim() 5362 .parse::<u32>() 5363 .unwrap_or(1), 5364 0 5365 ); 5366 }); 5367 5368 kill_child(&mut child); 5369 let output = child.wait_with_output().unwrap(); 5370 5371 handle_child_output(r, &output); 5372 } 5373 5374 #[test] 5375 fn test_disk_hotplug() { 5376 _test_disk_hotplug(false) 5377 } 5378 5379 #[test] 5380 #[cfg(target_arch = "x86_64")] 5381 fn test_disk_hotplug_with_landlock() { 5382 _test_disk_hotplug(true) 5383 } 5384 5385 fn create_loop_device(backing_file_path: &str, block_size: u32, num_retries: usize) -> String { 5386 const LOOP_CONFIGURE: u64 = 0x4c0a; 5387 const LOOP_CTL_GET_FREE: u64 = 0x4c82; 5388 const LOOP_CTL_PATH: &str = "/dev/loop-control"; 5389 const LOOP_DEVICE_PREFIX: &str = "/dev/loop"; 5390 5391 #[repr(C)] 5392 struct LoopInfo64 { 5393 lo_device: u64, 5394 lo_inode: u64, 5395 lo_rdevice: u64, 5396 lo_offset: u64, 5397 lo_sizelimit: u64, 5398 lo_number: u32, 5399 lo_encrypt_type: u32, 5400 lo_encrypt_key_size: u32, 5401 lo_flags: u32, 5402 lo_file_name: [u8; 64], 5403 lo_crypt_name: [u8; 64], 5404 lo_encrypt_key: [u8; 32], 5405 lo_init: [u64; 2], 5406 } 5407 5408 impl Default for LoopInfo64 { 5409 fn default() -> Self { 5410 LoopInfo64 { 5411 lo_device: 0, 5412 lo_inode: 0, 5413 lo_rdevice: 0, 5414 lo_offset: 0, 5415 lo_sizelimit: 0, 5416 lo_number: 0, 5417 lo_encrypt_type: 0, 5418 lo_encrypt_key_size: 0, 5419 lo_flags: 0, 5420 lo_file_name: [0; 64], 5421 lo_crypt_name: [0; 64], 5422 lo_encrypt_key: [0; 32], 5423 lo_init: [0; 2], 5424 } 5425 } 5426 } 5427 5428 #[derive(Default)] 5429 #[repr(C)] 5430 struct LoopConfig { 5431 fd: u32, 5432 block_size: u32, 5433 info: LoopInfo64, 5434 _reserved: [u64; 8], 5435 } 5436 5437 // Open loop-control device 5438 let loop_ctl_file = OpenOptions::new() 5439 .read(true) 5440 .write(true) 5441 .open(LOOP_CTL_PATH) 5442 .unwrap(); 5443 5444 // Request a free loop device 5445 let loop_device_number = 5446 unsafe { libc::ioctl(loop_ctl_file.as_raw_fd(), LOOP_CTL_GET_FREE as _) }; 5447 5448 if loop_device_number < 0 { 5449 panic!("Couldn't find a free loop device"); 5450 } 5451 5452 // Create loop device path 5453 let loop_device_path = format!("{LOOP_DEVICE_PREFIX}{loop_device_number}"); 5454 5455 // Open loop device 5456 let loop_device_file = OpenOptions::new() 5457 .read(true) 5458 .write(true) 5459 .open(&loop_device_path) 5460 .unwrap(); 5461 5462 // Open backing file 5463 let backing_file = OpenOptions::new() 5464 .read(true) 5465 .write(true) 5466 .open(backing_file_path) 5467 .unwrap(); 5468 5469 let loop_config = LoopConfig { 5470 fd: backing_file.as_raw_fd() as u32, 5471 block_size, 5472 ..Default::default() 5473 }; 5474 5475 for i in 0..num_retries { 5476 let ret = unsafe { 5477 libc::ioctl( 5478 loop_device_file.as_raw_fd(), 5479 LOOP_CONFIGURE as _, 5480 &loop_config, 5481 ) 5482 }; 5483 if ret != 0 { 5484 if i < num_retries - 1 { 5485 println!( 5486 "Iteration {}: Failed to configure the loop device {}: {}", 5487 i, 5488 loop_device_path, 5489 std::io::Error::last_os_error() 5490 ); 5491 } else { 5492 panic!( 5493 "Failed {} times trying to configure the loop device {}: {}", 5494 num_retries, 5495 loop_device_path, 5496 std::io::Error::last_os_error() 5497 ); 5498 } 5499 } else { 5500 break; 5501 } 5502 5503 // Wait for a bit before retrying 5504 thread::sleep(std::time::Duration::new(5, 0)); 5505 } 5506 5507 loop_device_path 5508 } 5509 5510 #[test] 5511 fn test_virtio_block_topology() { 5512 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5513 let guest = Guest::new(Box::new(focal)); 5514 5515 let kernel_path = direct_kernel_boot_path(); 5516 let test_disk_path = guest.tmp_dir.as_path().join("test.img"); 5517 5518 let output = exec_host_command_output( 5519 format!( 5520 "qemu-img create -f raw {} 16M", 5521 test_disk_path.to_str().unwrap() 5522 ) 5523 .as_str(), 5524 ); 5525 if !output.status.success() { 5526 let stdout = String::from_utf8_lossy(&output.stdout); 5527 let stderr = String::from_utf8_lossy(&output.stderr); 5528 panic!("qemu-img command failed\nstdout\n{stdout}\nstderr\n{stderr}"); 5529 } 5530 5531 let loop_dev = create_loop_device(test_disk_path.to_str().unwrap(), 4096, 5); 5532 5533 let mut child = GuestCommand::new(&guest) 5534 .args(["--cpus", "boot=1"]) 5535 .args(["--memory", "size=512M"]) 5536 .args(["--kernel", kernel_path.to_str().unwrap()]) 5537 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5538 .args([ 5539 "--disk", 5540 format!( 5541 "path={}", 5542 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 5543 ) 5544 .as_str(), 5545 format!( 5546 "path={}", 5547 guest.disk_config.disk(DiskType::CloudInit).unwrap() 5548 ) 5549 .as_str(), 5550 format!("path={}", &loop_dev).as_str(), 5551 ]) 5552 .default_net() 5553 .capture_output() 5554 .spawn() 5555 .unwrap(); 5556 5557 let r = std::panic::catch_unwind(|| { 5558 guest.wait_vm_boot(None).unwrap(); 5559 5560 // MIN-IO column 5561 assert_eq!( 5562 guest 5563 .ssh_command("lsblk -t| grep vdc | awk '{print $3}'") 5564 .unwrap() 5565 .trim() 5566 .parse::<u32>() 5567 .unwrap_or_default(), 5568 4096 5569 ); 5570 // PHY-SEC column 5571 assert_eq!( 5572 guest 5573 .ssh_command("lsblk -t| grep vdc | awk '{print $5}'") 5574 .unwrap() 5575 .trim() 5576 .parse::<u32>() 5577 .unwrap_or_default(), 5578 4096 5579 ); 5580 // LOG-SEC column 5581 assert_eq!( 5582 guest 5583 .ssh_command("lsblk -t| grep vdc | awk '{print $6}'") 5584 .unwrap() 5585 .trim() 5586 .parse::<u32>() 5587 .unwrap_or_default(), 5588 4096 5589 ); 5590 }); 5591 5592 kill_child(&mut child); 5593 let output = child.wait_with_output().unwrap(); 5594 5595 handle_child_output(r, &output); 5596 5597 Command::new("losetup") 5598 .args(["-d", &loop_dev]) 5599 .output() 5600 .expect("loop device not found"); 5601 } 5602 5603 #[test] 5604 fn test_virtio_balloon_deflate_on_oom() { 5605 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5606 let guest = Guest::new(Box::new(focal)); 5607 5608 let kernel_path = direct_kernel_boot_path(); 5609 5610 let api_socket = temp_api_path(&guest.tmp_dir); 5611 5612 //Let's start a 4G guest with balloon occupied 2G memory 5613 let mut child = GuestCommand::new(&guest) 5614 .args(["--api-socket", &api_socket]) 5615 .args(["--cpus", "boot=1"]) 5616 .args(["--memory", "size=4G"]) 5617 .args(["--kernel", kernel_path.to_str().unwrap()]) 5618 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5619 .args(["--balloon", "size=2G,deflate_on_oom=on"]) 5620 .default_disks() 5621 .default_net() 5622 .capture_output() 5623 .spawn() 5624 .unwrap(); 5625 5626 let r = std::panic::catch_unwind(|| { 5627 guest.wait_vm_boot(None).unwrap(); 5628 5629 // Wait for balloon memory's initialization and check its size. 5630 // The virtio-balloon driver might take a few seconds to report the 5631 // balloon effective size back to the VMM. 5632 thread::sleep(std::time::Duration::new(20, 0)); 5633 5634 let orig_balloon = balloon_size(&api_socket); 5635 println!("The original balloon memory size is {orig_balloon} bytes"); 5636 assert!(orig_balloon == 2147483648); 5637 5638 // Two steps to verify if the 'deflate_on_oom' parameter works. 5639 // 1st: run a command to trigger an OOM in the guest. 5640 guest 5641 .ssh_command("echo f | sudo tee /proc/sysrq-trigger") 5642 .unwrap(); 5643 5644 // Give some time for the OOM to happen in the guest and be reported 5645 // back to the host. 5646 thread::sleep(std::time::Duration::new(20, 0)); 5647 5648 // 2nd: check balloon_mem's value to verify balloon has been automatically deflated 5649 let deflated_balloon = balloon_size(&api_socket); 5650 println!("After deflating, balloon memory size is {deflated_balloon} bytes"); 5651 // Verify the balloon size deflated 5652 assert!(deflated_balloon < 2147483648); 5653 }); 5654 5655 kill_child(&mut child); 5656 let output = child.wait_with_output().unwrap(); 5657 5658 handle_child_output(r, &output); 5659 } 5660 5661 #[test] 5662 #[cfg(not(feature = "mshv"))] 5663 fn test_virtio_balloon_free_page_reporting() { 5664 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5665 let guest = Guest::new(Box::new(focal)); 5666 5667 //Let's start a 4G guest with balloon occupied 2G memory 5668 let mut child = GuestCommand::new(&guest) 5669 .args(["--cpus", "boot=1"]) 5670 .args(["--memory", "size=4G"]) 5671 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 5672 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5673 .args(["--balloon", "size=0,free_page_reporting=on"]) 5674 .default_disks() 5675 .default_net() 5676 .capture_output() 5677 .spawn() 5678 .unwrap(); 5679 5680 let pid = child.id(); 5681 let r = std::panic::catch_unwind(|| { 5682 guest.wait_vm_boot(None).unwrap(); 5683 5684 // Check the initial RSS is less than 1GiB 5685 let rss = process_rss_kib(pid); 5686 println!("RSS {rss} < 1048576"); 5687 assert!(rss < 1048576); 5688 5689 // Spawn a command inside the guest to consume 2GiB of RAM for 60 5690 // seconds 5691 let guest_ip = guest.network.guest_ip.clone(); 5692 thread::spawn(move || { 5693 ssh_command_ip( 5694 "stress --vm 1 --vm-bytes 2G --vm-keep --timeout 60", 5695 &guest_ip, 5696 DEFAULT_SSH_RETRIES, 5697 DEFAULT_SSH_TIMEOUT, 5698 ) 5699 .unwrap(); 5700 }); 5701 5702 // Wait for 50 seconds to make sure the stress command is consuming 5703 // the expected amount of memory. 5704 thread::sleep(std::time::Duration::new(50, 0)); 5705 let rss = process_rss_kib(pid); 5706 println!("RSS {rss} >= 2097152"); 5707 assert!(rss >= 2097152); 5708 5709 // Wait for an extra minute to make sure the stress command has 5710 // completed and that the guest reported the free pages to the VMM 5711 // through the virtio-balloon device. We expect the RSS to be under 5712 // 2GiB. 5713 thread::sleep(std::time::Duration::new(60, 0)); 5714 let rss = process_rss_kib(pid); 5715 println!("RSS {rss} < 2097152"); 5716 assert!(rss < 2097152); 5717 }); 5718 5719 kill_child(&mut child); 5720 let output = child.wait_with_output().unwrap(); 5721 5722 handle_child_output(r, &output); 5723 } 5724 5725 #[test] 5726 fn test_pmem_hotplug() { 5727 _test_pmem_hotplug(None) 5728 } 5729 5730 #[test] 5731 fn test_pmem_multi_segment_hotplug() { 5732 _test_pmem_hotplug(Some(15)) 5733 } 5734 5735 fn _test_pmem_hotplug(pci_segment: Option<u16>) { 5736 #[cfg(target_arch = "aarch64")] 5737 let focal_image = FOCAL_IMAGE_UPDATE_KERNEL_NAME.to_string(); 5738 #[cfg(target_arch = "x86_64")] 5739 let focal_image = FOCAL_IMAGE_NAME.to_string(); 5740 let focal = UbuntuDiskConfig::new(focal_image); 5741 let guest = Guest::new(Box::new(focal)); 5742 5743 #[cfg(target_arch = "x86_64")] 5744 let kernel_path = direct_kernel_boot_path(); 5745 #[cfg(target_arch = "aarch64")] 5746 let kernel_path = edk2_path(); 5747 5748 let api_socket = temp_api_path(&guest.tmp_dir); 5749 5750 let mut cmd = GuestCommand::new(&guest); 5751 5752 cmd.args(["--api-socket", &api_socket]) 5753 .args(["--cpus", "boot=1"]) 5754 .args(["--memory", "size=512M"]) 5755 .args(["--kernel", kernel_path.to_str().unwrap()]) 5756 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5757 .default_disks() 5758 .default_net() 5759 .capture_output(); 5760 5761 if pci_segment.is_some() { 5762 cmd.args([ 5763 "--platform", 5764 &format!("num_pci_segments={MAX_NUM_PCI_SEGMENTS}"), 5765 ]); 5766 } 5767 5768 let mut child = cmd.spawn().unwrap(); 5769 5770 let r = std::panic::catch_unwind(|| { 5771 guest.wait_vm_boot(None).unwrap(); 5772 5773 // Check /dev/pmem0 is not there 5774 assert_eq!( 5775 guest 5776 .ssh_command("lsblk | grep -c pmem0 || true") 5777 .unwrap() 5778 .trim() 5779 .parse::<u32>() 5780 .unwrap_or(1), 5781 0 5782 ); 5783 5784 let pmem_temp_file = TempFile::new().unwrap(); 5785 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 5786 let (cmd_success, cmd_output) = remote_command_w_output( 5787 &api_socket, 5788 "add-pmem", 5789 Some(&format!( 5790 "file={},id=test0{}", 5791 pmem_temp_file.as_path().to_str().unwrap(), 5792 if let Some(pci_segment) = pci_segment { 5793 format!(",pci_segment={pci_segment}") 5794 } else { 5795 "".to_owned() 5796 } 5797 )), 5798 ); 5799 assert!(cmd_success); 5800 if let Some(pci_segment) = pci_segment { 5801 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 5802 "{{\"id\":\"test0\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" 5803 ))); 5804 } else { 5805 assert!(String::from_utf8_lossy(&cmd_output) 5806 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 5807 } 5808 5809 // Check that /dev/pmem0 exists and the block size is 128M 5810 assert_eq!( 5811 guest 5812 .ssh_command("lsblk | grep pmem0 | grep -c 128M") 5813 .unwrap() 5814 .trim() 5815 .parse::<u32>() 5816 .unwrap_or_default(), 5817 1 5818 ); 5819 5820 guest.reboot_linux(0, None); 5821 5822 // Check still there after reboot 5823 assert_eq!( 5824 guest 5825 .ssh_command("lsblk | grep pmem0 | grep -c 128M") 5826 .unwrap() 5827 .trim() 5828 .parse::<u32>() 5829 .unwrap_or_default(), 5830 1 5831 ); 5832 5833 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 5834 5835 thread::sleep(std::time::Duration::new(20, 0)); 5836 5837 // Check device has gone away 5838 assert_eq!( 5839 guest 5840 .ssh_command("lsblk | grep -c pmem0.*128M || true") 5841 .unwrap() 5842 .trim() 5843 .parse::<u32>() 5844 .unwrap_or(1), 5845 0 5846 ); 5847 5848 guest.reboot_linux(1, None); 5849 5850 // Check still absent after reboot 5851 assert_eq!( 5852 guest 5853 .ssh_command("lsblk | grep -c pmem0.*128M || true") 5854 .unwrap() 5855 .trim() 5856 .parse::<u32>() 5857 .unwrap_or(1), 5858 0 5859 ); 5860 }); 5861 5862 kill_child(&mut child); 5863 let output = child.wait_with_output().unwrap(); 5864 5865 handle_child_output(r, &output); 5866 } 5867 5868 #[test] 5869 fn test_net_hotplug() { 5870 _test_net_hotplug(None) 5871 } 5872 5873 #[test] 5874 fn test_net_multi_segment_hotplug() { 5875 _test_net_hotplug(Some(15)) 5876 } 5877 5878 fn _test_net_hotplug(pci_segment: Option<u16>) { 5879 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5880 let guest = Guest::new(Box::new(focal)); 5881 5882 #[cfg(target_arch = "x86_64")] 5883 let kernel_path = direct_kernel_boot_path(); 5884 #[cfg(target_arch = "aarch64")] 5885 let kernel_path = edk2_path(); 5886 5887 let api_socket = temp_api_path(&guest.tmp_dir); 5888 5889 // Boot without network 5890 let mut cmd = GuestCommand::new(&guest); 5891 5892 cmd.args(["--api-socket", &api_socket]) 5893 .args(["--cpus", "boot=1"]) 5894 .args(["--memory", "size=512M"]) 5895 .args(["--kernel", kernel_path.to_str().unwrap()]) 5896 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5897 .default_disks() 5898 .capture_output(); 5899 5900 if pci_segment.is_some() { 5901 cmd.args([ 5902 "--platform", 5903 &format!("num_pci_segments={MAX_NUM_PCI_SEGMENTS}"), 5904 ]); 5905 } 5906 5907 let mut child = cmd.spawn().unwrap(); 5908 5909 thread::sleep(std::time::Duration::new(20, 0)); 5910 5911 let r = std::panic::catch_unwind(|| { 5912 // Add network 5913 let (cmd_success, cmd_output) = remote_command_w_output( 5914 &api_socket, 5915 "add-net", 5916 Some( 5917 format!( 5918 "{}{},id=test0", 5919 guest.default_net_string(), 5920 if let Some(pci_segment) = pci_segment { 5921 format!(",pci_segment={pci_segment}") 5922 } else { 5923 "".to_owned() 5924 } 5925 ) 5926 .as_str(), 5927 ), 5928 ); 5929 assert!(cmd_success); 5930 5931 if let Some(pci_segment) = pci_segment { 5932 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 5933 "{{\"id\":\"test0\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" 5934 ))); 5935 } else { 5936 assert!(String::from_utf8_lossy(&cmd_output) 5937 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:05.0\"}")); 5938 } 5939 5940 thread::sleep(std::time::Duration::new(5, 0)); 5941 5942 // 1 network interfaces + default localhost ==> 2 interfaces 5943 assert_eq!( 5944 guest 5945 .ssh_command("ip -o link | wc -l") 5946 .unwrap() 5947 .trim() 5948 .parse::<u32>() 5949 .unwrap_or_default(), 5950 2 5951 ); 5952 5953 // Remove network 5954 assert!(remote_command(&api_socket, "remove-device", Some("test0"),)); 5955 thread::sleep(std::time::Duration::new(5, 0)); 5956 5957 let (cmd_success, cmd_output) = remote_command_w_output( 5958 &api_socket, 5959 "add-net", 5960 Some( 5961 format!( 5962 "{}{},id=test1", 5963 guest.default_net_string(), 5964 if let Some(pci_segment) = pci_segment { 5965 format!(",pci_segment={pci_segment}") 5966 } else { 5967 "".to_owned() 5968 } 5969 ) 5970 .as_str(), 5971 ), 5972 ); 5973 assert!(cmd_success); 5974 5975 if let Some(pci_segment) = pci_segment { 5976 assert!(String::from_utf8_lossy(&cmd_output).contains(&format!( 5977 "{{\"id\":\"test1\",\"bdf\":\"{pci_segment:04x}:00:01.0\"}}" 5978 ))); 5979 } else { 5980 assert!(String::from_utf8_lossy(&cmd_output) 5981 .contains("{\"id\":\"test1\",\"bdf\":\"0000:00:05.0\"}")); 5982 } 5983 5984 thread::sleep(std::time::Duration::new(5, 0)); 5985 5986 // 1 network interfaces + default localhost ==> 2 interfaces 5987 assert_eq!( 5988 guest 5989 .ssh_command("ip -o link | wc -l") 5990 .unwrap() 5991 .trim() 5992 .parse::<u32>() 5993 .unwrap_or_default(), 5994 2 5995 ); 5996 5997 guest.reboot_linux(0, None); 5998 5999 // Check still there after reboot 6000 // 1 network interfaces + default localhost ==> 2 interfaces 6001 assert_eq!( 6002 guest 6003 .ssh_command("ip -o link | wc -l") 6004 .unwrap() 6005 .trim() 6006 .parse::<u32>() 6007 .unwrap_or_default(), 6008 2 6009 ); 6010 }); 6011 6012 kill_child(&mut child); 6013 let output = child.wait_with_output().unwrap(); 6014 6015 handle_child_output(r, &output); 6016 } 6017 6018 #[test] 6019 fn test_initramfs() { 6020 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6021 let guest = Guest::new(Box::new(focal)); 6022 let mut workload_path = dirs::home_dir().unwrap(); 6023 workload_path.push("workloads"); 6024 6025 #[cfg(target_arch = "x86_64")] 6026 let mut kernels = vec![direct_kernel_boot_path()]; 6027 #[cfg(target_arch = "aarch64")] 6028 let kernels = [direct_kernel_boot_path()]; 6029 6030 #[cfg(target_arch = "x86_64")] 6031 { 6032 let mut pvh_kernel_path = workload_path.clone(); 6033 pvh_kernel_path.push("vmlinux"); 6034 kernels.push(pvh_kernel_path); 6035 } 6036 6037 let mut initramfs_path = workload_path; 6038 initramfs_path.push("alpine_initramfs.img"); 6039 6040 let test_string = String::from("axz34i9rylotd8n50wbv6kcj7f2qushme1pg"); 6041 let cmdline = format!("console=hvc0 quiet TEST_STRING={test_string}"); 6042 6043 kernels.iter().for_each(|k_path| { 6044 let mut child = GuestCommand::new(&guest) 6045 .args(["--kernel", k_path.to_str().unwrap()]) 6046 .args(["--initramfs", initramfs_path.to_str().unwrap()]) 6047 .args(["--cmdline", &cmdline]) 6048 .capture_output() 6049 .spawn() 6050 .unwrap(); 6051 6052 thread::sleep(std::time::Duration::new(20, 0)); 6053 6054 kill_child(&mut child); 6055 let output = child.wait_with_output().unwrap(); 6056 6057 let r = std::panic::catch_unwind(|| { 6058 let s = String::from_utf8_lossy(&output.stdout); 6059 6060 assert_ne!(s.lines().position(|line| line == test_string), None); 6061 }); 6062 6063 handle_child_output(r, &output); 6064 }); 6065 } 6066 6067 #[test] 6068 fn test_counters() { 6069 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6070 let guest = Guest::new(Box::new(focal)); 6071 let api_socket = temp_api_path(&guest.tmp_dir); 6072 6073 let mut cmd = GuestCommand::new(&guest); 6074 cmd.args(["--cpus", "boot=1"]) 6075 .args(["--memory", "size=512M"]) 6076 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 6077 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6078 .default_disks() 6079 .args(["--net", guest.default_net_string().as_str()]) 6080 .args(["--api-socket", &api_socket]) 6081 .capture_output(); 6082 6083 let mut child = cmd.spawn().unwrap(); 6084 6085 let r = std::panic::catch_unwind(|| { 6086 guest.wait_vm_boot(None).unwrap(); 6087 6088 let orig_counters = get_counters(&api_socket); 6089 guest 6090 .ssh_command("dd if=/dev/zero of=test count=8 bs=1M") 6091 .unwrap(); 6092 6093 let new_counters = get_counters(&api_socket); 6094 6095 // Check that all the counters have increased 6096 assert!(new_counters > orig_counters); 6097 }); 6098 6099 kill_child(&mut child); 6100 let output = child.wait_with_output().unwrap(); 6101 6102 handle_child_output(r, &output); 6103 } 6104 6105 #[test] 6106 #[cfg(feature = "guest_debug")] 6107 fn test_coredump() { 6108 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6109 let guest = Guest::new(Box::new(focal)); 6110 let api_socket = temp_api_path(&guest.tmp_dir); 6111 6112 let mut cmd = GuestCommand::new(&guest); 6113 cmd.args(["--cpus", "boot=4"]) 6114 .args(["--memory", "size=4G"]) 6115 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 6116 .default_disks() 6117 .args(["--net", guest.default_net_string().as_str()]) 6118 .args(["--api-socket", &api_socket]) 6119 .capture_output(); 6120 6121 let mut child = cmd.spawn().unwrap(); 6122 let vmcore_file = temp_vmcore_file_path(&guest.tmp_dir); 6123 6124 let r = std::panic::catch_unwind(|| { 6125 guest.wait_vm_boot(None).unwrap(); 6126 6127 assert!(remote_command(&api_socket, "pause", None)); 6128 6129 assert!(remote_command( 6130 &api_socket, 6131 "coredump", 6132 Some(format!("file://{vmcore_file}").as_str()), 6133 )); 6134 6135 // the num of CORE notes should equals to vcpu 6136 let readelf_core_num_cmd = 6137 format!("readelf --all {vmcore_file} |grep CORE |grep -v Type |wc -l"); 6138 let core_num_in_elf = exec_host_command_output(&readelf_core_num_cmd); 6139 assert_eq!(String::from_utf8_lossy(&core_num_in_elf.stdout).trim(), "4"); 6140 6141 // the num of QEMU notes should equals to vcpu 6142 let readelf_vmm_num_cmd = format!("readelf --all {vmcore_file} |grep QEMU |wc -l"); 6143 let vmm_num_in_elf = exec_host_command_output(&readelf_vmm_num_cmd); 6144 assert_eq!(String::from_utf8_lossy(&vmm_num_in_elf.stdout).trim(), "4"); 6145 }); 6146 6147 kill_child(&mut child); 6148 let output = child.wait_with_output().unwrap(); 6149 6150 handle_child_output(r, &output); 6151 } 6152 6153 #[test] 6154 #[cfg(feature = "guest_debug")] 6155 fn test_coredump_no_pause() { 6156 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6157 let guest = Guest::new(Box::new(focal)); 6158 let api_socket = temp_api_path(&guest.tmp_dir); 6159 6160 let mut cmd = GuestCommand::new(&guest); 6161 cmd.args(["--cpus", "boot=4"]) 6162 .args(["--memory", "size=4G"]) 6163 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 6164 .default_disks() 6165 .args(["--net", guest.default_net_string().as_str()]) 6166 .args(["--api-socket", &api_socket]) 6167 .capture_output(); 6168 6169 let mut child = cmd.spawn().unwrap(); 6170 let vmcore_file = temp_vmcore_file_path(&guest.tmp_dir); 6171 6172 let r = std::panic::catch_unwind(|| { 6173 guest.wait_vm_boot(None).unwrap(); 6174 6175 assert!(remote_command( 6176 &api_socket, 6177 "coredump", 6178 Some(format!("file://{vmcore_file}").as_str()), 6179 )); 6180 6181 assert_eq!(vm_state(&api_socket), "Running"); 6182 }); 6183 6184 kill_child(&mut child); 6185 let output = child.wait_with_output().unwrap(); 6186 6187 handle_child_output(r, &output); 6188 } 6189 6190 #[test] 6191 fn test_watchdog() { 6192 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6193 let guest = Guest::new(Box::new(focal)); 6194 let api_socket = temp_api_path(&guest.tmp_dir); 6195 6196 let kernel_path = direct_kernel_boot_path(); 6197 let event_path = temp_event_monitor_path(&guest.tmp_dir); 6198 6199 let mut cmd = GuestCommand::new(&guest); 6200 cmd.args(["--cpus", "boot=1"]) 6201 .args(["--memory", "size=512M"]) 6202 .args(["--kernel", kernel_path.to_str().unwrap()]) 6203 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6204 .default_disks() 6205 .args(["--net", guest.default_net_string().as_str()]) 6206 .args(["--watchdog"]) 6207 .args(["--api-socket", &api_socket]) 6208 .args(["--event-monitor", format!("path={event_path}").as_str()]) 6209 .capture_output(); 6210 6211 let mut child = cmd.spawn().unwrap(); 6212 6213 let r = std::panic::catch_unwind(|| { 6214 guest.wait_vm_boot(None).unwrap(); 6215 6216 let mut expected_reboot_count = 1; 6217 6218 // Enable the watchdog with a 15s timeout 6219 enable_guest_watchdog(&guest, 15); 6220 6221 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 6222 assert_eq!( 6223 guest 6224 .ssh_command("sudo journalctl | grep -c -- \"Watchdog started\"") 6225 .unwrap() 6226 .trim() 6227 .parse::<u32>() 6228 .unwrap_or_default(), 6229 1 6230 ); 6231 6232 // Allow some normal time to elapse to check we don't get spurious reboots 6233 thread::sleep(std::time::Duration::new(40, 0)); 6234 // Check no reboot 6235 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 6236 6237 // Trigger a panic (sync first). We need to do this inside a screen with a delay so the SSH command returns. 6238 guest.ssh_command("screen -dmS reboot sh -c \"sleep 5; echo s | tee /proc/sysrq-trigger; echo c | sudo tee /proc/sysrq-trigger\"").unwrap(); 6239 // Allow some time for the watchdog to trigger (max 30s) and reboot to happen 6240 guest.wait_vm_boot(Some(50)).unwrap(); 6241 // Check a reboot is triggered by the watchdog 6242 expected_reboot_count += 1; 6243 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 6244 6245 #[cfg(target_arch = "x86_64")] 6246 { 6247 // Now pause the VM and remain offline for 30s 6248 assert!(remote_command(&api_socket, "pause", None)); 6249 let latest_events = [ 6250 &MetaEvent { 6251 event: "pausing".to_string(), 6252 device_id: None, 6253 }, 6254 &MetaEvent { 6255 event: "paused".to_string(), 6256 device_id: None, 6257 }, 6258 ]; 6259 assert!(check_latest_events_exact(&latest_events, &event_path)); 6260 assert!(remote_command(&api_socket, "resume", None)); 6261 6262 // Check no reboot 6263 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 6264 } 6265 }); 6266 6267 kill_child(&mut child); 6268 let output = child.wait_with_output().unwrap(); 6269 6270 handle_child_output(r, &output); 6271 } 6272 6273 #[test] 6274 fn test_pvpanic() { 6275 let jammy = UbuntuDiskConfig::new(JAMMY_IMAGE_NAME.to_string()); 6276 let guest = Guest::new(Box::new(jammy)); 6277 let api_socket = temp_api_path(&guest.tmp_dir); 6278 let event_path = temp_event_monitor_path(&guest.tmp_dir); 6279 6280 let kernel_path = direct_kernel_boot_path(); 6281 6282 let mut cmd = GuestCommand::new(&guest); 6283 cmd.args(["--cpus", "boot=1"]) 6284 .args(["--memory", "size=512M"]) 6285 .args(["--kernel", kernel_path.to_str().unwrap()]) 6286 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6287 .default_disks() 6288 .args(["--net", guest.default_net_string().as_str()]) 6289 .args(["--pvpanic"]) 6290 .args(["--api-socket", &api_socket]) 6291 .args(["--event-monitor", format!("path={event_path}").as_str()]) 6292 .capture_output(); 6293 6294 let mut child = cmd.spawn().unwrap(); 6295 6296 let r = std::panic::catch_unwind(|| { 6297 guest.wait_vm_boot(None).unwrap(); 6298 6299 // Trigger guest a panic 6300 make_guest_panic(&guest); 6301 6302 // Wait a while for guest 6303 thread::sleep(std::time::Duration::new(10, 0)); 6304 6305 let expected_sequential_events = [&MetaEvent { 6306 event: "panic".to_string(), 6307 device_id: None, 6308 }]; 6309 assert!(check_latest_events_exact( 6310 &expected_sequential_events, 6311 &event_path 6312 )); 6313 }); 6314 6315 kill_child(&mut child); 6316 let output = child.wait_with_output().unwrap(); 6317 6318 handle_child_output(r, &output); 6319 } 6320 6321 #[test] 6322 fn test_tap_from_fd() { 6323 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6324 let guest = Guest::new(Box::new(focal)); 6325 let kernel_path = direct_kernel_boot_path(); 6326 6327 // Create a TAP interface with multi-queue enabled 6328 let num_queue_pairs: usize = 2; 6329 6330 use std::str::FromStr; 6331 let taps = net_util::open_tap( 6332 Some("chtap0"), 6333 Some(std::net::Ipv4Addr::from_str(&guest.network.host_ip).unwrap()), 6334 None, 6335 &mut None, 6336 None, 6337 num_queue_pairs, 6338 Some(libc::O_RDWR | libc::O_NONBLOCK), 6339 ) 6340 .unwrap(); 6341 6342 let mut child = GuestCommand::new(&guest) 6343 .args(["--cpus", &format!("boot={num_queue_pairs}")]) 6344 .args(["--memory", "size=512M"]) 6345 .args(["--kernel", kernel_path.to_str().unwrap()]) 6346 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6347 .default_disks() 6348 .args([ 6349 "--net", 6350 &format!( 6351 "fd=[{},{}],mac={},num_queues={}", 6352 taps[0].as_raw_fd(), 6353 taps[1].as_raw_fd(), 6354 guest.network.guest_mac, 6355 num_queue_pairs * 2 6356 ), 6357 ]) 6358 .capture_output() 6359 .spawn() 6360 .unwrap(); 6361 6362 let r = std::panic::catch_unwind(|| { 6363 guest.wait_vm_boot(None).unwrap(); 6364 6365 assert_eq!( 6366 guest 6367 .ssh_command("ip -o link | wc -l") 6368 .unwrap() 6369 .trim() 6370 .parse::<u32>() 6371 .unwrap_or_default(), 6372 2 6373 ); 6374 6375 guest.reboot_linux(0, None); 6376 6377 assert_eq!( 6378 guest 6379 .ssh_command("ip -o link | wc -l") 6380 .unwrap() 6381 .trim() 6382 .parse::<u32>() 6383 .unwrap_or_default(), 6384 2 6385 ); 6386 }); 6387 6388 kill_child(&mut child); 6389 let output = child.wait_with_output().unwrap(); 6390 6391 handle_child_output(r, &output); 6392 } 6393 6394 // By design, a guest VM won't be able to connect to the host 6395 // machine when using a macvtap network interface (while it can 6396 // communicate externally). As a workaround, this integration 6397 // test creates two macvtap interfaces in 'bridge' mode on the 6398 // same physical net interface, one for the guest and one for 6399 // the host. With additional setup on the IP address and the 6400 // routing table, it enables the communications between the 6401 // guest VM and the host machine. 6402 // Details: https://wiki.libvirt.org/page/TroubleshootMacvtapHostFail 6403 fn _test_macvtap(hotplug: bool, guest_macvtap_name: &str, host_macvtap_name: &str) { 6404 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6405 let guest = Guest::new(Box::new(focal)); 6406 let api_socket = temp_api_path(&guest.tmp_dir); 6407 6408 #[cfg(target_arch = "x86_64")] 6409 let kernel_path = direct_kernel_boot_path(); 6410 #[cfg(target_arch = "aarch64")] 6411 let kernel_path = edk2_path(); 6412 6413 let phy_net = "eth0"; 6414 6415 // Create a macvtap interface for the guest VM to use 6416 assert!(exec_host_command_status(&format!( 6417 "sudo ip link add link {phy_net} name {guest_macvtap_name} type macvtap mod bridge" 6418 )) 6419 .success()); 6420 assert!(exec_host_command_status(&format!( 6421 "sudo ip link set {} address {} up", 6422 guest_macvtap_name, guest.network.guest_mac 6423 )) 6424 .success()); 6425 assert!( 6426 exec_host_command_status(&format!("sudo ip link show {guest_macvtap_name}")).success() 6427 ); 6428 6429 let tap_index = 6430 fs::read_to_string(format!("/sys/class/net/{guest_macvtap_name}/ifindex")).unwrap(); 6431 let tap_device = format!("/dev/tap{}", tap_index.trim()); 6432 6433 assert!(exec_host_command_status(&format!("sudo chown $UID.$UID {tap_device}")).success()); 6434 6435 let cstr_tap_device = std::ffi::CString::new(tap_device).unwrap(); 6436 let tap_fd1 = unsafe { libc::open(cstr_tap_device.as_ptr(), libc::O_RDWR) }; 6437 assert!(tap_fd1 > 0); 6438 let tap_fd2 = unsafe { libc::open(cstr_tap_device.as_ptr(), libc::O_RDWR) }; 6439 assert!(tap_fd2 > 0); 6440 6441 // Create a macvtap on the same physical net interface for 6442 // the host machine to use 6443 assert!(exec_host_command_status(&format!( 6444 "sudo ip link add link {phy_net} name {host_macvtap_name} type macvtap mod bridge" 6445 )) 6446 .success()); 6447 // Use default mask "255.255.255.0" 6448 assert!(exec_host_command_status(&format!( 6449 "sudo ip address add {}/24 dev {}", 6450 guest.network.host_ip, host_macvtap_name 6451 )) 6452 .success()); 6453 assert!( 6454 exec_host_command_status(&format!("sudo ip link set dev {host_macvtap_name} up")) 6455 .success() 6456 ); 6457 6458 let mut guest_command = GuestCommand::new(&guest); 6459 guest_command 6460 .args(["--cpus", "boot=2"]) 6461 .args(["--memory", "size=512M"]) 6462 .args(["--kernel", kernel_path.to_str().unwrap()]) 6463 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6464 .default_disks() 6465 .args(["--api-socket", &api_socket]); 6466 6467 let net_params = format!( 6468 "fd=[{},{}],mac={},num_queues=4", 6469 tap_fd1, tap_fd2, guest.network.guest_mac 6470 ); 6471 6472 if !hotplug { 6473 guest_command.args(["--net", &net_params]); 6474 } 6475 6476 let mut child = guest_command.capture_output().spawn().unwrap(); 6477 6478 if hotplug { 6479 // Give some time to the VMM process to listen to the API 6480 // socket. This is the only requirement to avoid the following 6481 // call to ch-remote from failing. 6482 thread::sleep(std::time::Duration::new(10, 0)); 6483 // Hotplug the virtio-net device 6484 let (cmd_success, cmd_output) = 6485 remote_command_w_output(&api_socket, "add-net", Some(&net_params)); 6486 assert!(cmd_success); 6487 #[cfg(target_arch = "x86_64")] 6488 assert!(String::from_utf8_lossy(&cmd_output) 6489 .contains("{\"id\":\"_net2\",\"bdf\":\"0000:00:05.0\"}")); 6490 #[cfg(target_arch = "aarch64")] 6491 assert!(String::from_utf8_lossy(&cmd_output) 6492 .contains("{\"id\":\"_net0\",\"bdf\":\"0000:00:05.0\"}")); 6493 } 6494 6495 // The functional connectivity provided by the virtio-net device 6496 // gets tested through wait_vm_boot() as it expects to receive a 6497 // HTTP request, and through the SSH command as well. 6498 let r = std::panic::catch_unwind(|| { 6499 guest.wait_vm_boot(None).unwrap(); 6500 6501 assert_eq!( 6502 guest 6503 .ssh_command("ip -o link | wc -l") 6504 .unwrap() 6505 .trim() 6506 .parse::<u32>() 6507 .unwrap_or_default(), 6508 2 6509 ); 6510 6511 guest.reboot_linux(0, None); 6512 6513 assert_eq!( 6514 guest 6515 .ssh_command("ip -o link | wc -l") 6516 .unwrap() 6517 .trim() 6518 .parse::<u32>() 6519 .unwrap_or_default(), 6520 2 6521 ); 6522 }); 6523 6524 kill_child(&mut child); 6525 6526 exec_host_command_status(&format!("sudo ip link del {guest_macvtap_name}")); 6527 exec_host_command_status(&format!("sudo ip link del {host_macvtap_name}")); 6528 6529 let output = child.wait_with_output().unwrap(); 6530 6531 handle_child_output(r, &output); 6532 } 6533 6534 #[test] 6535 #[cfg_attr(target_arch = "aarch64", ignore = "See #5443")] 6536 fn test_macvtap() { 6537 _test_macvtap(false, "guestmacvtap0", "hostmacvtap0") 6538 } 6539 6540 #[test] 6541 #[cfg_attr(target_arch = "aarch64", ignore = "See #5443")] 6542 fn test_macvtap_hotplug() { 6543 _test_macvtap(true, "guestmacvtap1", "hostmacvtap1") 6544 } 6545 6546 #[test] 6547 #[cfg(not(feature = "mshv"))] 6548 fn test_ovs_dpdk() { 6549 let focal1 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6550 let guest1 = Guest::new(Box::new(focal1)); 6551 6552 let focal2 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6553 let guest2 = Guest::new(Box::new(focal2)); 6554 let api_socket_source = format!("{}.1", temp_api_path(&guest2.tmp_dir)); 6555 6556 let (mut child1, mut child2) = 6557 setup_ovs_dpdk_guests(&guest1, &guest2, &api_socket_source, false); 6558 6559 // Create the snapshot directory 6560 let snapshot_dir = temp_snapshot_dir_path(&guest2.tmp_dir); 6561 6562 let r = std::panic::catch_unwind(|| { 6563 // Remove one of the two ports from the OVS bridge 6564 assert!(exec_host_command_status("ovs-vsctl del-port vhost-user1").success()); 6565 6566 // Spawn a new netcat listener in the first VM 6567 let guest_ip = guest1.network.guest_ip.clone(); 6568 thread::spawn(move || { 6569 ssh_command_ip( 6570 "nc -l 12345", 6571 &guest_ip, 6572 DEFAULT_SSH_RETRIES, 6573 DEFAULT_SSH_TIMEOUT, 6574 ) 6575 .unwrap(); 6576 }); 6577 6578 // Wait for the server to be listening 6579 thread::sleep(std::time::Duration::new(5, 0)); 6580 6581 // Check the connection fails this time 6582 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap_err(); 6583 6584 // Add the OVS port back 6585 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user1 -- set Interface vhost-user1 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient1").success()); 6586 6587 // And finally check the connection is functional again 6588 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 6589 6590 // Pause the VM 6591 assert!(remote_command(&api_socket_source, "pause", None)); 6592 6593 // Take a snapshot from the VM 6594 assert!(remote_command( 6595 &api_socket_source, 6596 "snapshot", 6597 Some(format!("file://{snapshot_dir}").as_str()), 6598 )); 6599 6600 // Wait to make sure the snapshot is completed 6601 thread::sleep(std::time::Duration::new(10, 0)); 6602 }); 6603 6604 // Shutdown the source VM 6605 kill_child(&mut child2); 6606 let output = child2.wait_with_output().unwrap(); 6607 handle_child_output(r, &output); 6608 6609 // Remove the vhost-user socket file. 6610 Command::new("rm") 6611 .arg("-f") 6612 .arg("/tmp/dpdkvhostclient2") 6613 .output() 6614 .unwrap(); 6615 6616 let api_socket_restored = format!("{}.2", temp_api_path(&guest2.tmp_dir)); 6617 // Restore the VM from the snapshot 6618 let mut child2 = GuestCommand::new(&guest2) 6619 .args(["--api-socket", &api_socket_restored]) 6620 .args([ 6621 "--restore", 6622 format!("source_url=file://{snapshot_dir}").as_str(), 6623 ]) 6624 .capture_output() 6625 .spawn() 6626 .unwrap(); 6627 6628 // Wait for the VM to be restored 6629 thread::sleep(std::time::Duration::new(10, 0)); 6630 6631 let r = std::panic::catch_unwind(|| { 6632 // Resume the VM 6633 assert!(remote_command(&api_socket_restored, "resume", None)); 6634 6635 // Spawn a new netcat listener in the first VM 6636 let guest_ip = guest1.network.guest_ip.clone(); 6637 thread::spawn(move || { 6638 ssh_command_ip( 6639 "nc -l 12345", 6640 &guest_ip, 6641 DEFAULT_SSH_RETRIES, 6642 DEFAULT_SSH_TIMEOUT, 6643 ) 6644 .unwrap(); 6645 }); 6646 6647 // Wait for the server to be listening 6648 thread::sleep(std::time::Duration::new(5, 0)); 6649 6650 // And check the connection is still functional after restore 6651 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 6652 }); 6653 6654 kill_child(&mut child1); 6655 kill_child(&mut child2); 6656 6657 let output = child1.wait_with_output().unwrap(); 6658 child2.wait().unwrap(); 6659 6660 cleanup_ovs_dpdk(); 6661 6662 handle_child_output(r, &output); 6663 } 6664 6665 fn setup_spdk_nvme(nvme_dir: &std::path::Path) -> Child { 6666 cleanup_spdk_nvme(); 6667 6668 assert!(exec_host_command_status(&format!( 6669 "mkdir -p {}", 6670 nvme_dir.join("nvme-vfio-user").to_str().unwrap() 6671 )) 6672 .success()); 6673 assert!(exec_host_command_status(&format!( 6674 "truncate {} -s 128M", 6675 nvme_dir.join("test-disk.raw").to_str().unwrap() 6676 )) 6677 .success()); 6678 assert!(exec_host_command_status(&format!( 6679 "mkfs.ext4 {}", 6680 nvme_dir.join("test-disk.raw").to_str().unwrap() 6681 )) 6682 .success()); 6683 6684 // Start the SPDK nvmf_tgt daemon to present NVMe device as a VFIO user device 6685 let child = Command::new("/usr/local/bin/spdk-nvme/nvmf_tgt") 6686 .args(["-i", "0", "-m", "0x1"]) 6687 .spawn() 6688 .unwrap(); 6689 thread::sleep(std::time::Duration::new(2, 0)); 6690 6691 assert!(exec_host_command_with_retries( 6692 "/usr/local/bin/spdk-nvme/rpc.py nvmf_create_transport -t VFIOUSER", 6693 3, 6694 std::time::Duration::new(5, 0), 6695 )); 6696 assert!(exec_host_command_status(&format!( 6697 "/usr/local/bin/spdk-nvme/rpc.py bdev_aio_create {} test 512", 6698 nvme_dir.join("test-disk.raw").to_str().unwrap() 6699 )) 6700 .success()); 6701 assert!(exec_host_command_status( 6702 "/usr/local/bin/spdk-nvme/rpc.py nvmf_create_subsystem nqn.2019-07.io.spdk:cnode -a -s test" 6703 ) 6704 .success()); 6705 assert!(exec_host_command_status( 6706 "/usr/local/bin/spdk-nvme/rpc.py nvmf_subsystem_add_ns nqn.2019-07.io.spdk:cnode test" 6707 ) 6708 .success()); 6709 assert!(exec_host_command_status(&format!( 6710 "/usr/local/bin/spdk-nvme/rpc.py nvmf_subsystem_add_listener nqn.2019-07.io.spdk:cnode -t VFIOUSER -a {} -s 0", 6711 nvme_dir.join("nvme-vfio-user").to_str().unwrap() 6712 )) 6713 .success()); 6714 6715 child 6716 } 6717 6718 fn cleanup_spdk_nvme() { 6719 exec_host_command_status("pkill -f nvmf_tgt"); 6720 } 6721 6722 #[test] 6723 fn test_vfio_user() { 6724 let jammy_image = JAMMY_IMAGE_NAME.to_string(); 6725 let jammy = UbuntuDiskConfig::new(jammy_image); 6726 let guest = Guest::new(Box::new(jammy)); 6727 6728 let spdk_nvme_dir = guest.tmp_dir.as_path().join("test-vfio-user"); 6729 let mut spdk_child = setup_spdk_nvme(spdk_nvme_dir.as_path()); 6730 6731 let api_socket = temp_api_path(&guest.tmp_dir); 6732 let mut child = GuestCommand::new(&guest) 6733 .args(["--api-socket", &api_socket]) 6734 .args(["--cpus", "boot=1"]) 6735 .args(["--memory", "size=1G,shared=on,hugepages=on"]) 6736 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 6737 .args(["--serial", "tty", "--console", "off"]) 6738 .default_disks() 6739 .default_net() 6740 .capture_output() 6741 .spawn() 6742 .unwrap(); 6743 6744 let r = std::panic::catch_unwind(|| { 6745 guest.wait_vm_boot(None).unwrap(); 6746 6747 // Hotplug the SPDK-NVMe device to the VM 6748 let (cmd_success, cmd_output) = remote_command_w_output( 6749 &api_socket, 6750 "add-user-device", 6751 Some(&format!( 6752 "socket={},id=vfio_user0", 6753 spdk_nvme_dir 6754 .as_path() 6755 .join("nvme-vfio-user/cntrl") 6756 .to_str() 6757 .unwrap(), 6758 )), 6759 ); 6760 assert!(cmd_success); 6761 assert!(String::from_utf8_lossy(&cmd_output) 6762 .contains("{\"id\":\"vfio_user0\",\"bdf\":\"0000:00:05.0\"}")); 6763 6764 thread::sleep(std::time::Duration::new(10, 0)); 6765 6766 // Check both if /dev/nvme exists and if the block size is 128M. 6767 assert_eq!( 6768 guest 6769 .ssh_command("lsblk | grep nvme0n1 | grep -c 128M") 6770 .unwrap() 6771 .trim() 6772 .parse::<u32>() 6773 .unwrap_or_default(), 6774 1 6775 ); 6776 6777 // Check changes persist after reboot 6778 assert_eq!( 6779 guest.ssh_command("sudo mount /dev/nvme0n1 /mnt").unwrap(), 6780 "" 6781 ); 6782 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), "lost+found\n"); 6783 guest 6784 .ssh_command("echo test123 | sudo tee /mnt/test") 6785 .unwrap(); 6786 assert_eq!(guest.ssh_command("sudo umount /mnt").unwrap(), ""); 6787 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), ""); 6788 6789 guest.reboot_linux(0, None); 6790 assert_eq!( 6791 guest.ssh_command("sudo mount /dev/nvme0n1 /mnt").unwrap(), 6792 "" 6793 ); 6794 assert_eq!( 6795 guest.ssh_command("sudo cat /mnt/test").unwrap().trim(), 6796 "test123" 6797 ); 6798 }); 6799 6800 let _ = spdk_child.kill(); 6801 let _ = spdk_child.wait(); 6802 6803 kill_child(&mut child); 6804 let output = child.wait_with_output().unwrap(); 6805 6806 handle_child_output(r, &output); 6807 } 6808 6809 #[test] 6810 #[cfg(target_arch = "x86_64")] 6811 fn test_vdpa_block() { 6812 // Before trying to run the test, verify the vdpa_sim_blk module is correctly loaded. 6813 assert!(exec_host_command_status("lsmod | grep vdpa_sim_blk").success()); 6814 6815 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6816 let guest = Guest::new(Box::new(focal)); 6817 let api_socket = temp_api_path(&guest.tmp_dir); 6818 6819 let kernel_path = direct_kernel_boot_path(); 6820 6821 let mut child = GuestCommand::new(&guest) 6822 .args(["--cpus", "boot=2"]) 6823 .args(["--memory", "size=512M,hugepages=on"]) 6824 .args(["--kernel", kernel_path.to_str().unwrap()]) 6825 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6826 .default_disks() 6827 .default_net() 6828 .args(["--vdpa", "path=/dev/vhost-vdpa-0,num_queues=1"]) 6829 .args(["--platform", "num_pci_segments=2,iommu_segments=1"]) 6830 .args(["--api-socket", &api_socket]) 6831 .capture_output() 6832 .spawn() 6833 .unwrap(); 6834 6835 let r = std::panic::catch_unwind(|| { 6836 guest.wait_vm_boot(None).unwrap(); 6837 6838 // Check both if /dev/vdc exists and if the block size is 128M. 6839 assert_eq!( 6840 guest 6841 .ssh_command("lsblk | grep vdc | grep -c 128M") 6842 .unwrap() 6843 .trim() 6844 .parse::<u32>() 6845 .unwrap_or_default(), 6846 1 6847 ); 6848 6849 // Check the content of the block device after we wrote to it. 6850 // The vpda-sim-blk should let us read what we previously wrote. 6851 guest 6852 .ssh_command("sudo bash -c 'echo foobar > /dev/vdc'") 6853 .unwrap(); 6854 assert_eq!( 6855 guest.ssh_command("sudo head -1 /dev/vdc").unwrap().trim(), 6856 "foobar" 6857 ); 6858 6859 // Hotplug an extra vDPA block device behind the vIOMMU 6860 // Add a new vDPA device to the VM 6861 let (cmd_success, cmd_output) = remote_command_w_output( 6862 &api_socket, 6863 "add-vdpa", 6864 Some("id=myvdpa0,path=/dev/vhost-vdpa-1,num_queues=1,pci_segment=1,iommu=on"), 6865 ); 6866 assert!(cmd_success); 6867 assert!(String::from_utf8_lossy(&cmd_output) 6868 .contains("{\"id\":\"myvdpa0\",\"bdf\":\"0001:00:01.0\"}")); 6869 6870 thread::sleep(std::time::Duration::new(10, 0)); 6871 6872 // Check IOMMU setup 6873 assert!(guest 6874 .does_device_vendor_pair_match("0x1057", "0x1af4") 6875 .unwrap_or_default()); 6876 assert_eq!( 6877 guest 6878 .ssh_command("ls /sys/kernel/iommu_groups/0/devices") 6879 .unwrap() 6880 .trim(), 6881 "0001:00:01.0" 6882 ); 6883 6884 // Check both if /dev/vdd exists and if the block size is 128M. 6885 assert_eq!( 6886 guest 6887 .ssh_command("lsblk | grep vdd | grep -c 128M") 6888 .unwrap() 6889 .trim() 6890 .parse::<u32>() 6891 .unwrap_or_default(), 6892 1 6893 ); 6894 6895 // Write some content to the block device we've just plugged. 6896 guest 6897 .ssh_command("sudo bash -c 'echo foobar > /dev/vdd'") 6898 .unwrap(); 6899 6900 // Check we can read the content back. 6901 assert_eq!( 6902 guest.ssh_command("sudo head -1 /dev/vdd").unwrap().trim(), 6903 "foobar" 6904 ); 6905 6906 // Unplug the device 6907 let cmd_success = remote_command(&api_socket, "remove-device", Some("myvdpa0")); 6908 assert!(cmd_success); 6909 thread::sleep(std::time::Duration::new(10, 0)); 6910 6911 // Check /dev/vdd doesn't exist anymore 6912 assert_eq!( 6913 guest 6914 .ssh_command("lsblk | grep -c vdd || true") 6915 .unwrap() 6916 .trim() 6917 .parse::<u32>() 6918 .unwrap_or(1), 6919 0 6920 ); 6921 }); 6922 6923 kill_child(&mut child); 6924 let output = child.wait_with_output().unwrap(); 6925 6926 handle_child_output(r, &output); 6927 } 6928 6929 #[test] 6930 #[cfg(target_arch = "x86_64")] 6931 #[ignore = "See #5756"] 6932 fn test_vdpa_net() { 6933 // Before trying to run the test, verify the vdpa_sim_net module is correctly loaded. 6934 if !exec_host_command_status("lsmod | grep vdpa_sim_net").success() { 6935 return; 6936 } 6937 6938 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 6939 let guest = Guest::new(Box::new(focal)); 6940 6941 let kernel_path = direct_kernel_boot_path(); 6942 6943 let mut child = GuestCommand::new(&guest) 6944 .args(["--cpus", "boot=2"]) 6945 .args(["--memory", "size=512M,hugepages=on"]) 6946 .args(["--kernel", kernel_path.to_str().unwrap()]) 6947 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6948 .default_disks() 6949 .default_net() 6950 .args(["--vdpa", "path=/dev/vhost-vdpa-2,num_queues=2"]) 6951 .capture_output() 6952 .spawn() 6953 .unwrap(); 6954 6955 let r = std::panic::catch_unwind(|| { 6956 guest.wait_vm_boot(None).unwrap(); 6957 6958 // Check we can find network interface related to vDPA device 6959 assert_eq!( 6960 guest 6961 .ssh_command("ip -o link | grep -c ens6") 6962 .unwrap() 6963 .trim() 6964 .parse::<u32>() 6965 .unwrap_or(0), 6966 1 6967 ); 6968 6969 guest 6970 .ssh_command("sudo ip addr add 172.16.1.2/24 dev ens6") 6971 .unwrap(); 6972 guest.ssh_command("sudo ip link set up dev ens6").unwrap(); 6973 6974 // Check there is no packet yet on both TX/RX of the network interface 6975 assert_eq!( 6976 guest 6977 .ssh_command("ip -j -p -s link show ens6 | grep -c '\"packets\": 0'") 6978 .unwrap() 6979 .trim() 6980 .parse::<u32>() 6981 .unwrap_or(0), 6982 2 6983 ); 6984 6985 // Send 6 packets with ping command 6986 guest.ssh_command("ping 172.16.1.10 -c 6 || true").unwrap(); 6987 6988 // Check we can find 6 packets on both TX/RX of the network interface 6989 assert_eq!( 6990 guest 6991 .ssh_command("ip -j -p -s link show ens6 | grep -c '\"packets\": 6'") 6992 .unwrap() 6993 .trim() 6994 .parse::<u32>() 6995 .unwrap_or(0), 6996 2 6997 ); 6998 6999 // No need to check for hotplug as we already tested it through 7000 // test_vdpa_block() 7001 }); 7002 7003 kill_child(&mut child); 7004 let output = child.wait_with_output().unwrap(); 7005 7006 handle_child_output(r, &output); 7007 } 7008 7009 #[test] 7010 #[cfg(target_arch = "x86_64")] 7011 fn test_tpm() { 7012 let focal = UbuntuDiskConfig::new(JAMMY_IMAGE_NAME.to_string()); 7013 let guest = Guest::new(Box::new(focal)); 7014 7015 let (mut swtpm_command, swtpm_socket_path) = prepare_swtpm_daemon(&guest.tmp_dir); 7016 7017 let mut guest_cmd = GuestCommand::new(&guest); 7018 guest_cmd 7019 .args(["--cpus", "boot=1"]) 7020 .args(["--memory", "size=1G"]) 7021 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 7022 .args(["--tpm", &format!("socket={swtpm_socket_path}")]) 7023 .capture_output() 7024 .default_disks() 7025 .default_net(); 7026 7027 // Start swtpm daemon 7028 let mut swtpm_child = swtpm_command.spawn().unwrap(); 7029 thread::sleep(std::time::Duration::new(10, 0)); 7030 let mut child = guest_cmd.spawn().unwrap(); 7031 let r = std::panic::catch_unwind(|| { 7032 guest.wait_vm_boot(None).unwrap(); 7033 assert_eq!( 7034 guest.ssh_command("ls /dev/tpm0").unwrap().trim(), 7035 "/dev/tpm0" 7036 ); 7037 guest.ssh_command("sudo tpm2_selftest -f").unwrap(); 7038 guest 7039 .ssh_command("echo 'hello' > /tmp/checksum_test; ") 7040 .unwrap(); 7041 guest.ssh_command("cmp <(sudo tpm2_pcrevent /tmp/checksum_test | grep sha256 | awk '{print $2}') <(sha256sum /tmp/checksum_test| awk '{print $1}')").unwrap(); 7042 }); 7043 7044 let _ = swtpm_child.kill(); 7045 let _d_out = swtpm_child.wait_with_output().unwrap(); 7046 7047 kill_child(&mut child); 7048 let output = child.wait_with_output().unwrap(); 7049 7050 handle_child_output(r, &output); 7051 } 7052 7053 #[test] 7054 #[cfg(target_arch = "x86_64")] 7055 fn test_double_tty() { 7056 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7057 let guest = Guest::new(Box::new(focal)); 7058 let mut cmd = GuestCommand::new(&guest); 7059 let api_socket = temp_api_path(&guest.tmp_dir); 7060 let tty_str: &str = "console=hvc0 earlyprintk=ttyS0 "; 7061 // linux printk module enable console log. 7062 let con_dis_str: &str = "console [hvc0] enabled"; 7063 // linux printk module disable console log. 7064 let con_enb_str: &str = "bootconsole [earlyser0] disabled"; 7065 7066 let kernel_path = direct_kernel_boot_path(); 7067 7068 cmd.args(["--cpus", "boot=1"]) 7069 .args(["--memory", "size=512M"]) 7070 .args(["--kernel", kernel_path.to_str().unwrap()]) 7071 .args([ 7072 "--cmdline", 7073 DIRECT_KERNEL_BOOT_CMDLINE 7074 .replace("console=hvc0 ", tty_str) 7075 .as_str(), 7076 ]) 7077 .capture_output() 7078 .default_disks() 7079 .default_net() 7080 .args(["--serial", "tty"]) 7081 .args(["--console", "tty"]) 7082 .args(["--api-socket", &api_socket]); 7083 7084 let mut child = cmd.spawn().unwrap(); 7085 7086 let mut r = std::panic::catch_unwind(|| { 7087 guest.wait_vm_boot(None).unwrap(); 7088 }); 7089 7090 kill_child(&mut child); 7091 let output = child.wait_with_output().unwrap(); 7092 7093 if r.is_ok() { 7094 r = std::panic::catch_unwind(|| { 7095 let s = String::from_utf8_lossy(&output.stdout); 7096 assert!(s.contains(tty_str)); 7097 assert!(s.contains(con_dis_str)); 7098 assert!(s.contains(con_enb_str)); 7099 }); 7100 } 7101 7102 handle_child_output(r, &output); 7103 } 7104 7105 #[test] 7106 #[cfg(target_arch = "x86_64")] 7107 fn test_nmi() { 7108 let jammy = UbuntuDiskConfig::new(JAMMY_IMAGE_NAME.to_string()); 7109 let guest = Guest::new(Box::new(jammy)); 7110 let api_socket = temp_api_path(&guest.tmp_dir); 7111 let event_path = temp_event_monitor_path(&guest.tmp_dir); 7112 7113 let kernel_path = direct_kernel_boot_path(); 7114 let cmd_line = format!("{} {}", DIRECT_KERNEL_BOOT_CMDLINE, "unknown_nmi_panic=1"); 7115 7116 let mut cmd = GuestCommand::new(&guest); 7117 cmd.args(["--cpus", "boot=4"]) 7118 .args(["--memory", "size=512M"]) 7119 .args(["--kernel", kernel_path.to_str().unwrap()]) 7120 .args(["--cmdline", cmd_line.as_str()]) 7121 .default_disks() 7122 .args(["--net", guest.default_net_string().as_str()]) 7123 .args(["--pvpanic"]) 7124 .args(["--api-socket", &api_socket]) 7125 .args(["--event-monitor", format!("path={event_path}").as_str()]) 7126 .capture_output(); 7127 7128 let mut child = cmd.spawn().unwrap(); 7129 7130 let r = std::panic::catch_unwind(|| { 7131 guest.wait_vm_boot(None).unwrap(); 7132 7133 assert!(remote_command(&api_socket, "nmi", None)); 7134 7135 // Wait a while for guest 7136 thread::sleep(std::time::Duration::new(3, 0)); 7137 7138 let expected_sequential_events = [&MetaEvent { 7139 event: "panic".to_string(), 7140 device_id: None, 7141 }]; 7142 assert!(check_latest_events_exact( 7143 &expected_sequential_events, 7144 &event_path 7145 )); 7146 }); 7147 7148 kill_child(&mut child); 7149 let output = child.wait_with_output().unwrap(); 7150 7151 handle_child_output(r, &output); 7152 } 7153 } 7154 7155 mod dbus_api { 7156 use crate::*; 7157 7158 // Start cloud-hypervisor with no VM parameters, running both the HTTP 7159 // and DBus APIs. Alternate calls to the external APIs (HTTP and DBus) 7160 // to create a VM, boot it, and verify that it can be shut down and then 7161 // booted again. 7162 #[test] 7163 fn test_api_dbus_and_http_interleaved() { 7164 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7165 let guest = Guest::new(Box::new(focal)); 7166 let dbus_api = TargetApi::new_dbus_api(&guest.tmp_dir); 7167 let http_api = TargetApi::new_http_api(&guest.tmp_dir); 7168 7169 let mut child = GuestCommand::new(&guest) 7170 .args(dbus_api.guest_args()) 7171 .args(http_api.guest_args()) 7172 .capture_output() 7173 .spawn() 7174 .unwrap(); 7175 7176 thread::sleep(std::time::Duration::new(1, 0)); 7177 7178 // Verify API servers are running 7179 assert!(dbus_api.remote_command("ping", None)); 7180 assert!(http_api.remote_command("ping", None)); 7181 7182 // Create the VM first 7183 let cpu_count: u8 = 4; 7184 let request_body = guest.api_create_body( 7185 cpu_count, 7186 direct_kernel_boot_path().to_str().unwrap(), 7187 DIRECT_KERNEL_BOOT_CMDLINE, 7188 ); 7189 7190 let temp_config_path = guest.tmp_dir.as_path().join("config"); 7191 std::fs::write(&temp_config_path, request_body).unwrap(); 7192 let create_config = temp_config_path.as_os_str().to_str().unwrap(); 7193 7194 let r = std::panic::catch_unwind(|| { 7195 // Create the VM 7196 assert!(dbus_api.remote_command("create", Some(create_config),)); 7197 7198 // Then boot it 7199 assert!(http_api.remote_command("boot", None)); 7200 guest.wait_vm_boot(None).unwrap(); 7201 7202 // Check that the VM booted as expected 7203 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 7204 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 7205 7206 // Sync and shutdown without powering off to prevent filesystem 7207 // corruption. 7208 guest.ssh_command("sync").unwrap(); 7209 guest.ssh_command("sudo shutdown -H now").unwrap(); 7210 7211 // Wait for the guest to be fully shutdown 7212 thread::sleep(std::time::Duration::new(20, 0)); 7213 7214 // Then shutdown the VM 7215 assert!(dbus_api.remote_command("shutdown", None)); 7216 7217 // Then boot it again 7218 assert!(http_api.remote_command("boot", None)); 7219 guest.wait_vm_boot(None).unwrap(); 7220 7221 // Check that the VM booted as expected 7222 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 7223 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 7224 }); 7225 7226 kill_child(&mut child); 7227 let output = child.wait_with_output().unwrap(); 7228 7229 handle_child_output(r, &output); 7230 } 7231 7232 #[test] 7233 fn test_api_dbus_create_boot() { 7234 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7235 let guest = Guest::new(Box::new(focal)); 7236 7237 _test_api_create_boot(TargetApi::new_dbus_api(&guest.tmp_dir), guest) 7238 } 7239 7240 #[test] 7241 fn test_api_dbus_shutdown() { 7242 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7243 let guest = Guest::new(Box::new(focal)); 7244 7245 _test_api_shutdown(TargetApi::new_dbus_api(&guest.tmp_dir), guest) 7246 } 7247 7248 #[test] 7249 fn test_api_dbus_delete() { 7250 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7251 let guest = Guest::new(Box::new(focal)); 7252 7253 _test_api_delete(TargetApi::new_dbus_api(&guest.tmp_dir), guest); 7254 } 7255 7256 #[test] 7257 fn test_api_dbus_pause_resume() { 7258 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7259 let guest = Guest::new(Box::new(focal)); 7260 7261 _test_api_pause_resume(TargetApi::new_dbus_api(&guest.tmp_dir), guest) 7262 } 7263 } 7264 7265 mod common_sequential { 7266 use std::fs::remove_dir_all; 7267 7268 use crate::*; 7269 7270 #[test] 7271 #[cfg(not(feature = "mshv"))] 7272 fn test_memory_mergeable_on() { 7273 test_memory_mergeable(true) 7274 } 7275 7276 fn snapshot_and_check_events(api_socket: &str, snapshot_dir: &str, event_path: &str) { 7277 // Pause the VM 7278 assert!(remote_command(api_socket, "pause", None)); 7279 let latest_events: [&MetaEvent; 2] = [ 7280 &MetaEvent { 7281 event: "pausing".to_string(), 7282 device_id: None, 7283 }, 7284 &MetaEvent { 7285 event: "paused".to_string(), 7286 device_id: None, 7287 }, 7288 ]; 7289 // See: #5938 7290 thread::sleep(std::time::Duration::new(1, 0)); 7291 assert!(check_latest_events_exact(&latest_events, event_path)); 7292 7293 // Take a snapshot from the VM 7294 assert!(remote_command( 7295 api_socket, 7296 "snapshot", 7297 Some(format!("file://{snapshot_dir}").as_str()), 7298 )); 7299 7300 // Wait to make sure the snapshot is completed 7301 thread::sleep(std::time::Duration::new(10, 0)); 7302 7303 let latest_events = [ 7304 &MetaEvent { 7305 event: "snapshotting".to_string(), 7306 device_id: None, 7307 }, 7308 &MetaEvent { 7309 event: "snapshotted".to_string(), 7310 device_id: None, 7311 }, 7312 ]; 7313 // See: #5938 7314 thread::sleep(std::time::Duration::new(1, 0)); 7315 assert!(check_latest_events_exact(&latest_events, event_path)); 7316 } 7317 7318 // One thing to note about this test. The virtio-net device is heavily used 7319 // through each ssh command. There's no need to perform a dedicated test to 7320 // verify the migration went well for virtio-net. 7321 #[test] 7322 #[cfg(not(feature = "mshv"))] 7323 fn test_snapshot_restore_hotplug_virtiomem() { 7324 _test_snapshot_restore(true); 7325 } 7326 7327 #[test] 7328 fn test_snapshot_restore_basic() { 7329 _test_snapshot_restore(false); 7330 } 7331 7332 fn _test_snapshot_restore(use_hotplug: bool) { 7333 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7334 let guest = Guest::new(Box::new(focal)); 7335 let kernel_path = direct_kernel_boot_path(); 7336 7337 let api_socket_source = format!("{}.1", temp_api_path(&guest.tmp_dir)); 7338 7339 let net_id = "net123"; 7340 let net_params = format!( 7341 "id={},tap=,mac={},ip={},mask=255.255.255.0", 7342 net_id, guest.network.guest_mac, guest.network.host_ip 7343 ); 7344 let mut mem_params = "size=2G"; 7345 7346 if use_hotplug { 7347 mem_params = "size=2G,hotplug_method=virtio-mem,hotplug_size=32G" 7348 } 7349 7350 let cloudinit_params = format!( 7351 "path={},iommu=on", 7352 guest.disk_config.disk(DiskType::CloudInit).unwrap() 7353 ); 7354 7355 let socket = temp_vsock_path(&guest.tmp_dir); 7356 let event_path = temp_event_monitor_path(&guest.tmp_dir); 7357 7358 let mut child = GuestCommand::new(&guest) 7359 .args(["--api-socket", &api_socket_source]) 7360 .args(["--event-monitor", format!("path={event_path}").as_str()]) 7361 .args(["--cpus", "boot=4"]) 7362 .args(["--memory", mem_params]) 7363 .args(["--balloon", "size=0"]) 7364 .args(["--kernel", kernel_path.to_str().unwrap()]) 7365 .args([ 7366 "--disk", 7367 format!( 7368 "path={}", 7369 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 7370 ) 7371 .as_str(), 7372 cloudinit_params.as_str(), 7373 ]) 7374 .args(["--net", net_params.as_str()]) 7375 .args(["--vsock", format!("cid=3,socket={socket}").as_str()]) 7376 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 7377 .capture_output() 7378 .spawn() 7379 .unwrap(); 7380 7381 let console_text = String::from("On a branch floating down river a cricket, singing."); 7382 // Create the snapshot directory 7383 let snapshot_dir = temp_snapshot_dir_path(&guest.tmp_dir); 7384 7385 let r = std::panic::catch_unwind(|| { 7386 guest.wait_vm_boot(None).unwrap(); 7387 7388 // Check the number of vCPUs 7389 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 4); 7390 // Check the guest RAM 7391 assert!(guest.get_total_memory().unwrap_or_default() > 1_920_000); 7392 if use_hotplug { 7393 // Increase guest RAM with virtio-mem 7394 resize_command( 7395 &api_socket_source, 7396 None, 7397 Some(6 << 30), 7398 None, 7399 Some(&event_path), 7400 ); 7401 thread::sleep(std::time::Duration::new(5, 0)); 7402 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 7403 // Use balloon to remove RAM from the VM 7404 resize_command( 7405 &api_socket_source, 7406 None, 7407 None, 7408 Some(1 << 30), 7409 Some(&event_path), 7410 ); 7411 thread::sleep(std::time::Duration::new(5, 0)); 7412 let total_memory = guest.get_total_memory().unwrap_or_default(); 7413 assert!(total_memory > 4_800_000); 7414 assert!(total_memory < 5_760_000); 7415 } 7416 // Check the guest virtio-devices, e.g. block, rng, vsock, console, and net 7417 guest.check_devices_common(Some(&socket), Some(&console_text), None); 7418 7419 // x86_64: We check that removing and adding back the virtio-net device 7420 // does not break the snapshot/restore support for virtio-pci. 7421 // This is an important thing to test as the hotplug will 7422 // trigger a PCI BAR reprogramming, which is a good way of 7423 // checking if the stored resources are correctly restored. 7424 // Unplug the virtio-net device 7425 // AArch64: Device hotplug is currently not supported, skipping here. 7426 #[cfg(target_arch = "x86_64")] 7427 { 7428 assert!(remote_command( 7429 &api_socket_source, 7430 "remove-device", 7431 Some(net_id), 7432 )); 7433 thread::sleep(std::time::Duration::new(10, 0)); 7434 let latest_events = [&MetaEvent { 7435 event: "device-removed".to_string(), 7436 device_id: Some(net_id.to_string()), 7437 }]; 7438 // See: #5938 7439 thread::sleep(std::time::Duration::new(1, 0)); 7440 assert!(check_latest_events_exact(&latest_events, &event_path)); 7441 7442 // Plug the virtio-net device again 7443 assert!(remote_command( 7444 &api_socket_source, 7445 "add-net", 7446 Some(net_params.as_str()), 7447 )); 7448 thread::sleep(std::time::Duration::new(10, 0)); 7449 } 7450 7451 snapshot_and_check_events(&api_socket_source, &snapshot_dir, &event_path); 7452 }); 7453 7454 // Shutdown the source VM and check console output 7455 kill_child(&mut child); 7456 let output = child.wait_with_output().unwrap(); 7457 handle_child_output(r, &output); 7458 7459 let r = std::panic::catch_unwind(|| { 7460 assert!(String::from_utf8_lossy(&output.stdout).contains(&console_text)); 7461 }); 7462 7463 handle_child_output(r, &output); 7464 7465 // Remove the vsock socket file. 7466 Command::new("rm") 7467 .arg("-f") 7468 .arg(socket.as_str()) 7469 .output() 7470 .unwrap(); 7471 7472 let api_socket_restored = format!("{}.2", temp_api_path(&guest.tmp_dir)); 7473 let event_path_restored = format!("{}.2", temp_event_monitor_path(&guest.tmp_dir)); 7474 7475 // Restore the VM from the snapshot 7476 let mut child = GuestCommand::new(&guest) 7477 .args(["--api-socket", &api_socket_restored]) 7478 .args([ 7479 "--event-monitor", 7480 format!("path={event_path_restored}").as_str(), 7481 ]) 7482 .args([ 7483 "--restore", 7484 format!("source_url=file://{snapshot_dir}").as_str(), 7485 ]) 7486 .capture_output() 7487 .spawn() 7488 .unwrap(); 7489 7490 // Wait for the VM to be restored 7491 thread::sleep(std::time::Duration::new(20, 0)); 7492 let expected_events = [ 7493 &MetaEvent { 7494 event: "starting".to_string(), 7495 device_id: None, 7496 }, 7497 &MetaEvent { 7498 event: "activated".to_string(), 7499 device_id: Some("__console".to_string()), 7500 }, 7501 &MetaEvent { 7502 event: "activated".to_string(), 7503 device_id: Some("__rng".to_string()), 7504 }, 7505 &MetaEvent { 7506 event: "restoring".to_string(), 7507 device_id: None, 7508 }, 7509 ]; 7510 assert!(check_sequential_events( 7511 &expected_events, 7512 &event_path_restored 7513 )); 7514 let latest_events = [&MetaEvent { 7515 event: "restored".to_string(), 7516 device_id: None, 7517 }]; 7518 assert!(check_latest_events_exact( 7519 &latest_events, 7520 &event_path_restored 7521 )); 7522 7523 // Remove the snapshot dir 7524 let _ = remove_dir_all(snapshot_dir.as_str()); 7525 7526 let r = std::panic::catch_unwind(|| { 7527 // Resume the VM 7528 assert!(remote_command(&api_socket_restored, "resume", None)); 7529 // There is no way that we can ensure the 'write()' to the 7530 // event file is completed when the 'resume' request is 7531 // returned successfully, because the 'write()' was done 7532 // asynchronously from a different thread of Cloud 7533 // Hypervisor (e.g. the event-monitor thread). 7534 thread::sleep(std::time::Duration::new(1, 0)); 7535 let latest_events = [ 7536 &MetaEvent { 7537 event: "resuming".to_string(), 7538 device_id: None, 7539 }, 7540 &MetaEvent { 7541 event: "resumed".to_string(), 7542 device_id: None, 7543 }, 7544 ]; 7545 assert!(check_latest_events_exact( 7546 &latest_events, 7547 &event_path_restored 7548 )); 7549 7550 // Perform same checks to validate VM has been properly restored 7551 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 4); 7552 let total_memory = guest.get_total_memory().unwrap_or_default(); 7553 if !use_hotplug { 7554 assert!(total_memory > 1_920_000); 7555 } else { 7556 assert!(total_memory > 4_800_000); 7557 assert!(total_memory < 5_760_000); 7558 // Deflate balloon to restore entire RAM to the VM 7559 resize_command(&api_socket_restored, None, None, Some(0), None); 7560 thread::sleep(std::time::Duration::new(5, 0)); 7561 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 7562 // Decrease guest RAM with virtio-mem 7563 resize_command(&api_socket_restored, None, Some(5 << 30), None, None); 7564 thread::sleep(std::time::Duration::new(5, 0)); 7565 let total_memory = guest.get_total_memory().unwrap_or_default(); 7566 assert!(total_memory > 4_800_000); 7567 assert!(total_memory < 5_760_000); 7568 } 7569 7570 guest.check_devices_common(Some(&socket), Some(&console_text), None); 7571 }); 7572 // Shutdown the target VM and check console output 7573 kill_child(&mut child); 7574 let output = child.wait_with_output().unwrap(); 7575 handle_child_output(r, &output); 7576 7577 let r = std::panic::catch_unwind(|| { 7578 assert!(String::from_utf8_lossy(&output.stdout).contains(&console_text)); 7579 }); 7580 7581 handle_child_output(r, &output); 7582 } 7583 7584 #[test] 7585 #[cfg_attr(target_arch = "aarch64", ignore = "See #6970")] 7586 fn test_snapshot_restore_with_fd() { 7587 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7588 let guest = Guest::new(Box::new(focal)); 7589 let kernel_path = direct_kernel_boot_path(); 7590 7591 let api_socket_source = format!("{}.1", temp_api_path(&guest.tmp_dir)); 7592 7593 let net_id = "net123"; 7594 let num_queue_pairs: usize = 2; 7595 // use a name that does not conflict with tap dev created from other tests 7596 let tap_name = "chtap999"; 7597 use std::str::FromStr; 7598 let taps = net_util::open_tap( 7599 Some(tap_name), 7600 Some(std::net::Ipv4Addr::from_str(&guest.network.host_ip).unwrap()), 7601 None, 7602 &mut None, 7603 None, 7604 num_queue_pairs, 7605 Some(libc::O_RDWR | libc::O_NONBLOCK), 7606 ) 7607 .unwrap(); 7608 let net_params = format!( 7609 "id={},fd=[{},{}],mac={},ip={},mask=255.255.255.0,num_queues={}", 7610 net_id, 7611 taps[0].as_raw_fd(), 7612 taps[1].as_raw_fd(), 7613 guest.network.guest_mac, 7614 guest.network.host_ip, 7615 num_queue_pairs * 2 7616 ); 7617 7618 let cloudinit_params = format!( 7619 "path={},iommu=on", 7620 guest.disk_config.disk(DiskType::CloudInit).unwrap() 7621 ); 7622 7623 let n_cpu = 2; 7624 let event_path = temp_event_monitor_path(&guest.tmp_dir); 7625 7626 let mut child = GuestCommand::new(&guest) 7627 .args(["--api-socket", &api_socket_source]) 7628 .args(["--event-monitor", format!("path={event_path}").as_str()]) 7629 .args(["--cpus", format!("boot={}", n_cpu).as_str()]) 7630 .args(["--memory", "size=1G"]) 7631 .args(["--kernel", kernel_path.to_str().unwrap()]) 7632 .args([ 7633 "--disk", 7634 format!( 7635 "path={}", 7636 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 7637 ) 7638 .as_str(), 7639 cloudinit_params.as_str(), 7640 ]) 7641 .args(["--net", net_params.as_str()]) 7642 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 7643 .capture_output() 7644 .spawn() 7645 .unwrap(); 7646 7647 let console_text = String::from("On a branch floating down river a cricket, singing."); 7648 // Create the snapshot directory 7649 let snapshot_dir = temp_snapshot_dir_path(&guest.tmp_dir); 7650 7651 let r = std::panic::catch_unwind(|| { 7652 guest.wait_vm_boot(None).unwrap(); 7653 7654 // close the fds after VM boots, as CH duplicates them before using 7655 for tap in taps.iter() { 7656 unsafe { libc::close(tap.as_raw_fd()) }; 7657 } 7658 7659 // Check the number of vCPUs 7660 assert_eq!(guest.get_cpu_count().unwrap_or_default(), n_cpu); 7661 // Check the guest RAM 7662 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 7663 7664 // Check the guest virtio-devices, e.g. block, rng, vsock, console, and net 7665 guest.check_devices_common(None, Some(&console_text), None); 7666 7667 snapshot_and_check_events(&api_socket_source, &snapshot_dir, &event_path); 7668 }); 7669 7670 // Shutdown the source VM and check console output 7671 kill_child(&mut child); 7672 let output = child.wait_with_output().unwrap(); 7673 handle_child_output(r, &output); 7674 7675 let r = std::panic::catch_unwind(|| { 7676 assert!(String::from_utf8_lossy(&output.stdout).contains(&console_text)); 7677 }); 7678 7679 handle_child_output(r, &output); 7680 7681 let api_socket_restored = format!("{}.2", temp_api_path(&guest.tmp_dir)); 7682 let event_path_restored = format!("{}.2", temp_event_monitor_path(&guest.tmp_dir)); 7683 7684 // Restore the VM from the snapshot 7685 let mut child = GuestCommand::new(&guest) 7686 .args(["--api-socket", &api_socket_restored]) 7687 .args([ 7688 "--event-monitor", 7689 format!("path={event_path_restored}").as_str(), 7690 ]) 7691 .capture_output() 7692 .spawn() 7693 .unwrap(); 7694 thread::sleep(std::time::Duration::new(2, 0)); 7695 7696 let taps = net_util::open_tap( 7697 Some(tap_name), 7698 Some(std::net::Ipv4Addr::from_str(&guest.network.host_ip).unwrap()), 7699 None, 7700 &mut None, 7701 None, 7702 num_queue_pairs, 7703 Some(libc::O_RDWR | libc::O_NONBLOCK), 7704 ) 7705 .unwrap(); 7706 let restore_params = format!( 7707 "source_url=file://{},net_fds=[{}@[{},{}]]", 7708 snapshot_dir, 7709 net_id, 7710 taps[0].as_raw_fd(), 7711 taps[1].as_raw_fd() 7712 ); 7713 assert!(remote_command( 7714 &api_socket_restored, 7715 "restore", 7716 Some(restore_params.as_str()) 7717 )); 7718 7719 // Wait for the VM to be restored 7720 thread::sleep(std::time::Duration::new(20, 0)); 7721 7722 // close the fds as CH duplicates them before using 7723 for tap in taps.iter() { 7724 unsafe { libc::close(tap.as_raw_fd()) }; 7725 } 7726 7727 let expected_events = [ 7728 &MetaEvent { 7729 event: "starting".to_string(), 7730 device_id: None, 7731 }, 7732 &MetaEvent { 7733 event: "activated".to_string(), 7734 device_id: Some("__console".to_string()), 7735 }, 7736 &MetaEvent { 7737 event: "activated".to_string(), 7738 device_id: Some("__rng".to_string()), 7739 }, 7740 &MetaEvent { 7741 event: "restoring".to_string(), 7742 device_id: None, 7743 }, 7744 ]; 7745 assert!(check_sequential_events( 7746 &expected_events, 7747 &event_path_restored 7748 )); 7749 let latest_events = [&MetaEvent { 7750 event: "restored".to_string(), 7751 device_id: None, 7752 }]; 7753 assert!(check_latest_events_exact( 7754 &latest_events, 7755 &event_path_restored 7756 )); 7757 7758 // Remove the snapshot dir 7759 let _ = remove_dir_all(snapshot_dir.as_str()); 7760 7761 let r = std::panic::catch_unwind(|| { 7762 // Resume the VM 7763 assert!(remote_command(&api_socket_restored, "resume", None)); 7764 // There is no way that we can ensure the 'write()' to the 7765 // event file is completed when the 'resume' request is 7766 // returned successfully, because the 'write()' was done 7767 // asynchronously from a different thread of Cloud 7768 // Hypervisor (e.g. the event-monitor thread). 7769 thread::sleep(std::time::Duration::new(1, 0)); 7770 let latest_events = [ 7771 &MetaEvent { 7772 event: "resuming".to_string(), 7773 device_id: None, 7774 }, 7775 &MetaEvent { 7776 event: "resumed".to_string(), 7777 device_id: None, 7778 }, 7779 ]; 7780 assert!(check_latest_events_exact( 7781 &latest_events, 7782 &event_path_restored 7783 )); 7784 7785 // Perform same checks to validate VM has been properly restored 7786 assert_eq!(guest.get_cpu_count().unwrap_or_default(), n_cpu); 7787 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 7788 7789 guest.check_devices_common(None, Some(&console_text), None); 7790 }); 7791 // Shutdown the target VM and check console output 7792 kill_child(&mut child); 7793 let output = child.wait_with_output().unwrap(); 7794 handle_child_output(r, &output); 7795 7796 let r = std::panic::catch_unwind(|| { 7797 assert!(String::from_utf8_lossy(&output.stdout).contains(&console_text)); 7798 }); 7799 7800 handle_child_output(r, &output); 7801 } 7802 7803 #[test] 7804 #[cfg(target_arch = "x86_64")] 7805 fn test_snapshot_restore_pvpanic() { 7806 _test_snapshot_restore_devices(true); 7807 } 7808 7809 fn _test_snapshot_restore_devices(pvpanic: bool) { 7810 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 7811 let guest = Guest::new(Box::new(focal)); 7812 let kernel_path = direct_kernel_boot_path(); 7813 7814 let api_socket_source = format!("{}.1", temp_api_path(&guest.tmp_dir)); 7815 7816 let device_params = { 7817 let mut data = vec![]; 7818 if pvpanic { 7819 data.push("--pvpanic"); 7820 } 7821 data 7822 }; 7823 7824 let socket = temp_vsock_path(&guest.tmp_dir); 7825 let event_path = temp_event_monitor_path(&guest.tmp_dir); 7826 7827 let mut child = GuestCommand::new(&guest) 7828 .args(["--api-socket", &api_socket_source]) 7829 .args(["--event-monitor", format!("path={}", event_path).as_str()]) 7830 .args(["--cpus", "boot=2"]) 7831 .args(["--memory", "size=1G"]) 7832 .args(["--kernel", kernel_path.to_str().unwrap()]) 7833 .default_disks() 7834 .default_net() 7835 .args(["--vsock", format!("cid=3,socket={}", socket).as_str()]) 7836 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 7837 .args(device_params) 7838 .capture_output() 7839 .spawn() 7840 .unwrap(); 7841 7842 let console_text = String::from("On a branch floating down river a cricket, singing."); 7843 // Create the snapshot directory 7844 let snapshot_dir = temp_snapshot_dir_path(&guest.tmp_dir); 7845 7846 let r = std::panic::catch_unwind(|| { 7847 guest.wait_vm_boot(None).unwrap(); 7848 7849 // Check the number of vCPUs 7850 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 7851 7852 snapshot_and_check_events(&api_socket_source, &snapshot_dir, &event_path); 7853 }); 7854 7855 // Shutdown the source VM and check console output 7856 kill_child(&mut child); 7857 let output = child.wait_with_output().unwrap(); 7858 handle_child_output(r, &output); 7859 7860 // Remove the vsock socket file. 7861 Command::new("rm") 7862 .arg("-f") 7863 .arg(socket.as_str()) 7864 .output() 7865 .unwrap(); 7866 7867 let api_socket_restored = format!("{}.2", temp_api_path(&guest.tmp_dir)); 7868 let event_path_restored = format!("{}.2", temp_event_monitor_path(&guest.tmp_dir)); 7869 7870 // Restore the VM from the snapshot 7871 let mut child = GuestCommand::new(&guest) 7872 .args(["--api-socket", &api_socket_restored]) 7873 .args([ 7874 "--event-monitor", 7875 format!("path={event_path_restored}").as_str(), 7876 ]) 7877 .args([ 7878 "--restore", 7879 format!("source_url=file://{snapshot_dir}").as_str(), 7880 ]) 7881 .capture_output() 7882 .spawn() 7883 .unwrap(); 7884 7885 // Wait for the VM to be restored 7886 thread::sleep(std::time::Duration::new(20, 0)); 7887 7888 let latest_events = [&MetaEvent { 7889 event: "restored".to_string(), 7890 device_id: None, 7891 }]; 7892 assert!(check_latest_events_exact( 7893 &latest_events, 7894 &event_path_restored 7895 )); 7896 7897 // Remove the snapshot dir 7898 let _ = remove_dir_all(snapshot_dir.as_str()); 7899 7900 let r = std::panic::catch_unwind(|| { 7901 // Resume the VM 7902 assert!(remote_command(&api_socket_restored, "resume", None)); 7903 // There is no way that we can ensure the 'write()' to the 7904 // event file is completed when the 'resume' request is 7905 // returned successfully, because the 'write()' was done 7906 // asynchronously from a different thread of Cloud 7907 // Hypervisor (e.g. the event-monitor thread). 7908 thread::sleep(std::time::Duration::new(1, 0)); 7909 let latest_events = [ 7910 &MetaEvent { 7911 event: "resuming".to_string(), 7912 device_id: None, 7913 }, 7914 &MetaEvent { 7915 event: "resumed".to_string(), 7916 device_id: None, 7917 }, 7918 ]; 7919 assert!(check_latest_events_exact( 7920 &latest_events, 7921 &event_path_restored 7922 )); 7923 7924 // Check the number of vCPUs 7925 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 7926 guest.check_devices_common(Some(&socket), Some(&console_text), None); 7927 7928 if pvpanic { 7929 // Trigger guest a panic 7930 make_guest_panic(&guest); 7931 // Wait a while for guest 7932 thread::sleep(std::time::Duration::new(10, 0)); 7933 7934 let expected_sequential_events = [&MetaEvent { 7935 event: "panic".to_string(), 7936 device_id: None, 7937 }]; 7938 assert!(check_latest_events_exact( 7939 &expected_sequential_events, 7940 &event_path_restored 7941 )); 7942 } 7943 }); 7944 // Shutdown the target VM and check console output 7945 kill_child(&mut child); 7946 let output = child.wait_with_output().unwrap(); 7947 handle_child_output(r, &output); 7948 7949 let r = std::panic::catch_unwind(|| { 7950 assert!(String::from_utf8_lossy(&output.stdout).contains(&console_text)); 7951 }); 7952 7953 handle_child_output(r, &output); 7954 } 7955 7956 #[test] 7957 fn test_virtio_pmem_persist_writes() { 7958 test_virtio_pmem(false, false) 7959 } 7960 } 7961 7962 mod windows { 7963 use once_cell::sync::Lazy; 7964 7965 use crate::*; 7966 7967 static NEXT_DISK_ID: Lazy<Mutex<u8>> = Lazy::new(|| Mutex::new(1)); 7968 7969 struct WindowsGuest { 7970 guest: Guest, 7971 auth: PasswordAuth, 7972 } 7973 7974 trait FsType { 7975 const FS_FAT: u8; 7976 const FS_NTFS: u8; 7977 } 7978 impl FsType for WindowsGuest { 7979 const FS_FAT: u8 = 0; 7980 const FS_NTFS: u8 = 1; 7981 } 7982 7983 impl WindowsGuest { 7984 fn new() -> Self { 7985 let disk = WindowsDiskConfig::new(WINDOWS_IMAGE_NAME.to_string()); 7986 let guest = Guest::new(Box::new(disk)); 7987 let auth = PasswordAuth { 7988 username: String::from("administrator"), 7989 password: String::from("Admin123"), 7990 }; 7991 7992 WindowsGuest { guest, auth } 7993 } 7994 7995 fn guest(&self) -> &Guest { 7996 &self.guest 7997 } 7998 7999 fn ssh_cmd(&self, cmd: &str) -> String { 8000 ssh_command_ip_with_auth( 8001 cmd, 8002 &self.auth, 8003 &self.guest.network.guest_ip, 8004 DEFAULT_SSH_RETRIES, 8005 DEFAULT_SSH_TIMEOUT, 8006 ) 8007 .unwrap() 8008 } 8009 8010 fn cpu_count(&self) -> u8 { 8011 self.ssh_cmd("powershell -Command \"(Get-CimInstance win32_computersystem).NumberOfLogicalProcessors\"") 8012 .trim() 8013 .parse::<u8>() 8014 .unwrap_or(0) 8015 } 8016 8017 fn ram_size(&self) -> usize { 8018 self.ssh_cmd("powershell -Command \"(Get-CimInstance win32_computersystem).TotalPhysicalMemory\"") 8019 .trim() 8020 .parse::<usize>() 8021 .unwrap_or(0) 8022 } 8023 8024 fn netdev_count(&self) -> u8 { 8025 self.ssh_cmd("powershell -Command \"netsh int ipv4 show interfaces | Select-String ethernet | Measure-Object -Line | Format-Table -HideTableHeaders\"") 8026 .trim() 8027 .parse::<u8>() 8028 .unwrap_or(0) 8029 } 8030 8031 fn disk_count(&self) -> u8 { 8032 self.ssh_cmd("powershell -Command \"Get-Disk | Measure-Object -Line | Format-Table -HideTableHeaders\"") 8033 .trim() 8034 .parse::<u8>() 8035 .unwrap_or(0) 8036 } 8037 8038 fn reboot(&self) { 8039 let _ = self.ssh_cmd("shutdown /r /t 0"); 8040 } 8041 8042 fn shutdown(&self) { 8043 let _ = self.ssh_cmd("shutdown /s /t 0"); 8044 } 8045 8046 fn run_dnsmasq(&self) -> std::process::Child { 8047 let listen_address = format!("--listen-address={}", self.guest.network.host_ip); 8048 let dhcp_host = format!( 8049 "--dhcp-host={},{}", 8050 self.guest.network.guest_mac, self.guest.network.guest_ip 8051 ); 8052 let dhcp_range = format!( 8053 "--dhcp-range=eth,{},{}", 8054 self.guest.network.guest_ip, self.guest.network.guest_ip 8055 ); 8056 8057 Command::new("dnsmasq") 8058 .arg("--no-daemon") 8059 .arg("--log-queries") 8060 .arg(listen_address.as_str()) 8061 .arg("--except-interface=lo") 8062 .arg("--bind-dynamic") // Allow listening to host_ip while the interface is not ready yet. 8063 .arg("--conf-file=/dev/null") 8064 .arg(dhcp_host.as_str()) 8065 .arg(dhcp_range.as_str()) 8066 .spawn() 8067 .unwrap() 8068 } 8069 8070 // TODO Cleanup image file explicitly after test, if there's some space issues. 8071 fn disk_new(&self, fs: u8, sz: usize) -> String { 8072 let mut guard = NEXT_DISK_ID.lock().unwrap(); 8073 let id = *guard; 8074 *guard = id + 1; 8075 8076 let img = PathBuf::from(format!("/tmp/test-hotplug-{id}.raw")); 8077 let _ = fs::remove_file(&img); 8078 8079 // Create an image file 8080 let out = Command::new("qemu-img") 8081 .args([ 8082 "create", 8083 "-f", 8084 "raw", 8085 img.to_str().unwrap(), 8086 format!("{sz}m").as_str(), 8087 ]) 8088 .output() 8089 .expect("qemu-img command failed") 8090 .stdout; 8091 println!("{out:?}"); 8092 8093 // Associate image to a loop device 8094 let out = Command::new("losetup") 8095 .args(["--show", "-f", img.to_str().unwrap()]) 8096 .output() 8097 .expect("failed to create loop device") 8098 .stdout; 8099 let _tmp = String::from_utf8_lossy(&out); 8100 let loop_dev = _tmp.trim(); 8101 println!("{out:?}"); 8102 8103 // Create a partition table 8104 // echo 'type=7' | sudo sfdisk "${LOOP}" 8105 let mut child = Command::new("sfdisk") 8106 .args([loop_dev]) 8107 .stdin(Stdio::piped()) 8108 .spawn() 8109 .unwrap(); 8110 let stdin = child.stdin.as_mut().expect("failed to open stdin"); 8111 stdin 8112 .write_all("type=7".as_bytes()) 8113 .expect("failed to write stdin"); 8114 let out = child.wait_with_output().expect("sfdisk failed").stdout; 8115 println!("{out:?}"); 8116 8117 // Disengage the loop device 8118 let out = Command::new("losetup") 8119 .args(["-d", loop_dev]) 8120 .output() 8121 .expect("loop device not found") 8122 .stdout; 8123 println!("{out:?}"); 8124 8125 // Re-associate loop device pointing to the partition only 8126 let out = Command::new("losetup") 8127 .args([ 8128 "--show", 8129 "--offset", 8130 (512 * 2048).to_string().as_str(), 8131 "-f", 8132 img.to_str().unwrap(), 8133 ]) 8134 .output() 8135 .expect("failed to create loop device") 8136 .stdout; 8137 let _tmp = String::from_utf8_lossy(&out); 8138 let loop_dev = _tmp.trim(); 8139 println!("{out:?}"); 8140 8141 // Create filesystem. 8142 let fs_cmd = match fs { 8143 WindowsGuest::FS_FAT => "mkfs.msdos", 8144 WindowsGuest::FS_NTFS => "mkfs.ntfs", 8145 _ => panic!("Unknown filesystem type '{fs}'"), 8146 }; 8147 let out = Command::new(fs_cmd) 8148 .args([&loop_dev]) 8149 .output() 8150 .unwrap_or_else(|_| panic!("{fs_cmd} failed")) 8151 .stdout; 8152 println!("{out:?}"); 8153 8154 // Disengage the loop device 8155 let out = Command::new("losetup") 8156 .args(["-d", loop_dev]) 8157 .output() 8158 .unwrap_or_else(|_| panic!("loop device '{loop_dev}' not found")) 8159 .stdout; 8160 println!("{out:?}"); 8161 8162 img.to_str().unwrap().to_string() 8163 } 8164 8165 fn disks_set_rw(&self) { 8166 let _ = self.ssh_cmd("powershell -Command \"Get-Disk | Where-Object IsOffline -eq $True | Set-Disk -IsReadOnly $False\""); 8167 } 8168 8169 fn disks_online(&self) { 8170 let _ = self.ssh_cmd("powershell -Command \"Get-Disk | Where-Object IsOffline -eq $True | Set-Disk -IsOffline $False\""); 8171 } 8172 8173 fn disk_file_put(&self, fname: &str, data: &str) { 8174 let _ = self.ssh_cmd(&format!( 8175 "powershell -Command \"'{data}' | Set-Content -Path {fname}\"" 8176 )); 8177 } 8178 8179 fn disk_file_read(&self, fname: &str) -> String { 8180 self.ssh_cmd(&format!( 8181 "powershell -Command \"Get-Content -Path {fname}\"" 8182 )) 8183 } 8184 8185 fn wait_for_boot(&self) -> bool { 8186 let cmd = "dir /b c:\\ | find \"Windows\""; 8187 let tmo_max = 180; 8188 // The timeout increase by n*1+n*2+n*3+..., therefore the initial 8189 // interval must be small. 8190 let tmo_int = 2; 8191 let out = ssh_command_ip_with_auth( 8192 cmd, 8193 &self.auth, 8194 &self.guest.network.guest_ip, 8195 { 8196 let mut ret = 1; 8197 let mut tmo_acc = 0; 8198 loop { 8199 tmo_acc += tmo_int * ret; 8200 if tmo_acc >= tmo_max { 8201 break; 8202 } 8203 ret += 1; 8204 } 8205 ret 8206 }, 8207 tmo_int, 8208 ) 8209 .unwrap(); 8210 8211 if "Windows" == out.trim() { 8212 return true; 8213 } 8214 8215 false 8216 } 8217 } 8218 8219 fn vcpu_threads_count(pid: u32) -> u8 { 8220 // ps -T -p 12345 | grep vcpu | wc -l 8221 let out = Command::new("ps") 8222 .args(["-T", "-p", format!("{pid}").as_str()]) 8223 .output() 8224 .expect("ps command failed") 8225 .stdout; 8226 String::from_utf8_lossy(&out).matches("vcpu").count() as u8 8227 } 8228 8229 fn netdev_ctrl_threads_count(pid: u32) -> u8 { 8230 // ps -T -p 12345 | grep "_net[0-9]*_ctrl" | wc -l 8231 let out = Command::new("ps") 8232 .args(["-T", "-p", format!("{pid}").as_str()]) 8233 .output() 8234 .expect("ps command failed") 8235 .stdout; 8236 let mut n = 0; 8237 String::from_utf8_lossy(&out) 8238 .split_whitespace() 8239 .for_each(|s| n += (s.starts_with("_net") && s.ends_with("_ctrl")) as u8); // _net1_ctrl 8240 n 8241 } 8242 8243 fn disk_ctrl_threads_count(pid: u32) -> u8 { 8244 // ps -T -p 15782 | grep "_disk[0-9]*_q0" | wc -l 8245 let out = Command::new("ps") 8246 .args(["-T", "-p", format!("{pid}").as_str()]) 8247 .output() 8248 .expect("ps command failed") 8249 .stdout; 8250 let mut n = 0; 8251 String::from_utf8_lossy(&out) 8252 .split_whitespace() 8253 .for_each(|s| n += (s.starts_with("_disk") && s.ends_with("_q0")) as u8); // _disk0_q0, don't care about multiple queues as they're related to the same hdd 8254 n 8255 } 8256 8257 #[test] 8258 fn test_windows_guest() { 8259 let windows_guest = WindowsGuest::new(); 8260 8261 let mut child = GuestCommand::new(windows_guest.guest()) 8262 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 8263 .args(["--memory", "size=4G"]) 8264 .args(["--kernel", edk2_path().to_str().unwrap()]) 8265 .args(["--serial", "tty"]) 8266 .args(["--console", "off"]) 8267 .default_disks() 8268 .default_net() 8269 .capture_output() 8270 .spawn() 8271 .unwrap(); 8272 8273 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 8274 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 8275 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 8276 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 8277 8278 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 8279 8280 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8281 8282 let r = std::panic::catch_unwind(|| { 8283 // Wait to make sure Windows boots up 8284 assert!(windows_guest.wait_for_boot()); 8285 8286 windows_guest.shutdown(); 8287 }); 8288 8289 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8290 let _ = child.kill(); 8291 let output = child.wait_with_output().unwrap(); 8292 8293 let _ = child_dnsmasq.kill(); 8294 let _ = child_dnsmasq.wait(); 8295 8296 handle_child_output(r, &output); 8297 } 8298 8299 #[test] 8300 fn test_windows_guest_multiple_queues() { 8301 let windows_guest = WindowsGuest::new(); 8302 8303 let mut ovmf_path = dirs::home_dir().unwrap(); 8304 ovmf_path.push("workloads"); 8305 ovmf_path.push(OVMF_NAME); 8306 8307 let mut child = GuestCommand::new(windows_guest.guest()) 8308 .args(["--cpus", "boot=4,kvm_hyperv=on"]) 8309 .args(["--memory", "size=4G"]) 8310 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8311 .args(["--serial", "tty"]) 8312 .args(["--console", "off"]) 8313 .args([ 8314 "--disk", 8315 format!( 8316 "path={},num_queues=4", 8317 windows_guest 8318 .guest() 8319 .disk_config 8320 .disk(DiskType::OperatingSystem) 8321 .unwrap() 8322 ) 8323 .as_str(), 8324 ]) 8325 .args([ 8326 "--net", 8327 format!( 8328 "tap=,mac={},ip={},mask=255.255.255.0,num_queues=8", 8329 windows_guest.guest().network.guest_mac, 8330 windows_guest.guest().network.host_ip 8331 ) 8332 .as_str(), 8333 ]) 8334 .capture_output() 8335 .spawn() 8336 .unwrap(); 8337 8338 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 8339 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 8340 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 8341 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 8342 8343 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 8344 8345 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8346 8347 let r = std::panic::catch_unwind(|| { 8348 // Wait to make sure Windows boots up 8349 assert!(windows_guest.wait_for_boot()); 8350 8351 windows_guest.shutdown(); 8352 }); 8353 8354 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8355 let _ = child.kill(); 8356 let output = child.wait_with_output().unwrap(); 8357 8358 let _ = child_dnsmasq.kill(); 8359 let _ = child_dnsmasq.wait(); 8360 8361 handle_child_output(r, &output); 8362 } 8363 8364 #[test] 8365 #[cfg(not(feature = "mshv"))] 8366 #[ignore = "See #4327"] 8367 fn test_windows_guest_snapshot_restore() { 8368 let windows_guest = WindowsGuest::new(); 8369 8370 let mut ovmf_path = dirs::home_dir().unwrap(); 8371 ovmf_path.push("workloads"); 8372 ovmf_path.push(OVMF_NAME); 8373 8374 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8375 let api_socket_source = format!("{}.1", temp_api_path(&tmp_dir)); 8376 8377 let mut child = GuestCommand::new(windows_guest.guest()) 8378 .args(["--api-socket", &api_socket_source]) 8379 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 8380 .args(["--memory", "size=4G"]) 8381 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8382 .args(["--serial", "tty"]) 8383 .args(["--console", "off"]) 8384 .default_disks() 8385 .default_net() 8386 .capture_output() 8387 .spawn() 8388 .unwrap(); 8389 8390 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 8391 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 8392 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 8393 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 8394 8395 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 8396 8397 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8398 8399 // Wait to make sure Windows boots up 8400 assert!(windows_guest.wait_for_boot()); 8401 8402 let snapshot_dir = temp_snapshot_dir_path(&tmp_dir); 8403 8404 // Pause the VM 8405 assert!(remote_command(&api_socket_source, "pause", None)); 8406 8407 // Take a snapshot from the VM 8408 assert!(remote_command( 8409 &api_socket_source, 8410 "snapshot", 8411 Some(format!("file://{snapshot_dir}").as_str()), 8412 )); 8413 8414 // Wait to make sure the snapshot is completed 8415 thread::sleep(std::time::Duration::new(30, 0)); 8416 8417 let _ = child.kill(); 8418 child.wait().unwrap(); 8419 8420 let api_socket_restored = format!("{}.2", temp_api_path(&tmp_dir)); 8421 8422 // Restore the VM from the snapshot 8423 let mut child = GuestCommand::new(windows_guest.guest()) 8424 .args(["--api-socket", &api_socket_restored]) 8425 .args([ 8426 "--restore", 8427 format!("source_url=file://{snapshot_dir}").as_str(), 8428 ]) 8429 .capture_output() 8430 .spawn() 8431 .unwrap(); 8432 8433 // Wait for the VM to be restored 8434 thread::sleep(std::time::Duration::new(20, 0)); 8435 8436 let r = std::panic::catch_unwind(|| { 8437 // Resume the VM 8438 assert!(remote_command(&api_socket_restored, "resume", None)); 8439 8440 windows_guest.shutdown(); 8441 }); 8442 8443 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8444 let _ = child.kill(); 8445 let output = child.wait_with_output().unwrap(); 8446 8447 let _ = child_dnsmasq.kill(); 8448 let _ = child_dnsmasq.wait(); 8449 8450 handle_child_output(r, &output); 8451 } 8452 8453 #[test] 8454 #[cfg(not(feature = "mshv"))] 8455 #[cfg(not(target_arch = "aarch64"))] 8456 fn test_windows_guest_cpu_hotplug() { 8457 let windows_guest = WindowsGuest::new(); 8458 8459 let mut ovmf_path = dirs::home_dir().unwrap(); 8460 ovmf_path.push("workloads"); 8461 ovmf_path.push(OVMF_NAME); 8462 8463 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8464 let api_socket = temp_api_path(&tmp_dir); 8465 8466 let mut child = GuestCommand::new(windows_guest.guest()) 8467 .args(["--api-socket", &api_socket]) 8468 .args(["--cpus", "boot=2,max=8,kvm_hyperv=on"]) 8469 .args(["--memory", "size=4G"]) 8470 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8471 .args(["--serial", "tty"]) 8472 .args(["--console", "off"]) 8473 .default_disks() 8474 .default_net() 8475 .capture_output() 8476 .spawn() 8477 .unwrap(); 8478 8479 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8480 8481 let r = std::panic::catch_unwind(|| { 8482 // Wait to make sure Windows boots up 8483 assert!(windows_guest.wait_for_boot()); 8484 8485 let vcpu_num = 2; 8486 // Check the initial number of CPUs the guest sees 8487 assert_eq!(windows_guest.cpu_count(), vcpu_num); 8488 // Check the initial number of vcpu threads in the CH process 8489 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 8490 8491 let vcpu_num = 6; 8492 // Hotplug some CPUs 8493 resize_command(&api_socket, Some(vcpu_num), None, None, None); 8494 // Wait to make sure CPUs are added 8495 thread::sleep(std::time::Duration::new(10, 0)); 8496 // Check the guest sees the correct number 8497 assert_eq!(windows_guest.cpu_count(), vcpu_num); 8498 // Check the CH process has the correct number of vcpu threads 8499 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 8500 8501 let vcpu_num = 4; 8502 // Remove some CPUs. Note that Windows doesn't support hot-remove. 8503 resize_command(&api_socket, Some(vcpu_num), None, None, None); 8504 // Wait to make sure CPUs are removed 8505 thread::sleep(std::time::Duration::new(10, 0)); 8506 // Reboot to let Windows catch up 8507 windows_guest.reboot(); 8508 // Wait to make sure Windows completely rebooted 8509 thread::sleep(std::time::Duration::new(60, 0)); 8510 // Check the guest sees the correct number 8511 assert_eq!(windows_guest.cpu_count(), vcpu_num); 8512 // Check the CH process has the correct number of vcpu threads 8513 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 8514 8515 windows_guest.shutdown(); 8516 }); 8517 8518 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8519 let _ = child.kill(); 8520 let output = child.wait_with_output().unwrap(); 8521 8522 let _ = child_dnsmasq.kill(); 8523 let _ = child_dnsmasq.wait(); 8524 8525 handle_child_output(r, &output); 8526 } 8527 8528 #[test] 8529 #[cfg(not(feature = "mshv"))] 8530 #[cfg(not(target_arch = "aarch64"))] 8531 fn test_windows_guest_ram_hotplug() { 8532 let windows_guest = WindowsGuest::new(); 8533 8534 let mut ovmf_path = dirs::home_dir().unwrap(); 8535 ovmf_path.push("workloads"); 8536 ovmf_path.push(OVMF_NAME); 8537 8538 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8539 let api_socket = temp_api_path(&tmp_dir); 8540 8541 let mut child = GuestCommand::new(windows_guest.guest()) 8542 .args(["--api-socket", &api_socket]) 8543 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 8544 .args(["--memory", "size=2G,hotplug_size=5G"]) 8545 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8546 .args(["--serial", "tty"]) 8547 .args(["--console", "off"]) 8548 .default_disks() 8549 .default_net() 8550 .capture_output() 8551 .spawn() 8552 .unwrap(); 8553 8554 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8555 8556 let r = std::panic::catch_unwind(|| { 8557 // Wait to make sure Windows boots up 8558 assert!(windows_guest.wait_for_boot()); 8559 8560 let ram_size = 2 * 1024 * 1024 * 1024; 8561 // Check the initial number of RAM the guest sees 8562 let current_ram_size = windows_guest.ram_size(); 8563 // This size seems to be reserved by the system and thus the 8564 // reported amount differs by this constant value. 8565 let reserved_ram_size = ram_size - current_ram_size; 8566 // Verify that there's not more than 4mb constant diff wasted 8567 // by the reserved ram. 8568 assert!(reserved_ram_size < 4 * 1024 * 1024); 8569 8570 let ram_size = 4 * 1024 * 1024 * 1024; 8571 // Hotplug some RAM 8572 resize_command(&api_socket, None, Some(ram_size), None, None); 8573 // Wait to make sure RAM has been added 8574 thread::sleep(std::time::Duration::new(10, 0)); 8575 // Check the guest sees the correct number 8576 assert_eq!(windows_guest.ram_size(), ram_size - reserved_ram_size); 8577 8578 let ram_size = 3 * 1024 * 1024 * 1024; 8579 // Unplug some RAM. Note that hot-remove most likely won't work. 8580 resize_command(&api_socket, None, Some(ram_size), None, None); 8581 // Wait to make sure RAM has been added 8582 thread::sleep(std::time::Duration::new(10, 0)); 8583 // Reboot to let Windows catch up 8584 windows_guest.reboot(); 8585 // Wait to make sure guest completely rebooted 8586 thread::sleep(std::time::Duration::new(60, 0)); 8587 // Check the guest sees the correct number 8588 assert_eq!(windows_guest.ram_size(), ram_size - reserved_ram_size); 8589 8590 windows_guest.shutdown(); 8591 }); 8592 8593 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8594 let _ = child.kill(); 8595 let output = child.wait_with_output().unwrap(); 8596 8597 let _ = child_dnsmasq.kill(); 8598 let _ = child_dnsmasq.wait(); 8599 8600 handle_child_output(r, &output); 8601 } 8602 8603 #[test] 8604 #[cfg(not(feature = "mshv"))] 8605 fn test_windows_guest_netdev_hotplug() { 8606 let windows_guest = WindowsGuest::new(); 8607 8608 let mut ovmf_path = dirs::home_dir().unwrap(); 8609 ovmf_path.push("workloads"); 8610 ovmf_path.push(OVMF_NAME); 8611 8612 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8613 let api_socket = temp_api_path(&tmp_dir); 8614 8615 let mut child = GuestCommand::new(windows_guest.guest()) 8616 .args(["--api-socket", &api_socket]) 8617 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 8618 .args(["--memory", "size=4G"]) 8619 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8620 .args(["--serial", "tty"]) 8621 .args(["--console", "off"]) 8622 .default_disks() 8623 .default_net() 8624 .capture_output() 8625 .spawn() 8626 .unwrap(); 8627 8628 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8629 8630 let r = std::panic::catch_unwind(|| { 8631 // Wait to make sure Windows boots up 8632 assert!(windows_guest.wait_for_boot()); 8633 8634 // Initially present network device 8635 let netdev_num = 1; 8636 assert_eq!(windows_guest.netdev_count(), netdev_num); 8637 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 8638 8639 // Hotplug network device 8640 let (cmd_success, cmd_output) = remote_command_w_output( 8641 &api_socket, 8642 "add-net", 8643 Some(windows_guest.guest().default_net_string().as_str()), 8644 ); 8645 assert!(cmd_success); 8646 assert!(String::from_utf8_lossy(&cmd_output).contains("\"id\":\"_net2\"")); 8647 thread::sleep(std::time::Duration::new(5, 0)); 8648 // Verify the device is on the system 8649 let netdev_num = 2; 8650 assert_eq!(windows_guest.netdev_count(), netdev_num); 8651 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 8652 8653 // Remove network device 8654 let cmd_success = remote_command(&api_socket, "remove-device", Some("_net2")); 8655 assert!(cmd_success); 8656 thread::sleep(std::time::Duration::new(5, 0)); 8657 // Verify the device has been removed 8658 let netdev_num = 1; 8659 assert_eq!(windows_guest.netdev_count(), netdev_num); 8660 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 8661 8662 windows_guest.shutdown(); 8663 }); 8664 8665 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8666 let _ = child.kill(); 8667 let output = child.wait_with_output().unwrap(); 8668 8669 let _ = child_dnsmasq.kill(); 8670 let _ = child_dnsmasq.wait(); 8671 8672 handle_child_output(r, &output); 8673 } 8674 8675 #[test] 8676 #[ignore = "See #6037"] 8677 #[cfg(not(feature = "mshv"))] 8678 #[cfg(not(target_arch = "aarch64"))] 8679 fn test_windows_guest_disk_hotplug() { 8680 let windows_guest = WindowsGuest::new(); 8681 8682 let mut ovmf_path = dirs::home_dir().unwrap(); 8683 ovmf_path.push("workloads"); 8684 ovmf_path.push(OVMF_NAME); 8685 8686 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8687 let api_socket = temp_api_path(&tmp_dir); 8688 8689 let mut child = GuestCommand::new(windows_guest.guest()) 8690 .args(["--api-socket", &api_socket]) 8691 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 8692 .args(["--memory", "size=4G"]) 8693 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8694 .args(["--serial", "tty"]) 8695 .args(["--console", "off"]) 8696 .default_disks() 8697 .default_net() 8698 .capture_output() 8699 .spawn() 8700 .unwrap(); 8701 8702 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8703 8704 let disk = windows_guest.disk_new(WindowsGuest::FS_FAT, 100); 8705 8706 let r = std::panic::catch_unwind(|| { 8707 // Wait to make sure Windows boots up 8708 assert!(windows_guest.wait_for_boot()); 8709 8710 // Initially present disk device 8711 let disk_num = 1; 8712 assert_eq!(windows_guest.disk_count(), disk_num); 8713 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8714 8715 // Hotplug disk device 8716 let (cmd_success, cmd_output) = remote_command_w_output( 8717 &api_socket, 8718 "add-disk", 8719 Some(format!("path={disk},readonly=off").as_str()), 8720 ); 8721 assert!(cmd_success); 8722 assert!(String::from_utf8_lossy(&cmd_output).contains("\"id\":\"_disk2\"")); 8723 thread::sleep(std::time::Duration::new(5, 0)); 8724 // Online disk device 8725 windows_guest.disks_set_rw(); 8726 windows_guest.disks_online(); 8727 // Verify the device is on the system 8728 let disk_num = 2; 8729 assert_eq!(windows_guest.disk_count(), disk_num); 8730 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8731 8732 let data = "hello"; 8733 let fname = "d:\\world"; 8734 windows_guest.disk_file_put(fname, data); 8735 8736 // Unmount disk device 8737 let cmd_success = remote_command(&api_socket, "remove-device", Some("_disk2")); 8738 assert!(cmd_success); 8739 thread::sleep(std::time::Duration::new(5, 0)); 8740 // Verify the device has been removed 8741 let disk_num = 1; 8742 assert_eq!(windows_guest.disk_count(), disk_num); 8743 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8744 8745 // Remount and check the file exists with the expected contents 8746 let (cmd_success, _cmd_output) = remote_command_w_output( 8747 &api_socket, 8748 "add-disk", 8749 Some(format!("path={disk},readonly=off").as_str()), 8750 ); 8751 assert!(cmd_success); 8752 thread::sleep(std::time::Duration::new(5, 0)); 8753 let out = windows_guest.disk_file_read(fname); 8754 assert_eq!(data, out.trim()); 8755 8756 // Intentionally no unmount, it'll happen at shutdown. 8757 8758 windows_guest.shutdown(); 8759 }); 8760 8761 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8762 let _ = child.kill(); 8763 let output = child.wait_with_output().unwrap(); 8764 8765 let _ = child_dnsmasq.kill(); 8766 let _ = child_dnsmasq.wait(); 8767 8768 handle_child_output(r, &output); 8769 } 8770 8771 #[test] 8772 #[ignore = "See #6037"] 8773 #[cfg(not(feature = "mshv"))] 8774 #[cfg(not(target_arch = "aarch64"))] 8775 fn test_windows_guest_disk_hotplug_multi() { 8776 let windows_guest = WindowsGuest::new(); 8777 8778 let mut ovmf_path = dirs::home_dir().unwrap(); 8779 ovmf_path.push("workloads"); 8780 ovmf_path.push(OVMF_NAME); 8781 8782 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8783 let api_socket = temp_api_path(&tmp_dir); 8784 8785 let mut child = GuestCommand::new(windows_guest.guest()) 8786 .args(["--api-socket", &api_socket]) 8787 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 8788 .args(["--memory", "size=2G"]) 8789 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8790 .args(["--serial", "tty"]) 8791 .args(["--console", "off"]) 8792 .default_disks() 8793 .default_net() 8794 .capture_output() 8795 .spawn() 8796 .unwrap(); 8797 8798 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8799 8800 // Predefined data to used at various test stages 8801 let disk_test_data: [[String; 4]; 2] = [ 8802 [ 8803 "_disk2".to_string(), 8804 windows_guest.disk_new(WindowsGuest::FS_FAT, 123), 8805 "d:\\world".to_string(), 8806 "hello".to_string(), 8807 ], 8808 [ 8809 "_disk3".to_string(), 8810 windows_guest.disk_new(WindowsGuest::FS_NTFS, 333), 8811 "e:\\hello".to_string(), 8812 "world".to_string(), 8813 ], 8814 ]; 8815 8816 let r = std::panic::catch_unwind(|| { 8817 // Wait to make sure Windows boots up 8818 assert!(windows_guest.wait_for_boot()); 8819 8820 // Initially present disk device 8821 let disk_num = 1; 8822 assert_eq!(windows_guest.disk_count(), disk_num); 8823 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8824 8825 for it in &disk_test_data { 8826 let disk_id = it[0].as_str(); 8827 let disk = it[1].as_str(); 8828 // Hotplug disk device 8829 let (cmd_success, cmd_output) = remote_command_w_output( 8830 &api_socket, 8831 "add-disk", 8832 Some(format!("path={disk},readonly=off").as_str()), 8833 ); 8834 assert!(cmd_success); 8835 assert!(String::from_utf8_lossy(&cmd_output) 8836 .contains(format!("\"id\":\"{disk_id}\"").as_str())); 8837 thread::sleep(std::time::Duration::new(5, 0)); 8838 // Online disk devices 8839 windows_guest.disks_set_rw(); 8840 windows_guest.disks_online(); 8841 } 8842 // Verify the devices are on the system 8843 let disk_num = (disk_test_data.len() + 1) as u8; 8844 assert_eq!(windows_guest.disk_count(), disk_num); 8845 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8846 8847 // Put test data 8848 for it in &disk_test_data { 8849 let fname = it[2].as_str(); 8850 let data = it[3].as_str(); 8851 windows_guest.disk_file_put(fname, data); 8852 } 8853 8854 // Unmount disk devices 8855 for it in &disk_test_data { 8856 let disk_id = it[0].as_str(); 8857 let cmd_success = remote_command(&api_socket, "remove-device", Some(disk_id)); 8858 assert!(cmd_success); 8859 thread::sleep(std::time::Duration::new(5, 0)); 8860 } 8861 8862 // Verify the devices have been removed 8863 let disk_num = 1; 8864 assert_eq!(windows_guest.disk_count(), disk_num); 8865 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 8866 8867 // Remount 8868 for it in &disk_test_data { 8869 let disk = it[1].as_str(); 8870 let (cmd_success, _cmd_output) = remote_command_w_output( 8871 &api_socket, 8872 "add-disk", 8873 Some(format!("path={disk},readonly=off").as_str()), 8874 ); 8875 assert!(cmd_success); 8876 thread::sleep(std::time::Duration::new(5, 0)); 8877 } 8878 8879 // Check the files exists with the expected contents 8880 for it in &disk_test_data { 8881 let fname = it[2].as_str(); 8882 let data = it[3].as_str(); 8883 let out = windows_guest.disk_file_read(fname); 8884 assert_eq!(data, out.trim()); 8885 } 8886 8887 // Intentionally no unmount, it'll happen at shutdown. 8888 8889 windows_guest.shutdown(); 8890 }); 8891 8892 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8893 let _ = child.kill(); 8894 let output = child.wait_with_output().unwrap(); 8895 8896 let _ = child_dnsmasq.kill(); 8897 let _ = child_dnsmasq.wait(); 8898 8899 handle_child_output(r, &output); 8900 } 8901 8902 #[test] 8903 #[cfg(not(feature = "mshv"))] 8904 #[cfg(not(target_arch = "aarch64"))] 8905 fn test_windows_guest_netdev_multi() { 8906 let windows_guest = WindowsGuest::new(); 8907 8908 let mut ovmf_path = dirs::home_dir().unwrap(); 8909 ovmf_path.push("workloads"); 8910 ovmf_path.push(OVMF_NAME); 8911 8912 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 8913 let api_socket = temp_api_path(&tmp_dir); 8914 8915 let mut child = GuestCommand::new(windows_guest.guest()) 8916 .args(["--api-socket", &api_socket]) 8917 .args(["--cpus", "boot=2,kvm_hyperv=on"]) 8918 .args(["--memory", "size=4G"]) 8919 .args(["--kernel", ovmf_path.to_str().unwrap()]) 8920 .args(["--serial", "tty"]) 8921 .args(["--console", "off"]) 8922 .default_disks() 8923 // The multi net dev config is borrowed from test_multiple_network_interfaces 8924 .args([ 8925 "--net", 8926 windows_guest.guest().default_net_string().as_str(), 8927 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 8928 "tap=mytap42,mac=fe:1f:9e:e1:60:f2,ip=192.168.4.1,mask=255.255.255.0", 8929 ]) 8930 .capture_output() 8931 .spawn() 8932 .unwrap(); 8933 8934 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 8935 8936 let r = std::panic::catch_unwind(|| { 8937 // Wait to make sure Windows boots up 8938 assert!(windows_guest.wait_for_boot()); 8939 8940 let netdev_num = 3; 8941 assert_eq!(windows_guest.netdev_count(), netdev_num); 8942 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 8943 8944 let tap_count = exec_host_command_output("ip link | grep -c mytap42"); 8945 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 8946 8947 windows_guest.shutdown(); 8948 }); 8949 8950 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 8951 let _ = child.kill(); 8952 let output = child.wait_with_output().unwrap(); 8953 8954 let _ = child_dnsmasq.kill(); 8955 let _ = child_dnsmasq.wait(); 8956 8957 handle_child_output(r, &output); 8958 } 8959 } 8960 8961 #[cfg(target_arch = "x86_64")] 8962 mod sgx { 8963 use crate::*; 8964 8965 #[test] 8966 fn test_sgx() { 8967 let jammy_image = JAMMY_IMAGE_NAME.to_string(); 8968 let jammy = UbuntuDiskConfig::new(jammy_image); 8969 let guest = Guest::new(Box::new(jammy)); 8970 8971 let mut child = GuestCommand::new(&guest) 8972 .args(["--cpus", "boot=1"]) 8973 .args(["--memory", "size=512M"]) 8974 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 8975 .default_disks() 8976 .default_net() 8977 .args(["--sgx-epc", "id=epc0,size=64M"]) 8978 .capture_output() 8979 .spawn() 8980 .unwrap(); 8981 8982 let r = std::panic::catch_unwind(|| { 8983 guest.wait_vm_boot(None).unwrap(); 8984 8985 // Check if SGX is correctly detected in the guest. 8986 guest.check_sgx_support().unwrap(); 8987 8988 // Validate the SGX EPC section is 64MiB. 8989 assert_eq!( 8990 guest 8991 .ssh_command("cpuid -l 0x12 -s 2 | grep 'section size' | cut -d '=' -f 2") 8992 .unwrap() 8993 .trim(), 8994 "0x0000000004000000" 8995 ); 8996 }); 8997 8998 let _ = child.kill(); 8999 let output = child.wait_with_output().unwrap(); 9000 9001 handle_child_output(r, &output); 9002 } 9003 } 9004 9005 #[cfg(target_arch = "x86_64")] 9006 mod vfio { 9007 use crate::*; 9008 const NVIDIA_VFIO_DEVICE: &str = "/sys/bus/pci/devices/0002:00:01.0"; 9009 9010 fn test_nvidia_card_memory_hotplug(hotplug_method: &str) { 9011 let jammy = UbuntuDiskConfig::new(JAMMY_VFIO_IMAGE_NAME.to_string()); 9012 let guest = Guest::new(Box::new(jammy)); 9013 let api_socket = temp_api_path(&guest.tmp_dir); 9014 9015 let mut child = GuestCommand::new(&guest) 9016 .args(["--cpus", "boot=4"]) 9017 .args([ 9018 "--memory", 9019 format!("size=4G,hotplug_size=4G,hotplug_method={hotplug_method}").as_str(), 9020 ]) 9021 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 9022 .args(["--device", format!("path={NVIDIA_VFIO_DEVICE}").as_str()]) 9023 .args(["--api-socket", &api_socket]) 9024 .default_disks() 9025 .default_net() 9026 .capture_output() 9027 .spawn() 9028 .unwrap(); 9029 9030 let r = std::panic::catch_unwind(|| { 9031 guest.wait_vm_boot(None).unwrap(); 9032 9033 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9034 9035 guest.enable_memory_hotplug(); 9036 9037 // Add RAM to the VM 9038 let desired_ram = 6 << 30; 9039 resize_command(&api_socket, None, Some(desired_ram), None, None); 9040 thread::sleep(std::time::Duration::new(30, 0)); 9041 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 9042 9043 // Check the VFIO device works when RAM is increased to 6GiB 9044 guest.check_nvidia_gpu(); 9045 }); 9046 9047 let _ = child.kill(); 9048 let output = child.wait_with_output().unwrap(); 9049 9050 handle_child_output(r, &output); 9051 } 9052 9053 #[test] 9054 fn test_nvidia_card_memory_hotplug_acpi() { 9055 test_nvidia_card_memory_hotplug("acpi") 9056 } 9057 9058 #[test] 9059 fn test_nvidia_card_memory_hotplug_virtio_mem() { 9060 test_nvidia_card_memory_hotplug("virtio-mem") 9061 } 9062 9063 #[test] 9064 fn test_nvidia_card_pci_hotplug() { 9065 let jammy = UbuntuDiskConfig::new(JAMMY_VFIO_IMAGE_NAME.to_string()); 9066 let guest = Guest::new(Box::new(jammy)); 9067 let api_socket = temp_api_path(&guest.tmp_dir); 9068 9069 let mut child = GuestCommand::new(&guest) 9070 .args(["--cpus", "boot=4"]) 9071 .args(["--memory", "size=4G"]) 9072 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 9073 .args(["--api-socket", &api_socket]) 9074 .default_disks() 9075 .default_net() 9076 .capture_output() 9077 .spawn() 9078 .unwrap(); 9079 9080 let r = std::panic::catch_unwind(|| { 9081 guest.wait_vm_boot(None).unwrap(); 9082 9083 // Hotplug the card to the VM 9084 let (cmd_success, cmd_output) = remote_command_w_output( 9085 &api_socket, 9086 "add-device", 9087 Some(format!("id=vfio0,path={NVIDIA_VFIO_DEVICE}").as_str()), 9088 ); 9089 assert!(cmd_success); 9090 assert!(String::from_utf8_lossy(&cmd_output) 9091 .contains("{\"id\":\"vfio0\",\"bdf\":\"0000:00:06.0\"}")); 9092 9093 thread::sleep(std::time::Duration::new(10, 0)); 9094 9095 // Check the VFIO device works after hotplug 9096 guest.check_nvidia_gpu(); 9097 }); 9098 9099 let _ = child.kill(); 9100 let output = child.wait_with_output().unwrap(); 9101 9102 handle_child_output(r, &output); 9103 } 9104 9105 #[test] 9106 fn test_nvidia_card_reboot() { 9107 let jammy = UbuntuDiskConfig::new(JAMMY_VFIO_IMAGE_NAME.to_string()); 9108 let guest = Guest::new(Box::new(jammy)); 9109 let api_socket = temp_api_path(&guest.tmp_dir); 9110 9111 let mut child = GuestCommand::new(&guest) 9112 .args(["--cpus", "boot=4"]) 9113 .args(["--memory", "size=4G"]) 9114 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 9115 .args([ 9116 "--device", 9117 format!("path={NVIDIA_VFIO_DEVICE},iommu=on").as_str(), 9118 ]) 9119 .args(["--api-socket", &api_socket]) 9120 .default_disks() 9121 .default_net() 9122 .capture_output() 9123 .spawn() 9124 .unwrap(); 9125 9126 let r = std::panic::catch_unwind(|| { 9127 guest.wait_vm_boot(None).unwrap(); 9128 9129 // Check the VFIO device works after boot 9130 guest.check_nvidia_gpu(); 9131 9132 guest.reboot_linux(0, None); 9133 9134 // Check the VFIO device works after reboot 9135 guest.check_nvidia_gpu(); 9136 }); 9137 9138 let _ = child.kill(); 9139 let output = child.wait_with_output().unwrap(); 9140 9141 handle_child_output(r, &output); 9142 } 9143 9144 #[test] 9145 fn test_nvidia_card_iommu_address_width() { 9146 let jammy = UbuntuDiskConfig::new(JAMMY_VFIO_IMAGE_NAME.to_string()); 9147 let guest = Guest::new(Box::new(jammy)); 9148 let api_socket = temp_api_path(&guest.tmp_dir); 9149 9150 let mut child = GuestCommand::new(&guest) 9151 .args(["--cpus", "boot=4"]) 9152 .args(["--memory", "size=4G"]) 9153 .args(["--kernel", fw_path(FwType::RustHypervisorFirmware).as_str()]) 9154 .args(["--device", format!("path={NVIDIA_VFIO_DEVICE}").as_str()]) 9155 .args([ 9156 "--platform", 9157 "num_pci_segments=2,iommu_segments=1,iommu_address_width=42", 9158 ]) 9159 .args(["--api-socket", &api_socket]) 9160 .default_disks() 9161 .default_net() 9162 .capture_output() 9163 .spawn() 9164 .unwrap(); 9165 9166 let r = std::panic::catch_unwind(|| { 9167 guest.wait_vm_boot(None).unwrap(); 9168 9169 assert!(guest 9170 .ssh_command("sudo dmesg") 9171 .unwrap() 9172 .contains("input address: 42 bits")); 9173 }); 9174 9175 let _ = child.kill(); 9176 let output = child.wait_with_output().unwrap(); 9177 9178 handle_child_output(r, &output); 9179 } 9180 } 9181 9182 mod live_migration { 9183 use crate::*; 9184 9185 fn start_live_migration( 9186 migration_socket: &str, 9187 src_api_socket: &str, 9188 dest_api_socket: &str, 9189 local: bool, 9190 ) -> bool { 9191 // Start to receive migration from the destination VM 9192 let mut receive_migration = Command::new(clh_command("ch-remote")) 9193 .args([ 9194 &format!("--api-socket={dest_api_socket}"), 9195 "receive-migration", 9196 &format! {"unix:{migration_socket}"}, 9197 ]) 9198 .stderr(Stdio::piped()) 9199 .stdout(Stdio::piped()) 9200 .spawn() 9201 .unwrap(); 9202 // Give it '1s' to make sure the 'migration_socket' file is properly created 9203 thread::sleep(std::time::Duration::new(1, 0)); 9204 // Start to send migration from the source VM 9205 9206 let mut args = [ 9207 format!("--api-socket={}", &src_api_socket), 9208 "send-migration".to_string(), 9209 format! {"unix:{migration_socket}"}, 9210 ] 9211 .to_vec(); 9212 9213 if local { 9214 args.insert(2, "--local".to_string()); 9215 } 9216 9217 let mut send_migration = Command::new(clh_command("ch-remote")) 9218 .args(&args) 9219 .stderr(Stdio::piped()) 9220 .stdout(Stdio::piped()) 9221 .spawn() 9222 .unwrap(); 9223 9224 // The 'send-migration' command should be executed successfully within the given timeout 9225 let send_success = if let Some(status) = send_migration 9226 .wait_timeout(std::time::Duration::from_secs(30)) 9227 .unwrap() 9228 { 9229 status.success() 9230 } else { 9231 false 9232 }; 9233 9234 if !send_success { 9235 let _ = send_migration.kill(); 9236 let output = send_migration.wait_with_output().unwrap(); 9237 eprintln!( 9238 "\n\n==== Start 'send_migration' output ==== \ 9239 \n\n---stdout---\n{}\n\n---stderr---\n{} \ 9240 \n\n==== End 'send_migration' output ====\n\n", 9241 String::from_utf8_lossy(&output.stdout), 9242 String::from_utf8_lossy(&output.stderr) 9243 ); 9244 } 9245 9246 // The 'receive-migration' command should be executed successfully within the given timeout 9247 let receive_success = if let Some(status) = receive_migration 9248 .wait_timeout(std::time::Duration::from_secs(30)) 9249 .unwrap() 9250 { 9251 status.success() 9252 } else { 9253 false 9254 }; 9255 9256 if !receive_success { 9257 let _ = receive_migration.kill(); 9258 let output = receive_migration.wait_with_output().unwrap(); 9259 eprintln!( 9260 "\n\n==== Start 'receive_migration' output ==== \ 9261 \n\n---stdout---\n{}\n\n---stderr---\n{} \ 9262 \n\n==== End 'receive_migration' output ====\n\n", 9263 String::from_utf8_lossy(&output.stdout), 9264 String::from_utf8_lossy(&output.stderr) 9265 ); 9266 } 9267 9268 send_success && receive_success 9269 } 9270 9271 fn print_and_panic(src_vm: Child, dest_vm: Child, ovs_vm: Option<Child>, message: &str) -> ! { 9272 let mut src_vm = src_vm; 9273 let mut dest_vm = dest_vm; 9274 9275 let _ = src_vm.kill(); 9276 let src_output = src_vm.wait_with_output().unwrap(); 9277 eprintln!( 9278 "\n\n==== Start 'source_vm' stdout ====\n\n{}\n\n==== End 'source_vm' stdout ====", 9279 String::from_utf8_lossy(&src_output.stdout) 9280 ); 9281 eprintln!( 9282 "\n\n==== Start 'source_vm' stderr ====\n\n{}\n\n==== End 'source_vm' stderr ====", 9283 String::from_utf8_lossy(&src_output.stderr) 9284 ); 9285 let _ = dest_vm.kill(); 9286 let dest_output = dest_vm.wait_with_output().unwrap(); 9287 eprintln!( 9288 "\n\n==== Start 'destination_vm' stdout ====\n\n{}\n\n==== End 'destination_vm' stdout ====", 9289 String::from_utf8_lossy(&dest_output.stdout) 9290 ); 9291 eprintln!( 9292 "\n\n==== Start 'destination_vm' stderr ====\n\n{}\n\n==== End 'destination_vm' stderr ====", 9293 String::from_utf8_lossy(&dest_output.stderr) 9294 ); 9295 9296 if let Some(ovs_vm) = ovs_vm { 9297 let mut ovs_vm = ovs_vm; 9298 let _ = ovs_vm.kill(); 9299 let ovs_output = ovs_vm.wait_with_output().unwrap(); 9300 eprintln!( 9301 "\n\n==== Start 'ovs_vm' stdout ====\n\n{}\n\n==== End 'ovs_vm' stdout ====", 9302 String::from_utf8_lossy(&ovs_output.stdout) 9303 ); 9304 eprintln!( 9305 "\n\n==== Start 'ovs_vm' stderr ====\n\n{}\n\n==== End 'ovs_vm' stderr ====", 9306 String::from_utf8_lossy(&ovs_output.stderr) 9307 ); 9308 9309 cleanup_ovs_dpdk(); 9310 } 9311 9312 panic!("Test failed: {message}") 9313 } 9314 9315 // This test exercises the local live-migration between two Cloud Hypervisor VMs on the 9316 // same host. It ensures the following behaviors: 9317 // 1. The source VM is up and functional (including various virtio-devices are working properly); 9318 // 2. The 'send-migration' and 'receive-migration' command finished successfully; 9319 // 3. The source VM terminated gracefully after live migration; 9320 // 4. The destination VM is functional (including various virtio-devices are working properly) after 9321 // live migration; 9322 // Note: This test does not use vsock as we can't create two identical vsock on the same host. 9323 fn _test_live_migration(upgrade_test: bool, local: bool) { 9324 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9325 let guest = Guest::new(Box::new(focal)); 9326 let kernel_path = direct_kernel_boot_path(); 9327 let console_text = String::from("On a branch floating down river a cricket, singing."); 9328 let net_id = "net123"; 9329 let net_params = format!( 9330 "id={},tap=,mac={},ip={},mask=255.255.255.0", 9331 net_id, guest.network.guest_mac, guest.network.host_ip 9332 ); 9333 9334 let memory_param: &[&str] = if local { 9335 &["--memory", "size=4G,shared=on"] 9336 } else { 9337 &["--memory", "size=4G"] 9338 }; 9339 9340 let boot_vcpus = 2; 9341 let max_vcpus = 4; 9342 9343 let pmem_temp_file = TempFile::new().unwrap(); 9344 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 9345 std::process::Command::new("mkfs.ext4") 9346 .arg(pmem_temp_file.as_path()) 9347 .output() 9348 .expect("Expect creating disk image to succeed"); 9349 let pmem_path = String::from("/dev/pmem0"); 9350 9351 // Start the source VM 9352 let src_vm_path = if !upgrade_test { 9353 clh_command("cloud-hypervisor") 9354 } else { 9355 cloud_hypervisor_release_path() 9356 }; 9357 let src_api_socket = temp_api_path(&guest.tmp_dir); 9358 let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); 9359 src_vm_cmd 9360 .args([ 9361 "--cpus", 9362 format!("boot={boot_vcpus},max={max_vcpus}").as_str(), 9363 ]) 9364 .args(memory_param) 9365 .args(["--kernel", kernel_path.to_str().unwrap()]) 9366 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 9367 .default_disks() 9368 .args(["--net", net_params.as_str()]) 9369 .args(["--api-socket", &src_api_socket]) 9370 .args([ 9371 "--pmem", 9372 format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(), 9373 ]); 9374 let mut src_child = src_vm_cmd.capture_output().spawn().unwrap(); 9375 9376 // Start the destination VM 9377 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 9378 dest_api_socket.push_str(".dest"); 9379 let mut dest_child = GuestCommand::new(&guest) 9380 .args(["--api-socket", &dest_api_socket]) 9381 .capture_output() 9382 .spawn() 9383 .unwrap(); 9384 9385 let r = std::panic::catch_unwind(|| { 9386 guest.wait_vm_boot(None).unwrap(); 9387 9388 // Make sure the source VM is functional 9389 // Check the number of vCPUs 9390 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9391 9392 // Check the guest RAM 9393 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9394 9395 // Check the guest virtio-devices, e.g. block, rng, console, and net 9396 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9397 9398 // x86_64: Following what's done in the `test_snapshot_restore`, we need 9399 // to make sure that removing and adding back the virtio-net device does 9400 // not break the live-migration support for virtio-pci. 9401 #[cfg(target_arch = "x86_64")] 9402 { 9403 assert!(remote_command( 9404 &src_api_socket, 9405 "remove-device", 9406 Some(net_id), 9407 )); 9408 thread::sleep(std::time::Duration::new(10, 0)); 9409 9410 // Plug the virtio-net device again 9411 assert!(remote_command( 9412 &src_api_socket, 9413 "add-net", 9414 Some(net_params.as_str()), 9415 )); 9416 thread::sleep(std::time::Duration::new(10, 0)); 9417 } 9418 9419 // Start the live-migration 9420 let migration_socket = String::from( 9421 guest 9422 .tmp_dir 9423 .as_path() 9424 .join("live-migration.sock") 9425 .to_str() 9426 .unwrap(), 9427 ); 9428 9429 assert!( 9430 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 9431 "Unsuccessful command: 'send-migration' or 'receive-migration'." 9432 ); 9433 }); 9434 9435 // Check and report any errors occurred during the live-migration 9436 if r.is_err() { 9437 print_and_panic( 9438 src_child, 9439 dest_child, 9440 None, 9441 "Error occurred during live-migration", 9442 ); 9443 } 9444 9445 // Check the source vm has been terminated successful (give it '3s' to settle) 9446 thread::sleep(std::time::Duration::new(3, 0)); 9447 if !src_child.try_wait().unwrap().is_some_and(|s| s.success()) { 9448 print_and_panic( 9449 src_child, 9450 dest_child, 9451 None, 9452 "source VM was not terminated successfully.", 9453 ); 9454 }; 9455 9456 // Post live-migration check to make sure the destination VM is functional 9457 let r = std::panic::catch_unwind(|| { 9458 // Perform same checks to validate VM has been properly migrated 9459 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9460 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9461 9462 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9463 }); 9464 9465 // Clean-up the destination VM and make sure it terminated correctly 9466 let _ = dest_child.kill(); 9467 let dest_output = dest_child.wait_with_output().unwrap(); 9468 handle_child_output(r, &dest_output); 9469 9470 // Check the destination VM has the expected 'console_text' from its output 9471 let r = std::panic::catch_unwind(|| { 9472 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 9473 }); 9474 handle_child_output(r, &dest_output); 9475 } 9476 9477 fn _test_live_migration_balloon(upgrade_test: bool, local: bool) { 9478 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9479 let guest = Guest::new(Box::new(focal)); 9480 let kernel_path = direct_kernel_boot_path(); 9481 let console_text = String::from("On a branch floating down river a cricket, singing."); 9482 let net_id = "net123"; 9483 let net_params = format!( 9484 "id={},tap=,mac={},ip={},mask=255.255.255.0", 9485 net_id, guest.network.guest_mac, guest.network.host_ip 9486 ); 9487 9488 let memory_param: &[&str] = if local { 9489 &[ 9490 "--memory", 9491 "size=4G,hotplug_method=virtio-mem,hotplug_size=8G,shared=on", 9492 "--balloon", 9493 "size=0", 9494 ] 9495 } else { 9496 &[ 9497 "--memory", 9498 "size=4G,hotplug_method=virtio-mem,hotplug_size=8G", 9499 "--balloon", 9500 "size=0", 9501 ] 9502 }; 9503 9504 let boot_vcpus = 2; 9505 let max_vcpus = 4; 9506 9507 let pmem_temp_file = TempFile::new().unwrap(); 9508 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 9509 std::process::Command::new("mkfs.ext4") 9510 .arg(pmem_temp_file.as_path()) 9511 .output() 9512 .expect("Expect creating disk image to succeed"); 9513 let pmem_path = String::from("/dev/pmem0"); 9514 9515 // Start the source VM 9516 let src_vm_path = if !upgrade_test { 9517 clh_command("cloud-hypervisor") 9518 } else { 9519 cloud_hypervisor_release_path() 9520 }; 9521 let src_api_socket = temp_api_path(&guest.tmp_dir); 9522 let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); 9523 src_vm_cmd 9524 .args([ 9525 "--cpus", 9526 format!("boot={boot_vcpus},max={max_vcpus}").as_str(), 9527 ]) 9528 .args(memory_param) 9529 .args(["--kernel", kernel_path.to_str().unwrap()]) 9530 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 9531 .default_disks() 9532 .args(["--net", net_params.as_str()]) 9533 .args(["--api-socket", &src_api_socket]) 9534 .args([ 9535 "--pmem", 9536 format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(), 9537 ]); 9538 let mut src_child = src_vm_cmd.capture_output().spawn().unwrap(); 9539 9540 // Start the destination VM 9541 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 9542 dest_api_socket.push_str(".dest"); 9543 let mut dest_child = GuestCommand::new(&guest) 9544 .args(["--api-socket", &dest_api_socket]) 9545 .capture_output() 9546 .spawn() 9547 .unwrap(); 9548 9549 let r = std::panic::catch_unwind(|| { 9550 guest.wait_vm_boot(None).unwrap(); 9551 9552 // Make sure the source VM is functional 9553 // Check the number of vCPUs 9554 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9555 9556 // Check the guest RAM 9557 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9558 // Increase the guest RAM 9559 resize_command(&src_api_socket, None, Some(6 << 30), None, None); 9560 thread::sleep(std::time::Duration::new(5, 0)); 9561 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 9562 // Use balloon to remove RAM from the VM 9563 resize_command(&src_api_socket, None, None, Some(1 << 30), None); 9564 thread::sleep(std::time::Duration::new(5, 0)); 9565 let total_memory = guest.get_total_memory().unwrap_or_default(); 9566 assert!(total_memory > 4_800_000); 9567 assert!(total_memory < 5_760_000); 9568 9569 // Check the guest virtio-devices, e.g. block, rng, console, and net 9570 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9571 9572 // x86_64: Following what's done in the `test_snapshot_restore`, we need 9573 // to make sure that removing and adding back the virtio-net device does 9574 // not break the live-migration support for virtio-pci. 9575 #[cfg(target_arch = "x86_64")] 9576 { 9577 assert!(remote_command( 9578 &src_api_socket, 9579 "remove-device", 9580 Some(net_id), 9581 )); 9582 thread::sleep(std::time::Duration::new(10, 0)); 9583 9584 // Plug the virtio-net device again 9585 assert!(remote_command( 9586 &src_api_socket, 9587 "add-net", 9588 Some(net_params.as_str()), 9589 )); 9590 thread::sleep(std::time::Duration::new(10, 0)); 9591 } 9592 9593 // Start the live-migration 9594 let migration_socket = String::from( 9595 guest 9596 .tmp_dir 9597 .as_path() 9598 .join("live-migration.sock") 9599 .to_str() 9600 .unwrap(), 9601 ); 9602 9603 assert!( 9604 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 9605 "Unsuccessful command: 'send-migration' or 'receive-migration'." 9606 ); 9607 }); 9608 9609 // Check and report any errors occurred during the live-migration 9610 if r.is_err() { 9611 print_and_panic( 9612 src_child, 9613 dest_child, 9614 None, 9615 "Error occurred during live-migration", 9616 ); 9617 } 9618 9619 // Check the source vm has been terminated successful (give it '3s' to settle) 9620 thread::sleep(std::time::Duration::new(3, 0)); 9621 if !src_child.try_wait().unwrap().is_some_and(|s| s.success()) { 9622 print_and_panic( 9623 src_child, 9624 dest_child, 9625 None, 9626 "source VM was not terminated successfully.", 9627 ); 9628 }; 9629 9630 // Post live-migration check to make sure the destination VM is functional 9631 let r = std::panic::catch_unwind(|| { 9632 // Perform same checks to validate VM has been properly migrated 9633 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9634 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9635 9636 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9637 9638 // Perform checks on guest RAM using balloon 9639 let total_memory = guest.get_total_memory().unwrap_or_default(); 9640 assert!(total_memory > 4_800_000); 9641 assert!(total_memory < 5_760_000); 9642 // Deflate balloon to restore entire RAM to the VM 9643 resize_command(&dest_api_socket, None, None, Some(0), None); 9644 thread::sleep(std::time::Duration::new(5, 0)); 9645 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 9646 // Decrease guest RAM with virtio-mem 9647 resize_command(&dest_api_socket, None, Some(5 << 30), None, None); 9648 thread::sleep(std::time::Duration::new(5, 0)); 9649 let total_memory = guest.get_total_memory().unwrap_or_default(); 9650 assert!(total_memory > 4_800_000); 9651 assert!(total_memory < 5_760_000); 9652 }); 9653 9654 // Clean-up the destination VM and make sure it terminated correctly 9655 let _ = dest_child.kill(); 9656 let dest_output = dest_child.wait_with_output().unwrap(); 9657 handle_child_output(r, &dest_output); 9658 9659 // Check the destination VM has the expected 'console_text' from its output 9660 let r = std::panic::catch_unwind(|| { 9661 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 9662 }); 9663 handle_child_output(r, &dest_output); 9664 } 9665 9666 fn _test_live_migration_numa(upgrade_test: bool, local: bool) { 9667 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9668 let guest = Guest::new(Box::new(focal)); 9669 let kernel_path = direct_kernel_boot_path(); 9670 let console_text = String::from("On a branch floating down river a cricket, singing."); 9671 let net_id = "net123"; 9672 let net_params = format!( 9673 "id={},tap=,mac={},ip={},mask=255.255.255.0", 9674 net_id, guest.network.guest_mac, guest.network.host_ip 9675 ); 9676 9677 let memory_param: &[&str] = if local { 9678 &[ 9679 "--memory", 9680 "size=0,hotplug_method=virtio-mem,shared=on", 9681 "--memory-zone", 9682 "id=mem0,size=1G,hotplug_size=4G,shared=on", 9683 "id=mem1,size=1G,hotplug_size=4G,shared=on", 9684 "id=mem2,size=2G,hotplug_size=4G,shared=on", 9685 "--numa", 9686 "guest_numa_id=0,cpus=[0-2,9],distances=[1@15,2@20],memory_zones=mem0", 9687 "guest_numa_id=1,cpus=[3-4,6-8],distances=[0@20,2@25],memory_zones=mem1", 9688 "guest_numa_id=2,cpus=[5,10-11],distances=[0@25,1@30],memory_zones=mem2", 9689 ] 9690 } else { 9691 &[ 9692 "--memory", 9693 "size=0,hotplug_method=virtio-mem", 9694 "--memory-zone", 9695 "id=mem0,size=1G,hotplug_size=4G", 9696 "id=mem1,size=1G,hotplug_size=4G", 9697 "id=mem2,size=2G,hotplug_size=4G", 9698 "--numa", 9699 "guest_numa_id=0,cpus=[0-2,9],distances=[1@15,2@20],memory_zones=mem0", 9700 "guest_numa_id=1,cpus=[3-4,6-8],distances=[0@20,2@25],memory_zones=mem1", 9701 "guest_numa_id=2,cpus=[5,10-11],distances=[0@25,1@30],memory_zones=mem2", 9702 ] 9703 }; 9704 9705 let boot_vcpus = 6; 9706 let max_vcpus = 12; 9707 9708 let pmem_temp_file = TempFile::new().unwrap(); 9709 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 9710 std::process::Command::new("mkfs.ext4") 9711 .arg(pmem_temp_file.as_path()) 9712 .output() 9713 .expect("Expect creating disk image to succeed"); 9714 let pmem_path = String::from("/dev/pmem0"); 9715 9716 // Start the source VM 9717 let src_vm_path = if !upgrade_test { 9718 clh_command("cloud-hypervisor") 9719 } else { 9720 cloud_hypervisor_release_path() 9721 }; 9722 let src_api_socket = temp_api_path(&guest.tmp_dir); 9723 let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); 9724 src_vm_cmd 9725 .args([ 9726 "--cpus", 9727 format!("boot={boot_vcpus},max={max_vcpus}").as_str(), 9728 ]) 9729 .args(memory_param) 9730 .args(["--kernel", kernel_path.to_str().unwrap()]) 9731 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 9732 .default_disks() 9733 .args(["--net", net_params.as_str()]) 9734 .args(["--api-socket", &src_api_socket]) 9735 .args([ 9736 "--pmem", 9737 format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(), 9738 ]); 9739 let mut src_child = src_vm_cmd.capture_output().spawn().unwrap(); 9740 9741 // Start the destination VM 9742 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 9743 dest_api_socket.push_str(".dest"); 9744 let mut dest_child = GuestCommand::new(&guest) 9745 .args(["--api-socket", &dest_api_socket]) 9746 .capture_output() 9747 .spawn() 9748 .unwrap(); 9749 9750 let r = std::panic::catch_unwind(|| { 9751 guest.wait_vm_boot(None).unwrap(); 9752 9753 // Make sure the source VM is functional 9754 // Check the number of vCPUs 9755 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9756 9757 // Check the guest RAM 9758 assert!(guest.get_total_memory().unwrap_or_default() > 2_880_000); 9759 9760 // Check the guest virtio-devices, e.g. block, rng, console, and net 9761 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9762 9763 // Check the NUMA parameters are applied correctly and resize 9764 // each zone to test the case where we migrate a VM with the 9765 // virtio-mem regions being used. 9766 { 9767 guest.check_numa_common( 9768 Some(&[960_000, 960_000, 1_920_000]), 9769 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 9770 Some(&["10 15 20", "20 10 25", "25 30 10"]), 9771 ); 9772 9773 // AArch64 currently does not support hotplug, and therefore we only 9774 // test hotplug-related function on x86_64 here. 9775 #[cfg(target_arch = "x86_64")] 9776 { 9777 guest.enable_memory_hotplug(); 9778 9779 // Resize every memory zone and check each associated NUMA node 9780 // has been assigned the right amount of memory. 9781 resize_zone_command(&src_api_socket, "mem0", "2G"); 9782 resize_zone_command(&src_api_socket, "mem1", "2G"); 9783 resize_zone_command(&src_api_socket, "mem2", "3G"); 9784 thread::sleep(std::time::Duration::new(5, 0)); 9785 9786 guest.check_numa_common(Some(&[1_920_000, 1_920_000, 1_920_000]), None, None); 9787 } 9788 } 9789 9790 // x86_64: Following what's done in the `test_snapshot_restore`, we need 9791 // to make sure that removing and adding back the virtio-net device does 9792 // not break the live-migration support for virtio-pci. 9793 #[cfg(target_arch = "x86_64")] 9794 { 9795 assert!(remote_command( 9796 &src_api_socket, 9797 "remove-device", 9798 Some(net_id), 9799 )); 9800 thread::sleep(std::time::Duration::new(10, 0)); 9801 9802 // Plug the virtio-net device again 9803 assert!(remote_command( 9804 &src_api_socket, 9805 "add-net", 9806 Some(net_params.as_str()), 9807 )); 9808 thread::sleep(std::time::Duration::new(10, 0)); 9809 } 9810 9811 // Start the live-migration 9812 let migration_socket = String::from( 9813 guest 9814 .tmp_dir 9815 .as_path() 9816 .join("live-migration.sock") 9817 .to_str() 9818 .unwrap(), 9819 ); 9820 9821 assert!( 9822 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 9823 "Unsuccessful command: 'send-migration' or 'receive-migration'." 9824 ); 9825 }); 9826 9827 // Check and report any errors occurred during the live-migration 9828 if r.is_err() { 9829 print_and_panic( 9830 src_child, 9831 dest_child, 9832 None, 9833 "Error occurred during live-migration", 9834 ); 9835 } 9836 9837 // Check the source vm has been terminated successful (give it '3s' to settle) 9838 thread::sleep(std::time::Duration::new(3, 0)); 9839 if !src_child.try_wait().unwrap().is_some_and(|s| s.success()) { 9840 print_and_panic( 9841 src_child, 9842 dest_child, 9843 None, 9844 "source VM was not terminated successfully.", 9845 ); 9846 }; 9847 9848 // Post live-migration check to make sure the destination VM is functional 9849 let r = std::panic::catch_unwind(|| { 9850 // Perform same checks to validate VM has been properly migrated 9851 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9852 #[cfg(target_arch = "x86_64")] 9853 assert!(guest.get_total_memory().unwrap_or_default() > 6_720_000); 9854 #[cfg(target_arch = "aarch64")] 9855 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9856 9857 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9858 9859 // Perform NUMA related checks 9860 { 9861 #[cfg(target_arch = "aarch64")] 9862 { 9863 guest.check_numa_common( 9864 Some(&[960_000, 960_000, 1_920_000]), 9865 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 9866 Some(&["10 15 20", "20 10 25", "25 30 10"]), 9867 ); 9868 } 9869 9870 // AArch64 currently does not support hotplug, and therefore we only 9871 // test hotplug-related function on x86_64 here. 9872 #[cfg(target_arch = "x86_64")] 9873 { 9874 guest.check_numa_common( 9875 Some(&[1_920_000, 1_920_000, 2_880_000]), 9876 Some(&[vec![0, 1, 2], vec![3, 4], vec![5]]), 9877 Some(&["10 15 20", "20 10 25", "25 30 10"]), 9878 ); 9879 9880 guest.enable_memory_hotplug(); 9881 9882 // Resize every memory zone and check each associated NUMA node 9883 // has been assigned the right amount of memory. 9884 resize_zone_command(&dest_api_socket, "mem0", "4G"); 9885 resize_zone_command(&dest_api_socket, "mem1", "4G"); 9886 resize_zone_command(&dest_api_socket, "mem2", "4G"); 9887 // Resize to the maximum amount of CPUs and check each NUMA 9888 // node has been assigned the right CPUs set. 9889 resize_command(&dest_api_socket, Some(max_vcpus), None, None, None); 9890 thread::sleep(std::time::Duration::new(5, 0)); 9891 9892 guest.check_numa_common( 9893 Some(&[3_840_000, 3_840_000, 3_840_000]), 9894 Some(&[vec![0, 1, 2, 9], vec![3, 4, 6, 7, 8], vec![5, 10, 11]]), 9895 None, 9896 ); 9897 } 9898 } 9899 }); 9900 9901 // Clean-up the destination VM and make sure it terminated correctly 9902 let _ = dest_child.kill(); 9903 let dest_output = dest_child.wait_with_output().unwrap(); 9904 handle_child_output(r, &dest_output); 9905 9906 // Check the destination VM has the expected 'console_text' from its output 9907 let r = std::panic::catch_unwind(|| { 9908 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 9909 }); 9910 handle_child_output(r, &dest_output); 9911 } 9912 9913 fn _test_live_migration_watchdog(upgrade_test: bool, local: bool) { 9914 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 9915 let guest = Guest::new(Box::new(focal)); 9916 let kernel_path = direct_kernel_boot_path(); 9917 let console_text = String::from("On a branch floating down river a cricket, singing."); 9918 let net_id = "net123"; 9919 let net_params = format!( 9920 "id={},tap=,mac={},ip={},mask=255.255.255.0", 9921 net_id, guest.network.guest_mac, guest.network.host_ip 9922 ); 9923 9924 let memory_param: &[&str] = if local { 9925 &["--memory", "size=4G,shared=on"] 9926 } else { 9927 &["--memory", "size=4G"] 9928 }; 9929 9930 let boot_vcpus = 2; 9931 let max_vcpus = 4; 9932 9933 let pmem_temp_file = TempFile::new().unwrap(); 9934 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 9935 std::process::Command::new("mkfs.ext4") 9936 .arg(pmem_temp_file.as_path()) 9937 .output() 9938 .expect("Expect creating disk image to succeed"); 9939 let pmem_path = String::from("/dev/pmem0"); 9940 9941 // Start the source VM 9942 let src_vm_path = if !upgrade_test { 9943 clh_command("cloud-hypervisor") 9944 } else { 9945 cloud_hypervisor_release_path() 9946 }; 9947 let src_api_socket = temp_api_path(&guest.tmp_dir); 9948 let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); 9949 src_vm_cmd 9950 .args([ 9951 "--cpus", 9952 format!("boot={boot_vcpus},max={max_vcpus}").as_str(), 9953 ]) 9954 .args(memory_param) 9955 .args(["--kernel", kernel_path.to_str().unwrap()]) 9956 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 9957 .default_disks() 9958 .args(["--net", net_params.as_str()]) 9959 .args(["--api-socket", &src_api_socket]) 9960 .args([ 9961 "--pmem", 9962 format!("file={}", pmem_temp_file.as_path().to_str().unwrap(),).as_str(), 9963 ]) 9964 .args(["--watchdog"]); 9965 let mut src_child = src_vm_cmd.capture_output().spawn().unwrap(); 9966 9967 // Start the destination VM 9968 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 9969 dest_api_socket.push_str(".dest"); 9970 let mut dest_child = GuestCommand::new(&guest) 9971 .args(["--api-socket", &dest_api_socket]) 9972 .capture_output() 9973 .spawn() 9974 .unwrap(); 9975 9976 let r = std::panic::catch_unwind(|| { 9977 guest.wait_vm_boot(None).unwrap(); 9978 9979 // Make sure the source VM is functional 9980 // Check the number of vCPUs 9981 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 9982 // Check the guest RAM 9983 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 9984 // Check the guest virtio-devices, e.g. block, rng, console, and net 9985 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 9986 // x86_64: Following what's done in the `test_snapshot_restore`, we need 9987 // to make sure that removing and adding back the virtio-net device does 9988 // not break the live-migration support for virtio-pci. 9989 #[cfg(target_arch = "x86_64")] 9990 { 9991 assert!(remote_command( 9992 &src_api_socket, 9993 "remove-device", 9994 Some(net_id), 9995 )); 9996 thread::sleep(std::time::Duration::new(10, 0)); 9997 9998 // Plug the virtio-net device again 9999 assert!(remote_command( 10000 &src_api_socket, 10001 "add-net", 10002 Some(net_params.as_str()), 10003 )); 10004 thread::sleep(std::time::Duration::new(10, 0)); 10005 } 10006 10007 // Enable watchdog and ensure its functional 10008 let expected_reboot_count = 1; 10009 // Enable the watchdog with a 15s timeout 10010 enable_guest_watchdog(&guest, 15); 10011 10012 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 10013 assert_eq!( 10014 guest 10015 .ssh_command("sudo journalctl | grep -c -- \"Watchdog started\"") 10016 .unwrap() 10017 .trim() 10018 .parse::<u32>() 10019 .unwrap_or_default(), 10020 1 10021 ); 10022 // Allow some normal time to elapse to check we don't get spurious reboots 10023 thread::sleep(std::time::Duration::new(40, 0)); 10024 // Check no reboot 10025 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 10026 10027 // Start the live-migration 10028 let migration_socket = String::from( 10029 guest 10030 .tmp_dir 10031 .as_path() 10032 .join("live-migration.sock") 10033 .to_str() 10034 .unwrap(), 10035 ); 10036 10037 assert!( 10038 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 10039 "Unsuccessful command: 'send-migration' or 'receive-migration'." 10040 ); 10041 }); 10042 10043 // Check and report any errors occurred during the live-migration 10044 if r.is_err() { 10045 print_and_panic( 10046 src_child, 10047 dest_child, 10048 None, 10049 "Error occurred during live-migration", 10050 ); 10051 } 10052 10053 // Check the source vm has been terminated successful (give it '3s' to settle) 10054 thread::sleep(std::time::Duration::new(3, 0)); 10055 if !src_child.try_wait().unwrap().is_some_and(|s| s.success()) { 10056 print_and_panic( 10057 src_child, 10058 dest_child, 10059 None, 10060 "source VM was not terminated successfully.", 10061 ); 10062 }; 10063 10064 // Post live-migration check to make sure the destination VM is functional 10065 let r = std::panic::catch_unwind(|| { 10066 // Perform same checks to validate VM has been properly migrated 10067 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 10068 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 10069 10070 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 10071 10072 // Perform checks on watchdog 10073 let mut expected_reboot_count = 1; 10074 10075 // Allow some normal time to elapse to check we don't get spurious reboots 10076 thread::sleep(std::time::Duration::new(40, 0)); 10077 // Check no reboot 10078 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 10079 10080 // Trigger a panic (sync first). We need to do this inside a screen with a delay so the SSH command returns. 10081 guest.ssh_command("screen -dmS reboot sh -c \"sleep 5; echo s | tee /proc/sysrq-trigger; echo c | sudo tee /proc/sysrq-trigger\"").unwrap(); 10082 // Allow some time for the watchdog to trigger (max 30s) and reboot to happen 10083 guest.wait_vm_boot(Some(50)).unwrap(); 10084 // Check a reboot is triggered by the watchdog 10085 expected_reboot_count += 1; 10086 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 10087 10088 #[cfg(target_arch = "x86_64")] 10089 { 10090 // Now pause the VM and remain offline for 30s 10091 assert!(remote_command(&dest_api_socket, "pause", None)); 10092 thread::sleep(std::time::Duration::new(30, 0)); 10093 assert!(remote_command(&dest_api_socket, "resume", None)); 10094 10095 // Check no reboot 10096 assert_eq!(get_reboot_count(&guest), expected_reboot_count); 10097 } 10098 }); 10099 10100 // Clean-up the destination VM and make sure it terminated correctly 10101 let _ = dest_child.kill(); 10102 let dest_output = dest_child.wait_with_output().unwrap(); 10103 handle_child_output(r, &dest_output); 10104 10105 // Check the destination VM has the expected 'console_text' from its output 10106 let r = std::panic::catch_unwind(|| { 10107 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 10108 }); 10109 handle_child_output(r, &dest_output); 10110 } 10111 10112 fn _test_live_migration_ovs_dpdk(upgrade_test: bool, local: bool) { 10113 let ovs_focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 10114 let ovs_guest = Guest::new(Box::new(ovs_focal)); 10115 10116 let migration_focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 10117 let migration_guest = Guest::new(Box::new(migration_focal)); 10118 let src_api_socket = temp_api_path(&migration_guest.tmp_dir); 10119 10120 // Start two VMs that are connected through ovs-dpdk and one of the VMs is the source VM for live-migration 10121 let (mut ovs_child, mut src_child) = 10122 setup_ovs_dpdk_guests(&ovs_guest, &migration_guest, &src_api_socket, upgrade_test); 10123 10124 // Start the destination VM 10125 let mut dest_api_socket = temp_api_path(&migration_guest.tmp_dir); 10126 dest_api_socket.push_str(".dest"); 10127 let mut dest_child = GuestCommand::new(&migration_guest) 10128 .args(["--api-socket", &dest_api_socket]) 10129 .capture_output() 10130 .spawn() 10131 .unwrap(); 10132 10133 let r = std::panic::catch_unwind(|| { 10134 // Give it '1s' to make sure the 'dest_api_socket' file is properly created 10135 thread::sleep(std::time::Duration::new(1, 0)); 10136 10137 // Start the live-migration 10138 let migration_socket = String::from( 10139 migration_guest 10140 .tmp_dir 10141 .as_path() 10142 .join("live-migration.sock") 10143 .to_str() 10144 .unwrap(), 10145 ); 10146 10147 assert!( 10148 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, local), 10149 "Unsuccessful command: 'send-migration' or 'receive-migration'." 10150 ); 10151 }); 10152 10153 // Check and report any errors occurred during the live-migration 10154 if r.is_err() { 10155 print_and_panic( 10156 src_child, 10157 dest_child, 10158 Some(ovs_child), 10159 "Error occurred during live-migration", 10160 ); 10161 } 10162 10163 // Check the source vm has been terminated successful (give it '3s' to settle) 10164 thread::sleep(std::time::Duration::new(3, 0)); 10165 if !src_child.try_wait().unwrap().is_some_and(|s| s.success()) { 10166 print_and_panic( 10167 src_child, 10168 dest_child, 10169 Some(ovs_child), 10170 "source VM was not terminated successfully.", 10171 ); 10172 }; 10173 10174 // Post live-migration check to make sure the destination VM is functional 10175 let r = std::panic::catch_unwind(|| { 10176 // Perform same checks to validate VM has been properly migrated 10177 // Spawn a new netcat listener in the OVS VM 10178 let guest_ip = ovs_guest.network.guest_ip.clone(); 10179 thread::spawn(move || { 10180 ssh_command_ip( 10181 "nc -l 12345", 10182 &guest_ip, 10183 DEFAULT_SSH_RETRIES, 10184 DEFAULT_SSH_TIMEOUT, 10185 ) 10186 .unwrap(); 10187 }); 10188 10189 // Wait for the server to be listening 10190 thread::sleep(std::time::Duration::new(5, 0)); 10191 10192 // And check the connection is still functional after live-migration 10193 migration_guest 10194 .ssh_command("nc -vz 172.100.0.1 12345") 10195 .unwrap(); 10196 }); 10197 10198 // Clean-up the destination VM and OVS VM, and make sure they terminated correctly 10199 let _ = dest_child.kill(); 10200 let _ = ovs_child.kill(); 10201 let dest_output = dest_child.wait_with_output().unwrap(); 10202 let ovs_output = ovs_child.wait_with_output().unwrap(); 10203 10204 cleanup_ovs_dpdk(); 10205 10206 handle_child_output(r, &dest_output); 10207 handle_child_output(Ok(()), &ovs_output); 10208 } 10209 10210 // This test exercises the local live-migration between two Cloud Hypervisor VMs on the 10211 // same host with Landlock enabled on both VMs. The test validates the following: 10212 // 1. The source VM is up and functional 10213 // 2. Ensure Landlock is enabled on source VM by hotplugging a disk. As the path for this 10214 // disk is not known to the source VM this step will fail. 10215 // 3. The 'send-migration' and 'receive-migration' command finished successfully; 10216 // 4. The source VM terminated gracefully after live migration; 10217 // 5. The destination VM is functional after live migration; 10218 // 6. Ensure Landlock is enabled on destination VM by hotplugging a disk. As the path for 10219 // this disk is not known to the destination VM this step will fail. 10220 fn _test_live_migration_with_landlock() { 10221 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 10222 let guest = Guest::new(Box::new(focal)); 10223 let kernel_path = direct_kernel_boot_path(); 10224 let net_id = "net123"; 10225 let net_params = format!( 10226 "id={},tap=,mac={},ip={},mask=255.255.255.0", 10227 net_id, guest.network.guest_mac, guest.network.host_ip 10228 ); 10229 10230 let boot_vcpus = 2; 10231 let max_vcpus = 4; 10232 10233 let mut blk_file_path = dirs::home_dir().unwrap(); 10234 blk_file_path.push("workloads"); 10235 blk_file_path.push("blk.img"); 10236 10237 let src_api_socket = temp_api_path(&guest.tmp_dir); 10238 let mut src_child = GuestCommand::new(&guest) 10239 .args([ 10240 "--cpus", 10241 format!("boot={boot_vcpus},max={max_vcpus}").as_str(), 10242 ]) 10243 .args(["--memory", "size=4G,shared=on"]) 10244 .args(["--kernel", kernel_path.to_str().unwrap()]) 10245 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 10246 .default_disks() 10247 .args(["--api-socket", &src_api_socket]) 10248 .args(["--landlock"]) 10249 .args(["--net", net_params.as_str()]) 10250 .args([ 10251 "--landlock-rules", 10252 format!("path={:?},access=rw", guest.tmp_dir.as_path()).as_str(), 10253 ]) 10254 .capture_output() 10255 .spawn() 10256 .unwrap(); 10257 10258 // Start the destination VM 10259 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 10260 dest_api_socket.push_str(".dest"); 10261 let mut dest_child = GuestCommand::new(&guest) 10262 .args(["--api-socket", &dest_api_socket]) 10263 .capture_output() 10264 .spawn() 10265 .unwrap(); 10266 10267 let r = std::panic::catch_unwind(|| { 10268 guest.wait_vm_boot(None).unwrap(); 10269 10270 // Make sure the source VM is functaionl 10271 // Check the number of vCPUs 10272 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 10273 10274 // Check the guest RAM 10275 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 10276 10277 // Check Landlock is enabled by hot-plugging a disk. 10278 assert!(!remote_command( 10279 &src_api_socket, 10280 "add-disk", 10281 Some(format!("path={},id=test0", blk_file_path.to_str().unwrap()).as_str()), 10282 )); 10283 10284 // Start the live-migration 10285 let migration_socket = String::from( 10286 guest 10287 .tmp_dir 10288 .as_path() 10289 .join("live-migration.sock") 10290 .to_str() 10291 .unwrap(), 10292 ); 10293 10294 assert!( 10295 start_live_migration(&migration_socket, &src_api_socket, &dest_api_socket, true), 10296 "Unsuccessful command: 'send-migration' or 'receive-migration'." 10297 ); 10298 }); 10299 10300 // Check and report any errors occurred during the live-migration 10301 if r.is_err() { 10302 print_and_panic( 10303 src_child, 10304 dest_child, 10305 None, 10306 "Error occurred during live-migration", 10307 ); 10308 } 10309 10310 // Check the source vm has been terminated successful (give it '3s' to settle) 10311 thread::sleep(std::time::Duration::new(3, 0)); 10312 if !src_child.try_wait().unwrap().is_some_and(|s| s.success()) { 10313 print_and_panic( 10314 src_child, 10315 dest_child, 10316 None, 10317 "source VM was not terminated successfully.", 10318 ); 10319 }; 10320 10321 // Post live-migration check to make sure the destination VM is funcational 10322 let r = std::panic::catch_unwind(|| { 10323 // Perform same checks to validate VM has been properly migrated 10324 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 10325 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 10326 }); 10327 10328 // Check Landlock is enabled on destination VM by hot-plugging a disk. 10329 assert!(!remote_command( 10330 &dest_api_socket, 10331 "add-disk", 10332 Some(format!("path={},id=test0", blk_file_path.to_str().unwrap()).as_str()), 10333 )); 10334 10335 // Clean-up the destination VM and make sure it terminated correctly 10336 let _ = dest_child.kill(); 10337 let dest_output = dest_child.wait_with_output().unwrap(); 10338 handle_child_output(r, &dest_output); 10339 } 10340 10341 // Function to get an available port 10342 fn get_available_port() -> u16 { 10343 TcpListener::bind("127.0.0.1:0") 10344 .expect("Failed to bind to address") 10345 .local_addr() 10346 .unwrap() 10347 .port() 10348 } 10349 10350 fn start_live_migration_tcp(src_api_socket: &str, dest_api_socket: &str) -> bool { 10351 // Get an available TCP port 10352 let migration_port = get_available_port(); 10353 let host_ip = "127.0.0.1"; 10354 10355 // Start the 'receive-migration' command on the destination 10356 let mut receive_migration = Command::new(clh_command("ch-remote")) 10357 .args([ 10358 &format!("--api-socket={}", dest_api_socket), 10359 "receive-migration", 10360 &format!("tcp:0.0.0.0:{}", migration_port), 10361 ]) 10362 .stdin(Stdio::null()) 10363 .stderr(Stdio::piped()) 10364 .stdout(Stdio::piped()) 10365 .spawn() 10366 .unwrap(); 10367 10368 // Give the destination some time to start listening 10369 thread::sleep(Duration::from_secs(1)); 10370 10371 // Start the 'send-migration' command on the source 10372 let mut send_migration = Command::new(clh_command("ch-remote")) 10373 .args([ 10374 &format!("--api-socket={}", src_api_socket), 10375 "send-migration", 10376 &format!("tcp:{}:{}", host_ip, migration_port), 10377 ]) 10378 .stdin(Stdio::null()) 10379 .stderr(Stdio::piped()) 10380 .stdout(Stdio::piped()) 10381 .spawn() 10382 .unwrap(); 10383 10384 // Check if the 'send-migration' command executed successfully 10385 let send_success = if let Some(status) = send_migration 10386 .wait_timeout(Duration::from_secs(60)) 10387 .unwrap() 10388 { 10389 status.success() 10390 } else { 10391 false 10392 }; 10393 10394 if !send_success { 10395 let _ = send_migration.kill(); 10396 let output = send_migration.wait_with_output().unwrap(); 10397 eprintln!( 10398 "\n\n==== Start 'send_migration' output ====\n\n---stdout---\n{}\n\n---stderr---\n{}\n\n==== End 'send_migration' output ====\n\n", 10399 String::from_utf8_lossy(&output.stdout), 10400 String::from_utf8_lossy(&output.stderr) 10401 ); 10402 } 10403 10404 // Check if the 'receive-migration' command executed successfully 10405 let receive_success = if let Some(status) = receive_migration 10406 .wait_timeout(Duration::from_secs(60)) 10407 .unwrap() 10408 { 10409 status.success() 10410 } else { 10411 false 10412 }; 10413 10414 if !receive_success { 10415 let _ = receive_migration.kill(); 10416 let output = receive_migration.wait_with_output().unwrap(); 10417 eprintln!( 10418 "\n\n==== Start 'receive_migration' output ====\n\n---stdout---\n{}\n\n---stderr---\n{}\n\n==== End 'receive_migration' output ====\n\n", 10419 String::from_utf8_lossy(&output.stdout), 10420 String::from_utf8_lossy(&output.stderr) 10421 ); 10422 } 10423 10424 send_success && receive_success 10425 } 10426 10427 fn _test_live_migration_tcp() { 10428 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 10429 let guest = Guest::new(Box::new(focal)); 10430 let kernel_path = direct_kernel_boot_path(); 10431 let console_text = String::from("On a branch floating down river a cricket, singing."); 10432 let net_id = "net123"; 10433 let net_params = format!( 10434 "id={},tap=,mac={},ip={},mask=255.255.255.0", 10435 net_id, guest.network.guest_mac, guest.network.host_ip 10436 ); 10437 let memory_param: &[&str] = &["--memory", "size=4G,shared=on"]; 10438 let boot_vcpus = 2; 10439 let max_vcpus = 4; 10440 let pmem_temp_file = TempFile::new().unwrap(); 10441 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 10442 std::process::Command::new("mkfs.ext4") 10443 .arg(pmem_temp_file.as_path()) 10444 .output() 10445 .expect("Expect creating disk image to succeed"); 10446 let pmem_path = String::from("/dev/pmem0"); 10447 10448 // Start the source VM 10449 let src_vm_path = clh_command("cloud-hypervisor"); 10450 let src_api_socket = temp_api_path(&guest.tmp_dir); 10451 let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); 10452 src_vm_cmd 10453 .args([ 10454 "--cpus", 10455 format!("boot={},max={}", boot_vcpus, max_vcpus).as_str(), 10456 ]) 10457 .args(memory_param) 10458 .args(["--kernel", kernel_path.to_str().unwrap()]) 10459 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 10460 .default_disks() 10461 .args(["--net", net_params.as_str()]) 10462 .args(["--api-socket", &src_api_socket]) 10463 .args([ 10464 "--pmem", 10465 format!( 10466 "file={},discard_writes=on", 10467 pmem_temp_file.as_path().to_str().unwrap(), 10468 ) 10469 .as_str(), 10470 ]) 10471 .capture_output(); 10472 let mut src_child = src_vm_cmd.spawn().unwrap(); 10473 10474 // Start the destination VM 10475 let mut dest_api_socket = temp_api_path(&guest.tmp_dir); 10476 dest_api_socket.push_str(".dest"); 10477 let mut dest_child = GuestCommand::new(&guest) 10478 .args(["--api-socket", &dest_api_socket]) 10479 .capture_output() 10480 .spawn() 10481 .unwrap(); 10482 10483 let r = std::panic::catch_unwind(|| { 10484 guest.wait_vm_boot(None).unwrap(); 10485 // Ensure the source VM is running normally 10486 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 10487 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 10488 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 10489 10490 // On x86_64 architecture, remove and re-add the virtio-net device 10491 #[cfg(target_arch = "x86_64")] 10492 { 10493 assert!(remote_command( 10494 &src_api_socket, 10495 "remove-device", 10496 Some(net_id), 10497 )); 10498 thread::sleep(Duration::new(10, 0)); 10499 // Re-add the virtio-net device 10500 assert!(remote_command( 10501 &src_api_socket, 10502 "add-net", 10503 Some(net_params.as_str()), 10504 )); 10505 thread::sleep(Duration::new(10, 0)); 10506 } 10507 // Start TCP live migration 10508 assert!( 10509 start_live_migration_tcp(&src_api_socket, &dest_api_socket), 10510 "Unsuccessful command: 'send-migration' or 'receive-migration'." 10511 ); 10512 }); 10513 10514 // Check and report any errors that occurred during live migration 10515 if r.is_err() { 10516 print_and_panic( 10517 src_child, 10518 dest_child, 10519 None, 10520 "Error occurred during live-migration", 10521 ); 10522 } 10523 10524 // Check the source vm has been terminated successful (give it '3s' to settle) 10525 thread::sleep(std::time::Duration::new(3, 0)); 10526 if !src_child.try_wait().unwrap().is_some_and(|s| s.success()) { 10527 print_and_panic( 10528 src_child, 10529 dest_child, 10530 None, 10531 "Source VM was not terminated successfully.", 10532 ); 10533 }; 10534 10535 // After live migration, ensure the destination VM is running normally 10536 let r = std::panic::catch_unwind(|| { 10537 // Perform the same checks to ensure the VM has migrated correctly 10538 assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); 10539 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 10540 guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); 10541 }); 10542 10543 // Clean up the destination VM and ensure it terminates properly 10544 let _ = dest_child.kill(); 10545 let dest_output = dest_child.wait_with_output().unwrap(); 10546 handle_child_output(r, &dest_output); 10547 10548 // Check if the expected `console_text` is present in the destination VM's output 10549 let r = std::panic::catch_unwind(|| { 10550 assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); 10551 }); 10552 handle_child_output(r, &dest_output); 10553 } 10554 10555 mod live_migration_parallel { 10556 use super::*; 10557 #[test] 10558 fn test_live_migration_basic() { 10559 _test_live_migration(false, false) 10560 } 10561 10562 #[test] 10563 fn test_live_migration_local() { 10564 _test_live_migration(false, true) 10565 } 10566 10567 #[test] 10568 fn test_live_migration_tcp() { 10569 _test_live_migration_tcp(); 10570 } 10571 10572 #[test] 10573 fn test_live_migration_watchdog() { 10574 _test_live_migration_watchdog(false, false) 10575 } 10576 10577 #[test] 10578 fn test_live_migration_watchdog_local() { 10579 _test_live_migration_watchdog(false, true) 10580 } 10581 10582 #[test] 10583 fn test_live_upgrade_basic() { 10584 _test_live_migration(true, false) 10585 } 10586 10587 #[test] 10588 fn test_live_upgrade_local() { 10589 _test_live_migration(true, true) 10590 } 10591 10592 #[test] 10593 fn test_live_upgrade_watchdog() { 10594 _test_live_migration_watchdog(true, false) 10595 } 10596 10597 #[test] 10598 fn test_live_upgrade_watchdog_local() { 10599 _test_live_migration_watchdog(true, true) 10600 } 10601 #[test] 10602 #[cfg(target_arch = "x86_64")] 10603 fn test_live_migration_with_landlock() { 10604 _test_live_migration_with_landlock() 10605 } 10606 } 10607 10608 mod live_migration_sequential { 10609 use super::*; 10610 10611 // NUMA & balloon live migration tests are large so run sequentially 10612 10613 #[test] 10614 fn test_live_migration_balloon() { 10615 _test_live_migration_balloon(false, false) 10616 } 10617 10618 #[test] 10619 fn test_live_migration_balloon_local() { 10620 _test_live_migration_balloon(false, true) 10621 } 10622 10623 #[test] 10624 fn test_live_upgrade_balloon() { 10625 _test_live_migration_balloon(true, false) 10626 } 10627 10628 #[test] 10629 fn test_live_upgrade_balloon_local() { 10630 _test_live_migration_balloon(true, true) 10631 } 10632 10633 #[test] 10634 #[cfg(not(feature = "mshv"))] 10635 fn test_live_migration_numa() { 10636 _test_live_migration_numa(false, false) 10637 } 10638 10639 #[test] 10640 #[cfg(not(feature = "mshv"))] 10641 fn test_live_migration_numa_local() { 10642 _test_live_migration_numa(false, true) 10643 } 10644 10645 #[test] 10646 #[cfg(not(feature = "mshv"))] 10647 fn test_live_upgrade_numa() { 10648 _test_live_migration_numa(true, false) 10649 } 10650 10651 #[test] 10652 #[cfg(not(feature = "mshv"))] 10653 fn test_live_upgrade_numa_local() { 10654 _test_live_migration_numa(true, true) 10655 } 10656 10657 // Require to run ovs-dpdk tests sequentially because they rely on the same ovs-dpdk setup 10658 #[test] 10659 #[ignore = "See #5532"] 10660 #[cfg(target_arch = "x86_64")] 10661 #[cfg(not(feature = "mshv"))] 10662 fn test_live_migration_ovs_dpdk() { 10663 _test_live_migration_ovs_dpdk(false, false); 10664 } 10665 10666 #[test] 10667 #[cfg(target_arch = "x86_64")] 10668 #[cfg(not(feature = "mshv"))] 10669 fn test_live_migration_ovs_dpdk_local() { 10670 _test_live_migration_ovs_dpdk(false, true); 10671 } 10672 10673 #[test] 10674 #[ignore = "See #5532"] 10675 #[cfg(target_arch = "x86_64")] 10676 #[cfg(not(feature = "mshv"))] 10677 fn test_live_upgrade_ovs_dpdk() { 10678 _test_live_migration_ovs_dpdk(true, false); 10679 } 10680 10681 #[test] 10682 #[ignore = "See #5532"] 10683 #[cfg(target_arch = "x86_64")] 10684 #[cfg(not(feature = "mshv"))] 10685 fn test_live_upgrade_ovs_dpdk_local() { 10686 _test_live_migration_ovs_dpdk(true, true); 10687 } 10688 } 10689 } 10690 10691 #[cfg(target_arch = "aarch64")] 10692 mod aarch64_acpi { 10693 use crate::*; 10694 10695 #[test] 10696 fn test_simple_launch_acpi() { 10697 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 10698 10699 vec![Box::new(focal)].drain(..).for_each(|disk_config| { 10700 let guest = Guest::new(disk_config); 10701 10702 let mut child = GuestCommand::new(&guest) 10703 .args(["--cpus", "boot=1"]) 10704 .args(["--memory", "size=512M"]) 10705 .args(["--kernel", edk2_path().to_str().unwrap()]) 10706 .default_disks() 10707 .default_net() 10708 .args(["--serial", "tty", "--console", "off"]) 10709 .capture_output() 10710 .spawn() 10711 .unwrap(); 10712 10713 let r = std::panic::catch_unwind(|| { 10714 guest.wait_vm_boot(Some(120)).unwrap(); 10715 10716 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 10717 assert!(guest.get_total_memory().unwrap_or_default() > 400_000); 10718 assert_eq!(guest.get_pci_bridge_class().unwrap_or_default(), "0x060000"); 10719 }); 10720 10721 let _ = child.kill(); 10722 let output = child.wait_with_output().unwrap(); 10723 10724 handle_child_output(r, &output); 10725 }); 10726 } 10727 10728 #[test] 10729 fn test_guest_numa_nodes_acpi() { 10730 _test_guest_numa_nodes(true); 10731 } 10732 10733 #[test] 10734 fn test_cpu_topology_421_acpi() { 10735 test_cpu_topology(4, 2, 1, true); 10736 } 10737 10738 #[test] 10739 fn test_cpu_topology_142_acpi() { 10740 test_cpu_topology(1, 4, 2, true); 10741 } 10742 10743 #[test] 10744 fn test_cpu_topology_262_acpi() { 10745 test_cpu_topology(2, 6, 2, true); 10746 } 10747 10748 #[test] 10749 fn test_power_button_acpi() { 10750 _test_power_button(true); 10751 } 10752 10753 #[test] 10754 fn test_virtio_iommu() { 10755 _test_virtio_iommu(true) 10756 } 10757 } 10758 10759 mod rate_limiter { 10760 use super::*; 10761 10762 // Check if the 'measured' rate is within the expected 'difference' (in percentage) 10763 // compared to given 'limit' rate. 10764 fn check_rate_limit(measured: f64, limit: f64, difference: f64) -> bool { 10765 let upper_limit = limit * (1_f64 + difference); 10766 let lower_limit = limit * (1_f64 - difference); 10767 10768 if measured > lower_limit && measured < upper_limit { 10769 return true; 10770 } 10771 10772 eprintln!( 10773 "\n\n==== Start 'check_rate_limit' failed ==== \ 10774 \n\nmeasured={measured}, , lower_limit={lower_limit}, upper_limit={upper_limit} \ 10775 \n\n==== End 'check_rate_limit' failed ====\n\n" 10776 ); 10777 10778 false 10779 } 10780 10781 fn _test_rate_limiter_net(rx: bool) { 10782 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 10783 let guest = Guest::new(Box::new(focal)); 10784 10785 let test_timeout = 10; 10786 let num_queues = 2; 10787 let queue_size = 256; 10788 let bw_size = 10485760_u64; // bytes 10789 let bw_refill_time = 100; // ms 10790 let limit_bps = (bw_size * 8 * 1000) as f64 / bw_refill_time as f64; 10791 10792 let net_params = format!( 10793 "tap=,mac={},ip={},mask=255.255.255.0,num_queues={},queue_size={},bw_size={},bw_refill_time={}", 10794 guest.network.guest_mac, 10795 guest.network.host_ip, 10796 num_queues, 10797 queue_size, 10798 bw_size, 10799 bw_refill_time, 10800 ); 10801 10802 let mut child = GuestCommand::new(&guest) 10803 .args(["--cpus", &format!("boot={}", num_queues / 2)]) 10804 .args(["--memory", "size=4G"]) 10805 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 10806 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 10807 .default_disks() 10808 .args(["--net", net_params.as_str()]) 10809 .capture_output() 10810 .spawn() 10811 .unwrap(); 10812 10813 let r = std::panic::catch_unwind(|| { 10814 guest.wait_vm_boot(None).unwrap(); 10815 let measured_bps = 10816 measure_virtio_net_throughput(test_timeout, num_queues / 2, &guest, rx, true) 10817 .unwrap(); 10818 assert!(check_rate_limit(measured_bps, limit_bps, 0.1)); 10819 }); 10820 10821 let _ = child.kill(); 10822 let output = child.wait_with_output().unwrap(); 10823 handle_child_output(r, &output); 10824 } 10825 10826 #[test] 10827 fn test_rate_limiter_net_rx() { 10828 _test_rate_limiter_net(true); 10829 } 10830 10831 #[test] 10832 fn test_rate_limiter_net_tx() { 10833 _test_rate_limiter_net(false); 10834 } 10835 10836 fn _test_rate_limiter_block(bandwidth: bool, num_queues: u32) { 10837 let test_timeout = 10; 10838 let fio_ops = FioOps::RandRW; 10839 10840 let bw_size = if bandwidth { 10841 10485760_u64 // bytes 10842 } else { 10843 100_u64 // I/O 10844 }; 10845 let bw_refill_time = 100; // ms 10846 let limit_rate = (bw_size * 1000) as f64 / bw_refill_time as f64; 10847 10848 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 10849 let guest = Guest::new(Box::new(focal)); 10850 let api_socket = temp_api_path(&guest.tmp_dir); 10851 let test_img_dir = TempDir::new_with_prefix("/var/tmp/ch").unwrap(); 10852 let blk_rate_limiter_test_img = 10853 String::from(test_img_dir.as_path().join("blk.img").to_str().unwrap()); 10854 10855 // Create the test block image 10856 assert!(exec_host_command_output(&format!( 10857 "dd if=/dev/zero of={blk_rate_limiter_test_img} bs=1M count=1024" 10858 )) 10859 .status 10860 .success()); 10861 10862 let test_blk_params = if bandwidth { 10863 format!( 10864 "path={blk_rate_limiter_test_img},num_queues={num_queues},bw_size={bw_size},bw_refill_time={bw_refill_time}" 10865 ) 10866 } else { 10867 format!( 10868 "path={blk_rate_limiter_test_img},num_queues={num_queues},ops_size={bw_size},ops_refill_time={bw_refill_time}" 10869 ) 10870 }; 10871 10872 let mut child = GuestCommand::new(&guest) 10873 .args(["--cpus", &format!("boot={num_queues}")]) 10874 .args(["--memory", "size=4G"]) 10875 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 10876 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 10877 .args([ 10878 "--disk", 10879 format!( 10880 "path={}", 10881 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 10882 ) 10883 .as_str(), 10884 format!( 10885 "path={}", 10886 guest.disk_config.disk(DiskType::CloudInit).unwrap() 10887 ) 10888 .as_str(), 10889 test_blk_params.as_str(), 10890 ]) 10891 .default_net() 10892 .args(["--api-socket", &api_socket]) 10893 .capture_output() 10894 .spawn() 10895 .unwrap(); 10896 10897 let r = std::panic::catch_unwind(|| { 10898 guest.wait_vm_boot(None).unwrap(); 10899 10900 let fio_command = format!( 10901 "sudo fio --filename=/dev/vdc --name=test --output-format=json \ 10902 --direct=1 --bs=4k --ioengine=io_uring --iodepth=64 \ 10903 --rw={fio_ops} --runtime={test_timeout} --numjobs={num_queues}" 10904 ); 10905 let output = guest.ssh_command(&fio_command).unwrap(); 10906 10907 // Parse fio output 10908 let measured_rate = if bandwidth { 10909 parse_fio_output(&output, &fio_ops, num_queues).unwrap() 10910 } else { 10911 parse_fio_output_iops(&output, &fio_ops, num_queues).unwrap() 10912 }; 10913 assert!(check_rate_limit(measured_rate, limit_rate, 0.1)); 10914 }); 10915 10916 let _ = child.kill(); 10917 let output = child.wait_with_output().unwrap(); 10918 handle_child_output(r, &output); 10919 } 10920 10921 fn _test_rate_limiter_group_block(bandwidth: bool, num_queues: u32, num_disks: u32) { 10922 let test_timeout = 10; 10923 let fio_ops = FioOps::RandRW; 10924 10925 let bw_size = if bandwidth { 10926 10485760_u64 // bytes 10927 } else { 10928 100_u64 // I/O 10929 }; 10930 let bw_refill_time = 100; // ms 10931 let limit_rate = (bw_size * 1000) as f64 / bw_refill_time as f64; 10932 10933 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 10934 let guest = Guest::new(Box::new(focal)); 10935 let api_socket = temp_api_path(&guest.tmp_dir); 10936 let test_img_dir = TempDir::new_with_prefix("/var/tmp/ch").unwrap(); 10937 10938 let rate_limit_group_arg = if bandwidth { 10939 format!("id=group0,bw_size={bw_size},bw_refill_time={bw_refill_time}") 10940 } else { 10941 format!("id=group0,ops_size={bw_size},ops_refill_time={bw_refill_time}") 10942 }; 10943 10944 let mut disk_args = vec![ 10945 "--disk".to_string(), 10946 format!( 10947 "path={}", 10948 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 10949 ), 10950 format!( 10951 "path={}", 10952 guest.disk_config.disk(DiskType::CloudInit).unwrap() 10953 ), 10954 ]; 10955 10956 for i in 0..num_disks { 10957 let test_img_path = String::from( 10958 test_img_dir 10959 .as_path() 10960 .join(format!("blk{}.img", i)) 10961 .to_str() 10962 .unwrap(), 10963 ); 10964 10965 assert!(exec_host_command_output(&format!( 10966 "dd if=/dev/zero of={test_img_path} bs=1M count=1024" 10967 )) 10968 .status 10969 .success()); 10970 10971 disk_args.push(format!( 10972 "path={test_img_path},num_queues={num_queues},rate_limit_group=group0" 10973 )); 10974 } 10975 10976 let mut child = GuestCommand::new(&guest) 10977 .args(["--cpus", &format!("boot={}", num_queues * num_disks)]) 10978 .args(["--memory", "size=4G"]) 10979 .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 10980 .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 10981 .args(["--rate-limit-group", &rate_limit_group_arg]) 10982 .args(disk_args) 10983 .default_net() 10984 .args(["--api-socket", &api_socket]) 10985 .capture_output() 10986 .spawn() 10987 .unwrap(); 10988 10989 let r = std::panic::catch_unwind(|| { 10990 guest.wait_vm_boot(None).unwrap(); 10991 10992 let mut fio_command = format!( 10993 "sudo fio --name=global --output-format=json \ 10994 --direct=1 --bs=4k --ioengine=io_uring --iodepth=64 \ 10995 --rw={fio_ops} --runtime={test_timeout} --numjobs={num_queues}" 10996 ); 10997 10998 // Generate additional argument for each disk: 10999 // --name=job0 --filename=/dev/vdc \ 11000 // --name=job1 --filename=/dev/vdd \ 11001 // --name=job2 --filename=/dev/vde \ 11002 // ... 11003 for i in 0..num_disks { 11004 let c: char = 'c'; 11005 let arg = format!( 11006 " --name=job{i} --filename=/dev/vd{}", 11007 char::from_u32((c as u32) + i).unwrap() 11008 ); 11009 fio_command += &arg; 11010 } 11011 let output = guest.ssh_command(&fio_command).unwrap(); 11012 11013 // Parse fio output 11014 let measured_rate = if bandwidth { 11015 parse_fio_output(&output, &fio_ops, num_queues * num_disks).unwrap() 11016 } else { 11017 parse_fio_output_iops(&output, &fio_ops, num_queues * num_disks).unwrap() 11018 }; 11019 assert!(check_rate_limit(measured_rate, limit_rate, 0.2)); 11020 }); 11021 11022 let _ = child.kill(); 11023 let output = child.wait_with_output().unwrap(); 11024 handle_child_output(r, &output); 11025 } 11026 11027 #[test] 11028 fn test_rate_limiter_block_bandwidth() { 11029 _test_rate_limiter_block(true, 1); 11030 _test_rate_limiter_block(true, 2) 11031 } 11032 11033 #[test] 11034 fn test_rate_limiter_group_block_bandwidth() { 11035 _test_rate_limiter_group_block(true, 1, 1); 11036 _test_rate_limiter_group_block(true, 2, 1); 11037 _test_rate_limiter_group_block(true, 1, 2); 11038 _test_rate_limiter_group_block(true, 2, 2); 11039 } 11040 11041 #[test] 11042 fn test_rate_limiter_block_iops() { 11043 _test_rate_limiter_block(false, 1); 11044 _test_rate_limiter_block(false, 2); 11045 } 11046 11047 #[test] 11048 fn test_rate_limiter_group_block_iops() { 11049 _test_rate_limiter_group_block(false, 1, 1); 11050 _test_rate_limiter_group_block(false, 2, 1); 11051 _test_rate_limiter_group_block(false, 1, 2); 11052 _test_rate_limiter_group_block(false, 2, 2); 11053 } 11054 } 11055