1 // Copyright © 2020 Intel Corporation 2 // 3 // SPDX-License-Identifier: Apache-2.0 4 // 5 6 #[cfg(test)] 7 #[cfg(feature = "integration_tests")] 8 #[macro_use] 9 extern crate lazy_static; 10 11 #[cfg(test)] 12 #[cfg(feature = "integration_tests")] 13 extern crate test_infra; 14 15 #[cfg(test)] 16 #[cfg(feature = "integration_tests")] 17 mod tests { 18 use net_util::MacAddr; 19 use std::collections::HashMap; 20 use std::env; 21 use std::ffi::OsStr; 22 use std::fs; 23 use std::io; 24 use std::io::BufRead; 25 use std::io::Read; 26 use std::io::Write; 27 use std::os::unix::io::AsRawFd; 28 use std::path::PathBuf; 29 use std::process::{Child, Command, Stdio}; 30 use std::string::String; 31 use std::sync::mpsc::Receiver; 32 use std::sync::{mpsc, Mutex}; 33 use std::thread; 34 use test_infra::*; 35 use vmm_sys_util::{tempdir::TempDir, tempfile::TempFile}; 36 #[cfg_attr(target_arch = "aarch64", allow(unused_imports))] 37 use wait_timeout::ChildExt; 38 39 lazy_static! { 40 static ref NEXT_VM_ID: Mutex<u8> = Mutex::new(1); 41 } 42 43 struct Guest { 44 tmp_dir: TempDir, 45 disk_config: Box<dyn DiskConfig>, 46 fw_path: String, 47 network: GuestNetworkConfig, 48 } 49 50 // Safe to implement as we know we have no interior mutability 51 impl std::panic::RefUnwindSafe for Guest {} 52 53 #[cfg(target_arch = "x86_64")] 54 const BIONIC_IMAGE_NAME: &str = "bionic-server-cloudimg-amd64.raw"; 55 #[cfg(target_arch = "x86_64")] 56 const FOCAL_IMAGE_NAME: &str = "focal-server-cloudimg-amd64-custom-20210609-0.raw"; 57 #[cfg(target_arch = "x86_64")] 58 const FOCAL_SGX_IMAGE_NAME: &str = "focal-server-cloudimg-amd64-sgx.raw"; 59 #[cfg(target_arch = "x86_64")] 60 const HIRSUTE_NVIDIA_IMAGE_NAME: &str = "hirsute-server-cloudimg-amd64-nvidia.raw"; 61 #[cfg(target_arch = "aarch64")] 62 const BIONIC_IMAGE_NAME: &str = "bionic-server-cloudimg-arm64.raw"; 63 #[cfg(target_arch = "aarch64")] 64 const FOCAL_IMAGE_NAME: &str = "focal-server-cloudimg-arm64-custom.raw"; 65 #[cfg(target_arch = "aarch64")] 66 const FOCAL_IMAGE_NAME_QCOW2: &str = "focal-server-cloudimg-arm64-custom.qcow2"; 67 #[cfg(target_arch = "x86_64")] 68 const FOCAL_IMAGE_NAME_QCOW2: &str = "focal-server-cloudimg-amd64-custom-20210609-0.qcow2"; 69 #[cfg(target_arch = "aarch64")] 70 const FOCAL_IMAGE_NAME_VHD: &str = "focal-server-cloudimg-arm64-custom.vhd"; 71 #[cfg(target_arch = "x86_64")] 72 const FOCAL_IMAGE_NAME_VHD: &str = "focal-server-cloudimg-amd64-custom-20210609-0.vhd"; 73 #[cfg(target_arch = "x86_64")] 74 const WINDOWS_IMAGE_NAME: &str = "windows-server-2019.raw"; 75 #[cfg(target_arch = "x86_64")] 76 const OVMF_NAME: &str = "OVMF-4b47d0c6c8.fd"; 77 78 const DIRECT_KERNEL_BOOT_CMDLINE: &str = 79 "root=/dev/vda1 console=hvc0 rw systemd.journald.forward_to_console=1"; 80 81 #[cfg(target_arch = "x86_64")] 82 const GREP_SERIAL_IRQ_CMD: &str = "grep -c 'IO-APIC.*ttyS0' /proc/interrupts || true"; 83 #[cfg(target_arch = "aarch64")] 84 const GREP_SERIAL_IRQ_CMD: &str = "grep -c 'GICv3.*uart-pl011' /proc/interrupts || true"; 85 86 const PIPE_SIZE: i32 = 32 << 20; 87 88 const CONSOLE_TEST_STRING: &str = "Started OpenBSD Secure Shell server"; 89 90 fn clh_command(cmd: &str) -> String { 91 env::var("BUILD_TARGET").map_or( 92 format!("target/x86_64-unknown-linux-gnu/release/{}", cmd), 93 |target| format!("target/{}/release/{}", target, cmd), 94 ) 95 } 96 97 fn prepare_virtiofsd( 98 tmp_dir: &TempDir, 99 shared_dir: &str, 100 cache: &str, 101 ) -> (std::process::Child, String) { 102 let mut workload_path = dirs::home_dir().unwrap(); 103 workload_path.push("workloads"); 104 105 let mut virtiofsd_path = workload_path; 106 virtiofsd_path.push("virtiofsd"); 107 let virtiofsd_path = String::from(virtiofsd_path.to_str().unwrap()); 108 109 let virtiofsd_socket_path = 110 String::from(tmp_dir.as_path().join("virtiofs.sock").to_str().unwrap()); 111 112 // Start the daemon 113 let child = Command::new(virtiofsd_path.as_str()) 114 .args(&[format!("--socket-path={}", virtiofsd_socket_path).as_str()]) 115 .args(&["-o", format!("source={}", shared_dir).as_str()]) 116 .args(&["-o", format!("cache={}", cache).as_str()]) 117 .spawn() 118 .unwrap(); 119 120 thread::sleep(std::time::Duration::new(10, 0)); 121 122 (child, virtiofsd_socket_path) 123 } 124 125 fn prepare_virtofsd_rs_daemon( 126 tmp_dir: &TempDir, 127 shared_dir: &str, 128 _cache: &str, 129 ) -> (std::process::Child, String) { 130 let mut workload_path = dirs::home_dir().unwrap(); 131 workload_path.push("workloads"); 132 133 let mut virtiofsd_path = workload_path; 134 virtiofsd_path.push("virtiofsd-rs"); 135 let virtiofsd_path = String::from(virtiofsd_path.to_str().unwrap()); 136 137 let virtiofsd_socket_path = 138 String::from(tmp_dir.as_path().join("virtiofs.sock").to_str().unwrap()); 139 140 // Start the daemon 141 let child = Command::new(virtiofsd_path.as_str()) 142 .args(&["--shared-dir", shared_dir]) 143 .args(&["--socket", virtiofsd_socket_path.as_str()]) 144 .spawn() 145 .unwrap(); 146 147 thread::sleep(std::time::Duration::new(10, 0)); 148 149 (child, virtiofsd_socket_path) 150 } 151 152 fn prepare_vubd( 153 tmp_dir: &TempDir, 154 blk_img: &str, 155 num_queues: usize, 156 rdonly: bool, 157 direct: bool, 158 ) -> (std::process::Child, String) { 159 let mut workload_path = dirs::home_dir().unwrap(); 160 workload_path.push("workloads"); 161 162 let mut blk_file_path = workload_path; 163 blk_file_path.push(blk_img); 164 let blk_file_path = String::from(blk_file_path.to_str().unwrap()); 165 166 let vubd_socket_path = String::from(tmp_dir.as_path().join("vub.sock").to_str().unwrap()); 167 168 // Start the daemon 169 let child = Command::new(clh_command("vhost_user_block")) 170 .args(&[ 171 "--block-backend", 172 format!( 173 "path={},socket={},num_queues={},readonly={},direct={}", 174 blk_file_path, vubd_socket_path, num_queues, rdonly, direct 175 ) 176 .as_str(), 177 ]) 178 .spawn() 179 .unwrap(); 180 181 thread::sleep(std::time::Duration::new(10, 0)); 182 183 (child, vubd_socket_path) 184 } 185 186 fn temp_vsock_path(tmp_dir: &TempDir) -> String { 187 String::from(tmp_dir.as_path().join("vsock").to_str().unwrap()) 188 } 189 190 fn temp_api_path(tmp_dir: &TempDir) -> String { 191 String::from( 192 tmp_dir 193 .as_path() 194 .join("cloud-hypervisor.sock") 195 .to_str() 196 .unwrap(), 197 ) 198 } 199 200 // Creates the directory and returns the path. 201 fn temp_snapshot_dir_path(tmp_dir: &TempDir) -> String { 202 let snapshot_dir = String::from(tmp_dir.as_path().join("snapshot").to_str().unwrap()); 203 std::fs::create_dir(&snapshot_dir).unwrap(); 204 snapshot_dir 205 } 206 207 // Creates the path for direct kernel boot and return the path. 208 // For x86_64, this function returns the vmlinux kernel path. 209 // For AArch64, this function returns the PE kernel path. 210 fn direct_kernel_boot_path() -> PathBuf { 211 let mut workload_path = dirs::home_dir().unwrap(); 212 workload_path.push("workloads"); 213 214 let mut kernel_path = workload_path; 215 #[cfg(target_arch = "x86_64")] 216 kernel_path.push("vmlinux"); 217 #[cfg(target_arch = "aarch64")] 218 kernel_path.push("Image"); 219 220 kernel_path 221 } 222 223 #[cfg(target_arch = "aarch64")] 224 fn edk2_path() -> PathBuf { 225 let mut workload_path = dirs::home_dir().unwrap(); 226 workload_path.push("workloads"); 227 let mut edk2_path = workload_path; 228 edk2_path.push("CLOUDHV_EFI.fd"); 229 230 edk2_path 231 } 232 233 fn prepare_vhost_user_net_daemon( 234 tmp_dir: &TempDir, 235 ip: &str, 236 tap: Option<&str>, 237 num_queues: usize, 238 client_mode: bool, 239 ) -> (std::process::Command, String) { 240 let vunet_socket_path = 241 String::from(tmp_dir.as_path().join("vunet.sock").to_str().unwrap()); 242 243 // Start the daemon 244 let net_params = if let Some(tap_str) = tap { 245 format!( 246 "tap={},ip={},mask=255.255.255.0,socket={},num_queues={},queue_size=1024,client={}", 247 tap_str, ip, vunet_socket_path, num_queues, client_mode 248 ) 249 } else { 250 format!( 251 "ip={},mask=255.255.255.0,socket={},num_queues={},queue_size=1024,client={}", 252 ip, vunet_socket_path, num_queues, client_mode 253 ) 254 }; 255 256 let mut command = Command::new(clh_command("vhost_user_net")); 257 command.args(&["--net-backend", net_params.as_str()]); 258 259 (command, vunet_socket_path) 260 } 261 262 fn curl_command(api_socket: &str, method: &str, url: &str, http_body: Option<&str>) { 263 let mut curl_args: Vec<&str> = 264 ["--unix-socket", api_socket, "-i", "-X", method, url].to_vec(); 265 266 if let Some(body) = http_body { 267 curl_args.push("-H"); 268 curl_args.push("Accept: application/json"); 269 curl_args.push("-H"); 270 curl_args.push("Content-Type: application/json"); 271 curl_args.push("-d"); 272 curl_args.push(body); 273 } 274 275 let status = Command::new("curl") 276 .args(curl_args) 277 .status() 278 .expect("Failed to launch curl command"); 279 280 assert!(status.success()); 281 } 282 283 fn remote_command(api_socket: &str, command: &str, arg: Option<&str>) -> bool { 284 let mut cmd = Command::new(clh_command("ch-remote")); 285 cmd.args(&[&format!("--api-socket={}", api_socket), command]); 286 287 if let Some(arg) = arg { 288 cmd.arg(arg); 289 } 290 291 cmd.status().expect("Failed to launch ch-remote").success() 292 } 293 294 fn remote_command_w_output( 295 api_socket: &str, 296 command: &str, 297 arg: Option<&str>, 298 ) -> (bool, Vec<u8>) { 299 let mut cmd = Command::new(clh_command("ch-remote")); 300 cmd.args(&[&format!("--api-socket={}", api_socket), command]); 301 302 if let Some(arg) = arg { 303 cmd.arg(arg); 304 } 305 306 let output = cmd.output().expect("Failed to launch ch-remote"); 307 308 (output.status.success(), output.stdout) 309 } 310 311 fn resize_command( 312 api_socket: &str, 313 desired_vcpus: Option<u8>, 314 desired_ram: Option<usize>, 315 desired_balloon: Option<usize>, 316 ) -> bool { 317 let mut cmd = Command::new(clh_command("ch-remote")); 318 cmd.args(&[&format!("--api-socket={}", api_socket), "resize"]); 319 320 if let Some(desired_vcpus) = desired_vcpus { 321 cmd.arg(format!("--cpus={}", desired_vcpus)); 322 } 323 324 if let Some(desired_ram) = desired_ram { 325 cmd.arg(format!("--memory={}", desired_ram)); 326 } 327 328 if let Some(desired_balloon) = desired_balloon { 329 cmd.arg(format!("--balloon={}", desired_balloon)); 330 } 331 332 cmd.status().expect("Failed to launch ch-remote").success() 333 } 334 335 fn resize_zone_command(api_socket: &str, id: &str, desired_size: &str) -> bool { 336 let mut cmd = Command::new(clh_command("ch-remote")); 337 cmd.args(&[ 338 &format!("--api-socket={}", api_socket), 339 "resize-zone", 340 &format!("--id={}", id), 341 &format!("--size={}", desired_size), 342 ]); 343 344 cmd.status().expect("Failed to launch ch-remote").success() 345 } 346 347 #[derive(Debug)] 348 enum Error { 349 Parsing(std::num::ParseIntError), 350 SshCommand(SshCommandError), 351 WaitForBoot(WaitForBootError), 352 } 353 354 impl From<SshCommandError> for Error { 355 fn from(e: SshCommandError) -> Self { 356 Self::SshCommand(e) 357 } 358 } 359 360 impl Guest { 361 fn new_from_ip_range(mut disk_config: Box<dyn DiskConfig>, class: &str, id: u8) -> Self { 362 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 363 364 let mut workload_path = dirs::home_dir().unwrap(); 365 workload_path.push("workloads"); 366 367 let mut fw_path = workload_path; 368 #[cfg(target_arch = "aarch64")] 369 fw_path.push("CLOUDHV_EFI.fd"); 370 #[cfg(target_arch = "x86_64")] 371 fw_path.push("hypervisor-fw"); 372 let fw_path = String::from(fw_path.to_str().unwrap()); 373 let network = GuestNetworkConfig { 374 guest_ip: format!("{}.{}.2", class, id), 375 l2_guest_ip1: format!("{}.{}.3", class, id), 376 l2_guest_ip2: format!("{}.{}.4", class, id), 377 l2_guest_ip3: format!("{}.{}.5", class, id), 378 host_ip: format!("{}.{}.1", class, id), 379 guest_mac: format!("12:34:56:78:90:{:02x}", id), 380 l2_guest_mac1: format!("de:ad:be:ef:12:{:02x}", id), 381 l2_guest_mac2: format!("de:ad:be:ef:34:{:02x}", id), 382 l2_guest_mac3: format!("de:ad:be:ef:56:{:02x}", id), 383 tcp_listener_port: DEFAULT_TCP_LISTENER_PORT + id as u16, 384 }; 385 386 disk_config.prepare_files(&tmp_dir, &network); 387 388 Guest { 389 tmp_dir, 390 disk_config, 391 fw_path, 392 network, 393 } 394 } 395 396 fn new(disk_config: Box<dyn DiskConfig>) -> Self { 397 let mut guard = NEXT_VM_ID.lock().unwrap(); 398 let id = *guard; 399 *guard = id + 1; 400 401 Self::new_from_ip_range(disk_config, "192.168", id) 402 } 403 404 fn default_net_string(&self) -> String { 405 format!( 406 "tap=,mac={},ip={},mask=255.255.255.0", 407 self.network.guest_mac, self.network.host_ip 408 ) 409 } 410 411 fn default_net_string_w_iommu(&self) -> String { 412 format!( 413 "tap=,mac={},ip={},mask=255.255.255.0,iommu=on", 414 self.network.guest_mac, self.network.host_ip 415 ) 416 } 417 418 fn ssh_command(&self, command: &str) -> Result<String, SshCommandError> { 419 ssh_command_ip( 420 command, 421 &self.network.guest_ip, 422 DEFAULT_SSH_RETRIES, 423 DEFAULT_SSH_TIMEOUT, 424 ) 425 } 426 427 fn ssh_command_l1(&self, command: &str) -> Result<String, SshCommandError> { 428 ssh_command_ip( 429 command, 430 &self.network.guest_ip, 431 DEFAULT_SSH_RETRIES, 432 DEFAULT_SSH_TIMEOUT, 433 ) 434 } 435 436 fn ssh_command_l2_1(&self, command: &str) -> Result<String, SshCommandError> { 437 ssh_command_ip( 438 command, 439 &self.network.l2_guest_ip1, 440 DEFAULT_SSH_RETRIES, 441 DEFAULT_SSH_TIMEOUT, 442 ) 443 } 444 445 fn ssh_command_l2_2(&self, command: &str) -> Result<String, SshCommandError> { 446 ssh_command_ip( 447 command, 448 &self.network.l2_guest_ip2, 449 DEFAULT_SSH_RETRIES, 450 DEFAULT_SSH_TIMEOUT, 451 ) 452 } 453 454 fn ssh_command_l2_3(&self, command: &str) -> Result<String, SshCommandError> { 455 ssh_command_ip( 456 command, 457 &self.network.l2_guest_ip3, 458 DEFAULT_SSH_RETRIES, 459 DEFAULT_SSH_TIMEOUT, 460 ) 461 } 462 463 fn api_create_body(&self, cpu_count: u8) -> String { 464 #[cfg(target_arch = "x86_64")] 465 format! {"{{\"cpus\":{{\"boot_vcpus\":{},\"max_vcpus\":{}}},\"kernel\":{{\"path\":\"{}\"}},\"cmdline\":{{\"args\": \"\"}},\"net\":[{{\"ip\":\"{}\", \"mask\":\"255.255.255.0\", \"mac\":\"{}\"}}], \"disks\":[{{\"path\":\"{}\"}}, {{\"path\":\"{}\"}}]}}", 466 cpu_count, 467 cpu_count, 468 self.fw_path.as_str(), 469 self.network.host_ip, 470 self.network.guest_mac, 471 self.disk_config.disk(DiskType::OperatingSystem).unwrap().as_str(), 472 self.disk_config.disk(DiskType::CloudInit).unwrap().as_str(), 473 } 474 #[cfg(target_arch = "aarch64")] 475 format! {"{{\"cpus\":{{\"boot_vcpus\":{},\"max_vcpus\":{}}},\"kernel\":{{\"path\":\"{}\"}},\"cmdline\":{{\"args\": \"{}\"}},\"net\":[{{\"ip\":\"{}\", \"mask\":\"255.255.255.0\", \"mac\":\"{}\"}}], \"disks\":[{{\"path\":\"{}\"}}, {{\"path\":\"{}\"}}]}}", 476 cpu_count, 477 cpu_count, 478 direct_kernel_boot_path().to_str().unwrap(), 479 DIRECT_KERNEL_BOOT_CMDLINE, 480 self.network.host_ip, 481 self.network.guest_mac, 482 self.disk_config.disk(DiskType::OperatingSystem).unwrap().as_str(), 483 self.disk_config.disk(DiskType::CloudInit).unwrap().as_str(), 484 } 485 } 486 487 fn get_cpu_count(&self) -> Result<u32, Error> { 488 self.ssh_command("grep -c processor /proc/cpuinfo")? 489 .trim() 490 .parse() 491 .map_err(Error::Parsing) 492 } 493 494 fn get_initial_apicid(&self) -> Result<u32, Error> { 495 self.ssh_command("grep \"initial apicid\" /proc/cpuinfo | grep -o \"[0-9]*\"")? 496 .trim() 497 .parse() 498 .map_err(Error::Parsing) 499 } 500 501 fn get_total_memory(&self) -> Result<u32, Error> { 502 self.ssh_command("grep MemTotal /proc/meminfo | grep -o \"[0-9]*\"")? 503 .trim() 504 .parse() 505 .map_err(Error::Parsing) 506 } 507 508 fn get_total_memory_l2(&self) -> Result<u32, Error> { 509 self.ssh_command_l2_1("grep MemTotal /proc/meminfo | grep -o \"[0-9]*\"")? 510 .trim() 511 .parse() 512 .map_err(Error::Parsing) 513 } 514 515 fn get_numa_node_memory(&self, node_id: usize) -> Result<u32, Error> { 516 self.ssh_command( 517 format!( 518 "grep MemTotal /sys/devices/system/node/node{}/meminfo \ 519 | cut -d \":\" -f 2 | grep -o \"[0-9]*\"", 520 node_id 521 ) 522 .as_str(), 523 )? 524 .trim() 525 .parse() 526 .map_err(Error::Parsing) 527 } 528 529 fn wait_vm_boot(&self, custom_timeout: Option<i32>) -> Result<(), Error> { 530 self.network 531 .wait_vm_boot(custom_timeout) 532 .map_err(Error::WaitForBoot) 533 } 534 535 fn check_numa_node_cpus(&self, node_id: usize, cpus: Vec<usize>) -> Result<(), Error> { 536 for cpu in cpus.iter() { 537 let cmd = format!( 538 "[ -d \"/sys/devices/system/node/node{}/cpu{}\" ]", 539 node_id, cpu 540 ); 541 self.ssh_command(cmd.as_str())?; 542 } 543 544 Ok(()) 545 } 546 547 fn check_numa_node_distances( 548 &self, 549 node_id: usize, 550 distances: &str, 551 ) -> Result<bool, Error> { 552 let cmd = format!("cat /sys/devices/system/node/node{}/distance", node_id); 553 if self.ssh_command(cmd.as_str())?.trim() == distances { 554 Ok(true) 555 } else { 556 Ok(false) 557 } 558 } 559 560 fn check_sgx_support(&self) -> Result<(), Error> { 561 self.ssh_command( 562 "cpuid -l 0x7 -s 0 | tr -s [:space:] | grep -q 'SGX: \ 563 Software Guard Extensions supported = true'", 564 )?; 565 self.ssh_command( 566 "cpuid -l 0x7 -s 0 | tr -s [:space:] | grep -q 'SGX_LC: \ 567 SGX launch config supported = true'", 568 )?; 569 self.ssh_command( 570 "cpuid -l 0x12 -s 0 | tr -s [:space:] | grep -q 'SGX1 \ 571 supported = true'", 572 )?; 573 574 Ok(()) 575 } 576 577 fn get_entropy(&self) -> Result<u32, Error> { 578 self.ssh_command("cat /proc/sys/kernel/random/entropy_avail")? 579 .trim() 580 .parse() 581 .map_err(Error::Parsing) 582 } 583 584 fn get_pci_bridge_class(&self) -> Result<String, Error> { 585 Ok(self 586 .ssh_command("cat /sys/bus/pci/devices/0000:00:00.0/class")? 587 .trim() 588 .to_string()) 589 } 590 591 fn get_pci_device_ids(&self) -> Result<String, Error> { 592 Ok(self 593 .ssh_command("cat /sys/bus/pci/devices/*/device")? 594 .trim() 595 .to_string()) 596 } 597 598 fn get_pci_vendor_ids(&self) -> Result<String, Error> { 599 Ok(self 600 .ssh_command("cat /sys/bus/pci/devices/*/vendor")? 601 .trim() 602 .to_string()) 603 } 604 605 fn does_device_vendor_pair_match( 606 &self, 607 device_id: &str, 608 vendor_id: &str, 609 ) -> Result<bool, Error> { 610 // We are checking if console device's device id and vendor id pair matches 611 let devices = self.get_pci_device_ids()?; 612 let devices: Vec<&str> = devices.split('\n').collect(); 613 let vendors = self.get_pci_vendor_ids()?; 614 let vendors: Vec<&str> = vendors.split('\n').collect(); 615 616 for (index, d_id) in devices.iter().enumerate() { 617 if *d_id == device_id { 618 if let Some(v_id) = vendors.get(index) { 619 if *v_id == vendor_id { 620 return Ok(true); 621 } 622 } 623 } 624 } 625 626 Ok(false) 627 } 628 629 fn valid_virtio_fs_cache_size( 630 &self, 631 dax: bool, 632 cache_size: Option<u64>, 633 ) -> Result<bool, Error> { 634 // SHM region is called different things depending on kernel 635 let shm_region = self 636 .ssh_command("sudo grep 'virtio[0-9]\\|virtio-pci-shm' /proc/iomem || true")? 637 .trim() 638 .to_string(); 639 640 if shm_region.is_empty() { 641 return Ok(!dax); 642 } 643 644 // From this point, the region is not empty, hence it is an error 645 // if DAX is off. 646 if !dax { 647 return Ok(false); 648 } 649 650 let cache = if let Some(cache) = cache_size { 651 cache 652 } else { 653 // 8Gib by default 654 0x0002_0000_0000 655 }; 656 657 let args: Vec<&str> = shm_region.split(':').collect(); 658 if args.is_empty() { 659 return Ok(false); 660 } 661 662 let args: Vec<&str> = args[0].trim().split('-').collect(); 663 if args.len() != 2 { 664 return Ok(false); 665 } 666 667 let start_addr = u64::from_str_radix(args[0], 16).map_err(Error::Parsing)?; 668 let end_addr = u64::from_str_radix(args[1], 16).map_err(Error::Parsing)?; 669 670 Ok(cache == (end_addr - start_addr + 1)) 671 } 672 673 fn check_vsock(&self, socket: &str) { 674 // Listen from guest on vsock CID=3 PORT=16 675 // SOCKET-LISTEN:<domain>:<protocol>:<local-address> 676 let guest_ip = self.network.guest_ip.clone(); 677 let listen_socat = thread::spawn(move || { 678 ssh_command_ip("sudo socat - SOCKET-LISTEN:40:0:x00x00x10x00x00x00x03x00x00x00x00x00x00x00 > vsock_log", &guest_ip, DEFAULT_SSH_RETRIES, DEFAULT_SSH_TIMEOUT).unwrap(); 679 }); 680 681 // Make sure socat is listening, which might take a few second on slow systems 682 thread::sleep(std::time::Duration::new(10, 0)); 683 684 // Write something to vsock from the host 685 assert!(exec_host_command_status(&format!( 686 "echo -e \"CONNECT 16\\nHelloWorld!\" | socat - UNIX-CONNECT:{}", 687 socket 688 )) 689 .success()); 690 691 // Wait for the thread to terminate. 692 listen_socat.join().unwrap(); 693 694 assert_eq!( 695 self.ssh_command("cat vsock_log").unwrap().trim(), 696 "HelloWorld!" 697 ); 698 } 699 700 fn check_nvidia_gpu(&self) { 701 // Run CUDA sample to validate it can find the device 702 let device_query_result = self 703 .ssh_command( 704 "sudo /root/NVIDIA_CUDA-11.3_Samples/bin/x86_64/linux/release/deviceQuery", 705 ) 706 .unwrap(); 707 assert!(device_query_result.contains("Detected 1 CUDA Capable device")); 708 assert!(device_query_result.contains("Device 0: \"NVIDIA Tesla T4\"")); 709 assert!(device_query_result.contains("Result = PASS")); 710 711 // Run NVIDIA DCGM Diagnostics to validate the device is functional 712 self.ssh_command("sudo nv-hostengine").unwrap(); 713 714 assert!(self 715 .ssh_command("sudo dcgmi discovery -l") 716 .unwrap() 717 .contains("Name: NVIDIA Tesla T4")); 718 assert_eq!( 719 self.ssh_command("sudo dcgmi diag -r 'diagnostic' | grep Pass | wc -l") 720 .unwrap() 721 .trim(), 722 "10" 723 ); 724 } 725 726 fn reboot_linux(&self, current_reboot_count: u32, custom_timeout: Option<i32>) { 727 let list_boots_cmd = "sudo journalctl --list-boots | wc -l"; 728 let boot_count = self 729 .ssh_command(list_boots_cmd) 730 .unwrap() 731 .trim() 732 .parse::<u32>() 733 .unwrap_or_default(); 734 735 assert_eq!(boot_count, current_reboot_count + 1); 736 self.ssh_command("sudo reboot").unwrap(); 737 738 self.wait_vm_boot(custom_timeout).unwrap(); 739 let boot_count = self 740 .ssh_command(list_boots_cmd) 741 .unwrap() 742 .trim() 743 .parse::<u32>() 744 .unwrap_or_default(); 745 assert_eq!(boot_count, current_reboot_count + 2); 746 } 747 748 fn enable_memory_hotplug(&self) { 749 self.ssh_command( 750 "echo online | sudo tee /sys/devices/system/memory/auto_online_blocks", 751 ) 752 .unwrap(); 753 } 754 } 755 756 struct GuestCommand<'a> { 757 command: Command, 758 guest: &'a Guest, 759 capture_output: bool, 760 } 761 762 impl<'a> GuestCommand<'a> { 763 fn new(guest: &'a Guest) -> Self { 764 Self::new_with_binary_name(guest, "cloud-hypervisor") 765 } 766 767 fn new_with_binary_name(guest: &'a Guest, binary_name: &str) -> Self { 768 Self { 769 command: Command::new(clh_command(binary_name)), 770 guest, 771 capture_output: false, 772 } 773 } 774 775 fn capture_output(&mut self) -> &mut Self { 776 self.capture_output = true; 777 self 778 } 779 780 fn spawn(&mut self) -> io::Result<Child> { 781 println!( 782 "\n\n==== Start cloud-hypervisor command-line ====\n\n\ 783 {:?}\n\ 784 \n==== End cloud-hypervisor command-line ====\n\n", 785 self.command 786 ); 787 788 if self.capture_output { 789 let child = self 790 .command 791 .arg("-v") 792 .stderr(Stdio::piped()) 793 .stdout(Stdio::piped()) 794 .spawn() 795 .unwrap(); 796 797 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 798 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 799 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 800 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 801 802 if pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE { 803 Ok(child) 804 } else { 805 Err(std::io::Error::new( 806 std::io::ErrorKind::Other, 807 "resizing pipe w/ 'fnctl' failed!", 808 )) 809 } 810 } else { 811 self.command.arg("-v").spawn() 812 } 813 } 814 815 fn args<I, S>(&mut self, args: I) -> &mut Self 816 where 817 I: IntoIterator<Item = S>, 818 S: AsRef<OsStr>, 819 { 820 self.command.args(args); 821 self 822 } 823 824 fn default_disks(&mut self) -> &mut Self { 825 if self.guest.disk_config.disk(DiskType::CloudInit).is_some() { 826 self.args(&[ 827 "--disk", 828 format!( 829 "path={}", 830 self.guest 831 .disk_config 832 .disk(DiskType::OperatingSystem) 833 .unwrap() 834 ) 835 .as_str(), 836 format!( 837 "path={}", 838 self.guest.disk_config.disk(DiskType::CloudInit).unwrap() 839 ) 840 .as_str(), 841 ]) 842 } else { 843 self.args(&[ 844 "--disk", 845 format!( 846 "path={}", 847 self.guest 848 .disk_config 849 .disk(DiskType::OperatingSystem) 850 .unwrap() 851 ) 852 .as_str(), 853 ]) 854 } 855 } 856 857 fn default_net(&mut self) -> &mut Self { 858 self.args(&["--net", self.guest.default_net_string().as_str()]) 859 } 860 } 861 862 fn test_cpu_topology(threads_per_core: u8, cores_per_package: u8, packages: u8, use_fw: bool) { 863 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 864 let guest = Guest::new(Box::new(focal)); 865 let total_vcpus = threads_per_core * cores_per_package * packages; 866 let direct_kernel_boot_path = direct_kernel_boot_path(); 867 let mut kernel_path = direct_kernel_boot_path.to_str().unwrap(); 868 if use_fw { 869 kernel_path = guest.fw_path.as_str(); 870 } 871 872 let mut child = GuestCommand::new(&guest) 873 .args(&[ 874 "--cpus", 875 &format!( 876 "boot={},topology={}:{}:1:{}", 877 total_vcpus, threads_per_core, cores_per_package, packages 878 ), 879 ]) 880 .args(&["--memory", "size=512M"]) 881 .args(&["--kernel", kernel_path]) 882 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 883 .default_disks() 884 .default_net() 885 .capture_output() 886 .spawn() 887 .unwrap(); 888 889 let r = std::panic::catch_unwind(|| { 890 guest.wait_vm_boot(None).unwrap(); 891 assert_eq!( 892 guest.get_cpu_count().unwrap_or_default(), 893 u32::from(total_vcpus) 894 ); 895 assert_eq!( 896 guest 897 .ssh_command("lscpu | grep \"per core\" | cut -f 2 -d \":\" | sed \"s# *##\"") 898 .unwrap() 899 .trim() 900 .parse::<u8>() 901 .unwrap_or(0), 902 threads_per_core 903 ); 904 905 assert_eq!( 906 guest 907 .ssh_command("lscpu | grep \"per socket\" | cut -f 2 -d \":\" | sed \"s# *##\"") 908 .unwrap() 909 .trim() 910 .parse::<u8>() 911 .unwrap_or(0), 912 cores_per_package 913 ); 914 915 assert_eq!( 916 guest 917 .ssh_command("lscpu | grep \"Socket\" | cut -f 2 -d \":\" | sed \"s# *##\"") 918 .unwrap() 919 .trim() 920 .parse::<u8>() 921 .unwrap_or(0), 922 packages 923 ); 924 }); 925 926 let _ = child.kill(); 927 let output = child.wait_with_output().unwrap(); 928 929 handle_child_output(r, &output); 930 } 931 932 type PrepareNetDaemon = 933 dyn Fn(&TempDir, &str, Option<&str>, usize, bool) -> (std::process::Command, String); 934 935 fn test_vhost_user_net( 936 tap: Option<&str>, 937 num_queues: usize, 938 prepare_daemon: &PrepareNetDaemon, 939 generate_host_mac: bool, 940 client_mode_daemon: bool, 941 ) { 942 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 943 let guest = Guest::new(Box::new(focal)); 944 let api_socket = temp_api_path(&guest.tmp_dir); 945 946 let kernel_path = direct_kernel_boot_path(); 947 948 let host_mac = if generate_host_mac { 949 Some(MacAddr::local_random()) 950 } else { 951 None 952 }; 953 954 let (mut daemon_command, vunet_socket_path) = prepare_daemon( 955 &guest.tmp_dir, 956 &guest.network.host_ip, 957 tap, 958 num_queues, 959 client_mode_daemon, 960 ); 961 962 let net_params = format!( 963 "vhost_user=true,mac={},socket={},num_queues={},queue_size=1024{},vhost_mode={}", 964 guest.network.guest_mac, 965 vunet_socket_path, 966 num_queues, 967 if let Some(host_mac) = host_mac { 968 format!(",host_mac={}", host_mac) 969 } else { 970 "".to_owned() 971 }, 972 if client_mode_daemon { 973 "server" 974 } else { 975 "client" 976 }, 977 ); 978 979 let mut ch_command = GuestCommand::new(&guest); 980 ch_command 981 .args(&["--cpus", format!("boot={}", num_queues / 2).as_str()]) 982 .args(&["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 983 .args(&["--kernel", kernel_path.to_str().unwrap()]) 984 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 985 .default_disks() 986 .args(&["--net", net_params.as_str()]) 987 .args(&["--api-socket", &api_socket]) 988 .capture_output(); 989 990 let mut daemon_child: std::process::Child; 991 let mut child: std::process::Child; 992 993 if client_mode_daemon { 994 child = ch_command.spawn().unwrap(); 995 // Make sure the VMM is waiting for the backend to connect 996 thread::sleep(std::time::Duration::new(10, 0)); 997 daemon_child = daemon_command.spawn().unwrap(); 998 } else { 999 daemon_child = daemon_command.spawn().unwrap(); 1000 // Make sure the backend is waiting for the VMM to connect 1001 thread::sleep(std::time::Duration::new(10, 0)); 1002 child = ch_command.spawn().unwrap(); 1003 } 1004 1005 let r = std::panic::catch_unwind(|| { 1006 guest.wait_vm_boot(None).unwrap(); 1007 1008 if let Some(tap_name) = tap { 1009 let tap_count = 1010 exec_host_command_output(&format!("ip link | grep -c {}", tap_name)); 1011 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 1012 } 1013 1014 if let Some(host_mac) = tap { 1015 let mac_count = 1016 exec_host_command_output(&format!("ip link | grep -c {}", host_mac)); 1017 assert_eq!(String::from_utf8_lossy(&mac_count.stdout).trim(), "1"); 1018 } 1019 1020 // 1 network interface + default localhost ==> 2 interfaces 1021 // It's important to note that this test is fully exercising the 1022 // vhost-user-net implementation and the associated backend since 1023 // it does not define any --net network interface. That means all 1024 // the ssh communication in that test happens through the network 1025 // interface backed by vhost-user-net. 1026 assert_eq!( 1027 guest 1028 .ssh_command("ip -o link | wc -l") 1029 .unwrap() 1030 .trim() 1031 .parse::<u32>() 1032 .unwrap_or_default(), 1033 2 1034 ); 1035 1036 // The following pci devices will appear on guest with PCI-MSI 1037 // interrupt vectors assigned. 1038 // 1 virtio-console with 3 vectors: config, Rx, Tx 1039 // 1 virtio-blk with 2 vectors: config, Request 1040 // 1 virtio-blk with 2 vectors: config, Request 1041 // 1 virtio-rng with 2 vectors: config, Request 1042 // Since virtio-net has 2 queue pairs, its vectors is as follows: 1043 // 1 virtio-net with 5 vectors: config, Rx (2), Tx (2) 1044 // Based on the above, the total vectors should 14. 1045 #[cfg(target_arch = "x86_64")] 1046 let grep_cmd = "grep -c PCI-MSI /proc/interrupts"; 1047 #[cfg(target_arch = "aarch64")] 1048 let grep_cmd = "grep -c ITS-MSI /proc/interrupts"; 1049 assert_eq!( 1050 guest 1051 .ssh_command(grep_cmd) 1052 .unwrap() 1053 .trim() 1054 .parse::<u32>() 1055 .unwrap_or_default(), 1056 10 + (num_queues as u32) 1057 ); 1058 1059 // ACPI feature is needed. 1060 #[cfg(all(target_arch = "x86_64", feature = "acpi"))] 1061 { 1062 guest.enable_memory_hotplug(); 1063 1064 // Add RAM to the VM 1065 let desired_ram = 1024 << 20; 1066 resize_command(&api_socket, None, Some(desired_ram), None); 1067 1068 thread::sleep(std::time::Duration::new(10, 0)); 1069 1070 // Here by simply checking the size (through ssh), we validate 1071 // the connection is still working, which means vhost-user-net 1072 // keeps working after the resize. 1073 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 1074 } 1075 }); 1076 1077 let _ = child.kill(); 1078 let output = child.wait_with_output().unwrap(); 1079 1080 thread::sleep(std::time::Duration::new(5, 0)); 1081 let _ = daemon_child.kill(); 1082 let _ = daemon_child.wait(); 1083 1084 handle_child_output(r, &output); 1085 } 1086 1087 type PrepareBlkDaemon = 1088 dyn Fn(&TempDir, &str, usize, bool, bool) -> (std::process::Child, String); 1089 1090 fn test_vhost_user_blk( 1091 num_queues: usize, 1092 readonly: bool, 1093 direct: bool, 1094 prepare_vhost_user_blk_daemon: Option<&PrepareBlkDaemon>, 1095 ) { 1096 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1097 let guest = Guest::new(Box::new(focal)); 1098 let api_socket = temp_api_path(&guest.tmp_dir); 1099 1100 let kernel_path = direct_kernel_boot_path(); 1101 1102 let (blk_params, daemon_child) = { 1103 let prepare_daemon = prepare_vhost_user_blk_daemon.unwrap(); 1104 // Start the daemon 1105 let (daemon_child, vubd_socket_path) = 1106 prepare_daemon(&guest.tmp_dir, "blk.img", num_queues, readonly, direct); 1107 1108 ( 1109 format!( 1110 "vhost_user=true,socket={},num_queues={},queue_size=128", 1111 vubd_socket_path, num_queues, 1112 ), 1113 Some(daemon_child), 1114 ) 1115 }; 1116 1117 let mut child = GuestCommand::new(&guest) 1118 .args(&["--cpus", format!("boot={}", num_queues).as_str()]) 1119 .args(&["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 1120 .args(&["--kernel", kernel_path.to_str().unwrap()]) 1121 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1122 .args(&[ 1123 "--disk", 1124 format!( 1125 "path={}", 1126 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 1127 ) 1128 .as_str(), 1129 format!( 1130 "path={}", 1131 guest.disk_config.disk(DiskType::CloudInit).unwrap() 1132 ) 1133 .as_str(), 1134 blk_params.as_str(), 1135 ]) 1136 .default_net() 1137 .args(&["--api-socket", &api_socket]) 1138 .capture_output() 1139 .spawn() 1140 .unwrap(); 1141 1142 let r = std::panic::catch_unwind(|| { 1143 guest.wait_vm_boot(None).unwrap(); 1144 1145 // Check both if /dev/vdc exists and if the block size is 16M. 1146 assert_eq!( 1147 guest 1148 .ssh_command("lsblk | grep vdc | grep -c 16M") 1149 .unwrap() 1150 .trim() 1151 .parse::<u32>() 1152 .unwrap_or_default(), 1153 1 1154 ); 1155 1156 // Check if this block is RO or RW. 1157 assert_eq!( 1158 guest 1159 .ssh_command("lsblk | grep vdc | awk '{print $5}'") 1160 .unwrap() 1161 .trim() 1162 .parse::<u32>() 1163 .unwrap_or_default(), 1164 readonly as u32 1165 ); 1166 1167 // Check if the number of queues in /sys/block/vdc/mq matches the 1168 // expected num_queues. 1169 assert_eq!( 1170 guest 1171 .ssh_command("ls -ll /sys/block/vdc/mq | grep ^d | wc -l") 1172 .unwrap() 1173 .trim() 1174 .parse::<u32>() 1175 .unwrap_or_default(), 1176 num_queues as u32 1177 ); 1178 1179 // Mount the device 1180 let mount_ro_rw_flag = if readonly { "ro,noload" } else { "rw" }; 1181 guest.ssh_command("mkdir mount_image").unwrap(); 1182 guest 1183 .ssh_command( 1184 format!( 1185 "sudo mount -o {} -t ext4 /dev/vdc mount_image/", 1186 mount_ro_rw_flag 1187 ) 1188 .as_str(), 1189 ) 1190 .unwrap(); 1191 1192 // Check the content of the block device. The file "foo" should 1193 // contain "bar". 1194 assert_eq!( 1195 guest.ssh_command("cat mount_image/foo").unwrap().trim(), 1196 "bar" 1197 ); 1198 1199 // ACPI feature is needed. 1200 #[cfg(all(target_arch = "x86_64", feature = "acpi"))] 1201 { 1202 guest.enable_memory_hotplug(); 1203 1204 // Add RAM to the VM 1205 let desired_ram = 1024 << 20; 1206 resize_command(&api_socket, None, Some(desired_ram), None); 1207 1208 thread::sleep(std::time::Duration::new(10, 0)); 1209 1210 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 1211 1212 // Check again the content of the block device after the resize 1213 // has been performed. 1214 assert_eq!( 1215 guest.ssh_command("cat mount_image/foo").unwrap().trim(), 1216 "bar" 1217 ); 1218 } 1219 1220 // Unmount the device 1221 guest.ssh_command("sudo umount /dev/vdc").unwrap(); 1222 guest.ssh_command("rm -r mount_image").unwrap(); 1223 }); 1224 1225 let _ = child.kill(); 1226 let output = child.wait_with_output().unwrap(); 1227 1228 if let Some(mut daemon_child) = daemon_child { 1229 thread::sleep(std::time::Duration::new(5, 0)); 1230 let _ = daemon_child.kill(); 1231 let _ = daemon_child.wait(); 1232 } 1233 1234 handle_child_output(r, &output); 1235 } 1236 1237 fn test_boot_from_vhost_user_blk( 1238 num_queues: usize, 1239 readonly: bool, 1240 direct: bool, 1241 prepare_vhost_user_blk_daemon: Option<&PrepareBlkDaemon>, 1242 ) { 1243 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1244 let guest = Guest::new(Box::new(focal)); 1245 1246 let kernel_path = direct_kernel_boot_path(); 1247 1248 let disk_path = guest.disk_config.disk(DiskType::OperatingSystem).unwrap(); 1249 1250 let (blk_boot_params, daemon_child) = { 1251 let prepare_daemon = prepare_vhost_user_blk_daemon.unwrap(); 1252 // Start the daemon 1253 let (daemon_child, vubd_socket_path) = prepare_daemon( 1254 &guest.tmp_dir, 1255 disk_path.as_str(), 1256 num_queues, 1257 readonly, 1258 direct, 1259 ); 1260 1261 ( 1262 format!( 1263 "vhost_user=true,socket={},num_queues={},queue_size=128", 1264 vubd_socket_path, num_queues, 1265 ), 1266 Some(daemon_child), 1267 ) 1268 }; 1269 1270 let mut child = GuestCommand::new(&guest) 1271 .args(&["--cpus", format!("boot={}", num_queues).as_str()]) 1272 .args(&["--memory", "size=512M,shared=on"]) 1273 .args(&["--kernel", kernel_path.to_str().unwrap()]) 1274 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1275 .args(&[ 1276 "--disk", 1277 blk_boot_params.as_str(), 1278 format!( 1279 "path={}", 1280 guest.disk_config.disk(DiskType::CloudInit).unwrap() 1281 ) 1282 .as_str(), 1283 ]) 1284 .default_net() 1285 .capture_output() 1286 .spawn() 1287 .unwrap(); 1288 1289 let r = std::panic::catch_unwind(|| { 1290 guest.wait_vm_boot(None).unwrap(); 1291 1292 // Just check the VM booted correctly. 1293 assert_eq!(guest.get_cpu_count().unwrap_or_default(), num_queues as u32); 1294 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 1295 }); 1296 let _ = child.kill(); 1297 let output = child.wait_with_output().unwrap(); 1298 1299 if let Some(mut daemon_child) = daemon_child { 1300 thread::sleep(std::time::Duration::new(5, 0)); 1301 let _ = daemon_child.kill(); 1302 let _ = daemon_child.wait(); 1303 } 1304 1305 handle_child_output(r, &output); 1306 } 1307 1308 fn test_virtio_fs( 1309 dax: bool, 1310 cache_size: Option<u64>, 1311 virtiofsd_cache: &str, 1312 prepare_daemon: &dyn Fn(&TempDir, &str, &str) -> (std::process::Child, String), 1313 hotplug: bool, 1314 ) { 1315 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1316 let guest = Guest::new(Box::new(focal)); 1317 let api_socket = temp_api_path(&guest.tmp_dir); 1318 1319 let mut workload_path = dirs::home_dir().unwrap(); 1320 workload_path.push("workloads"); 1321 1322 let mut shared_dir = workload_path; 1323 shared_dir.push("shared_dir"); 1324 1325 let kernel_path = direct_kernel_boot_path(); 1326 1327 let (dax_vmm_param, dax_mount_param) = if dax { ("on", "-o dax") } else { ("off", "") }; 1328 let cache_size_vmm_param = if let Some(cache) = cache_size { 1329 format!(",cache_size={}", cache) 1330 } else { 1331 "".to_string() 1332 }; 1333 1334 let (mut daemon_child, virtiofsd_socket_path) = prepare_daemon( 1335 &guest.tmp_dir, 1336 shared_dir.to_str().unwrap(), 1337 virtiofsd_cache, 1338 ); 1339 1340 let mut guest_command = GuestCommand::new(&guest); 1341 guest_command 1342 .args(&["--cpus", "boot=1"]) 1343 .args(&["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 1344 .args(&["--kernel", kernel_path.to_str().unwrap()]) 1345 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1346 .default_disks() 1347 .default_net() 1348 .args(&["--api-socket", &api_socket]); 1349 1350 let fs_params = format!( 1351 "id=myfs0,tag=myfs,socket={},num_queues=1,queue_size=1024,dax={}{}", 1352 virtiofsd_socket_path, dax_vmm_param, cache_size_vmm_param 1353 ); 1354 1355 if !hotplug { 1356 guest_command.args(&["--fs", fs_params.as_str()]); 1357 } 1358 1359 let mut child = guest_command.capture_output().spawn().unwrap(); 1360 1361 let r = std::panic::catch_unwind(|| { 1362 guest.wait_vm_boot(None).unwrap(); 1363 1364 if hotplug { 1365 // Add fs to the VM 1366 let (cmd_success, cmd_output) = 1367 remote_command_w_output(&api_socket, "add-fs", Some(&fs_params)); 1368 assert!(cmd_success); 1369 assert!(String::from_utf8_lossy(&cmd_output) 1370 .contains("{\"id\":\"myfs0\",\"bdf\":\"0000:00:06.0\"}")); 1371 1372 thread::sleep(std::time::Duration::new(10, 0)); 1373 } 1374 1375 // Mount shared directory through virtio_fs filesystem 1376 let mount_cmd = format!( 1377 "mkdir -p mount_dir && \ 1378 sudo mount -t virtiofs {} myfs mount_dir/", 1379 dax_mount_param 1380 ); 1381 guest.ssh_command(&mount_cmd).unwrap(); 1382 1383 assert!(guest 1384 .valid_virtio_fs_cache_size(dax, cache_size) 1385 .unwrap_or_default()); 1386 1387 // Check file1 exists and its content is "foo" 1388 assert_eq!( 1389 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1390 "foo" 1391 ); 1392 // Check file2 does not exist 1393 guest 1394 .ssh_command("[ ! -f 'mount_dir/file2' ] || true") 1395 .unwrap(); 1396 1397 // Check file3 exists and its content is "bar" 1398 assert_eq!( 1399 guest.ssh_command("cat mount_dir/file3").unwrap().trim(), 1400 "bar" 1401 ); 1402 1403 // ACPI feature is needed. 1404 #[cfg(all(target_arch = "x86_64", feature = "acpi"))] 1405 { 1406 guest.enable_memory_hotplug(); 1407 1408 // Add RAM to the VM 1409 let desired_ram = 1024 << 20; 1410 resize_command(&api_socket, None, Some(desired_ram), None); 1411 1412 thread::sleep(std::time::Duration::new(30, 0)); 1413 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 1414 1415 // After the resize, check again that file1 exists and its 1416 // content is "foo". 1417 assert_eq!( 1418 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1419 "foo" 1420 ); 1421 } 1422 1423 if hotplug { 1424 // Remove from VM 1425 guest.ssh_command("sudo umount mount_dir").unwrap(); 1426 assert!(remote_command(&api_socket, "remove-device", Some("myfs0"))); 1427 } 1428 }); 1429 1430 let (r, hotplug_daemon_child) = if r.is_ok() && hotplug { 1431 thread::sleep(std::time::Duration::new(10, 0)); 1432 let (daemon_child, virtiofsd_socket_path) = prepare_daemon( 1433 &guest.tmp_dir, 1434 shared_dir.to_str().unwrap(), 1435 virtiofsd_cache, 1436 ); 1437 1438 let r = std::panic::catch_unwind(|| { 1439 thread::sleep(std::time::Duration::new(10, 0)); 1440 let fs_params = format!( 1441 "id=myfs0,tag=myfs,socket={},num_queues=1,queue_size=1024,dax={}{}", 1442 virtiofsd_socket_path, dax_vmm_param, cache_size_vmm_param 1443 ); 1444 1445 // Add back and check it works 1446 let (cmd_success, cmd_output) = 1447 remote_command_w_output(&api_socket, "add-fs", Some(&fs_params)); 1448 assert!(cmd_success); 1449 assert!(String::from_utf8_lossy(&cmd_output) 1450 .contains("{\"id\":\"myfs0\",\"bdf\":\"0000:00:06.0\"}")); 1451 thread::sleep(std::time::Duration::new(10, 0)); 1452 // Mount shared directory through virtio_fs filesystem 1453 let mount_cmd = format!( 1454 "mkdir -p mount_dir && \ 1455 sudo mount -t virtiofs {} myfs mount_dir/", 1456 dax_mount_param 1457 ); 1458 guest.ssh_command(&mount_cmd).unwrap(); 1459 // Check file1 exists and its content is "foo" 1460 assert_eq!( 1461 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1462 "foo" 1463 ); 1464 }); 1465 1466 (r, Some(daemon_child)) 1467 } else { 1468 (r, None) 1469 }; 1470 1471 let _ = child.kill(); 1472 let output = child.wait_with_output().unwrap(); 1473 1474 let _ = daemon_child.kill(); 1475 let _ = daemon_child.wait(); 1476 1477 if let Some(mut daemon_child) = hotplug_daemon_child { 1478 let _ = daemon_child.kill(); 1479 let _ = daemon_child.wait(); 1480 } 1481 1482 handle_child_output(r, &output); 1483 } 1484 1485 fn test_virtio_pmem(discard_writes: bool, specify_size: bool) { 1486 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1487 let guest = Guest::new(Box::new(focal)); 1488 1489 let kernel_path = direct_kernel_boot_path(); 1490 1491 let pmem_temp_file = TempFile::new().unwrap(); 1492 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 1493 1494 std::process::Command::new("mkfs.ext4") 1495 .arg(pmem_temp_file.as_path()) 1496 .output() 1497 .expect("Expect creating disk image to succeed"); 1498 1499 let mut child = GuestCommand::new(&guest) 1500 .args(&["--cpus", "boot=1"]) 1501 .args(&["--memory", "size=512M"]) 1502 .args(&["--kernel", kernel_path.to_str().unwrap()]) 1503 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1504 .default_disks() 1505 .default_net() 1506 .args(&[ 1507 "--pmem", 1508 format!( 1509 "file={}{}{}", 1510 pmem_temp_file.as_path().to_str().unwrap(), 1511 if specify_size { ",size=128M" } else { "" }, 1512 if discard_writes { 1513 ",discard_writes=on" 1514 } else { 1515 "" 1516 } 1517 ) 1518 .as_str(), 1519 ]) 1520 .capture_output() 1521 .spawn() 1522 .unwrap(); 1523 1524 let r = std::panic::catch_unwind(|| { 1525 guest.wait_vm_boot(None).unwrap(); 1526 1527 // Check for the presence of /dev/pmem0 1528 assert_eq!( 1529 guest.ssh_command("ls /dev/pmem0").unwrap().trim(), 1530 "/dev/pmem0" 1531 ); 1532 1533 // Check changes persist after reboot 1534 assert_eq!(guest.ssh_command("sudo mount /dev/pmem0 /mnt").unwrap(), ""); 1535 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), "lost+found\n"); 1536 guest 1537 .ssh_command("echo test123 | sudo tee /mnt/test") 1538 .unwrap(); 1539 assert_eq!(guest.ssh_command("sudo umount /mnt").unwrap(), ""); 1540 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), ""); 1541 1542 guest.reboot_linux(0, None); 1543 assert_eq!(guest.ssh_command("sudo mount /dev/pmem0 /mnt").unwrap(), ""); 1544 assert_eq!( 1545 guest 1546 .ssh_command("sudo cat /mnt/test || true") 1547 .unwrap() 1548 .trim(), 1549 if discard_writes { "" } else { "test123" } 1550 ); 1551 }); 1552 1553 let _ = child.kill(); 1554 let output = child.wait_with_output().unwrap(); 1555 1556 handle_child_output(r, &output); 1557 } 1558 1559 fn get_fd_count(pid: u32) -> usize { 1560 fs::read_dir(format!("/proc/{}/fd", pid)).unwrap().count() 1561 } 1562 1563 fn _test_virtio_vsock(hotplug: bool) { 1564 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1565 let guest = Guest::new(Box::new(focal)); 1566 1567 let kernel_path = direct_kernel_boot_path(); 1568 1569 let socket = temp_vsock_path(&guest.tmp_dir); 1570 let api_socket = temp_api_path(&guest.tmp_dir); 1571 1572 let mut cmd = GuestCommand::new(&guest); 1573 cmd.args(&["--api-socket", &api_socket]); 1574 cmd.args(&["--cpus", "boot=1"]); 1575 cmd.args(&["--memory", "size=512M"]); 1576 cmd.args(&["--kernel", kernel_path.to_str().unwrap()]); 1577 cmd.args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]); 1578 cmd.default_disks(); 1579 cmd.default_net(); 1580 1581 if !hotplug { 1582 cmd.args(&["--vsock", format!("cid=3,socket={}", socket).as_str()]); 1583 } 1584 1585 let mut child = cmd.capture_output().spawn().unwrap(); 1586 1587 let r = std::panic::catch_unwind(|| { 1588 guest.wait_vm_boot(None).unwrap(); 1589 1590 if hotplug { 1591 let (cmd_success, cmd_output) = remote_command_w_output( 1592 &api_socket, 1593 "add-vsock", 1594 Some(format!("cid=3,socket={},id=test0", socket).as_str()), 1595 ); 1596 assert!(cmd_success); 1597 assert!(String::from_utf8_lossy(&cmd_output) 1598 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 1599 thread::sleep(std::time::Duration::new(10, 0)); 1600 // Check adding a second one fails 1601 assert!(!remote_command( 1602 &api_socket, 1603 "add-vsock", 1604 Some("cid=1234,socket=/tmp/fail") 1605 )); 1606 } 1607 1608 // Validate vsock works as expected. 1609 guest.check_vsock(socket.as_str()); 1610 1611 // AArch64 currently does not support reboot, and therefore we 1612 // skip the reboot test here. 1613 #[cfg(target_arch = "x86_64")] 1614 { 1615 guest.reboot_linux(0, None); 1616 1617 // Validate vsock still works after a reboot. 1618 guest.check_vsock(socket.as_str()); 1619 } 1620 if hotplug { 1621 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 1622 } 1623 }); 1624 1625 let _ = child.kill(); 1626 let output = child.wait_with_output().unwrap(); 1627 1628 handle_child_output(r, &output); 1629 } 1630 1631 fn get_ksm_pages_shared() -> u32 { 1632 fs::read_to_string("/sys/kernel/mm/ksm/pages_shared") 1633 .unwrap() 1634 .trim() 1635 .parse::<u32>() 1636 .unwrap() 1637 } 1638 1639 fn test_memory_mergeable(mergeable: bool) { 1640 let memory_param = if mergeable { 1641 "mergeable=on" 1642 } else { 1643 "mergeable=off" 1644 }; 1645 1646 // We are assuming the rest of the system in our CI is not using mergeable memeory 1647 let ksm_ps_init = get_ksm_pages_shared(); 1648 assert!(ksm_ps_init == 0); 1649 1650 let focal1 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1651 let guest1 = Guest::new(Box::new(focal1)); 1652 let mut child1 = GuestCommand::new(&guest1) 1653 .args(&["--cpus", "boot=1"]) 1654 .args(&["--memory", format!("size=512M,{}", memory_param).as_str()]) 1655 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 1656 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1657 .default_disks() 1658 .args(&["--net", guest1.default_net_string().as_str()]) 1659 .args(&["--serial", "tty", "--console", "off"]) 1660 .capture_output() 1661 .spawn() 1662 .unwrap(); 1663 1664 let r = std::panic::catch_unwind(|| { 1665 guest1.wait_vm_boot(None).unwrap(); 1666 }); 1667 if r.is_err() { 1668 let _ = child1.kill(); 1669 let output = child1.wait_with_output().unwrap(); 1670 handle_child_output(r, &output); 1671 panic!("Test should already be failed/panicked"); // To explicitly mark this block never return 1672 } 1673 1674 let ksm_ps_guest1 = get_ksm_pages_shared(); 1675 1676 let focal2 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1677 let guest2 = Guest::new(Box::new(focal2)); 1678 let mut child2 = GuestCommand::new(&guest2) 1679 .args(&["--cpus", "boot=1"]) 1680 .args(&["--memory", format!("size=512M,{}", memory_param).as_str()]) 1681 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 1682 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1683 .default_disks() 1684 .args(&["--net", guest2.default_net_string().as_str()]) 1685 .args(&["--serial", "tty", "--console", "off"]) 1686 .capture_output() 1687 .spawn() 1688 .unwrap(); 1689 1690 let r = std::panic::catch_unwind(|| { 1691 guest2.wait_vm_boot(None).unwrap(); 1692 let ksm_ps_guest2 = get_ksm_pages_shared(); 1693 1694 if mergeable { 1695 println!( 1696 "ksm pages_shared after vm1 booted '{}', ksm pages_shared after vm2 booted '{}'", 1697 ksm_ps_guest1, ksm_ps_guest2 1698 ); 1699 // We are expecting the number of shared pages to increase as the number of VM increases 1700 assert!(ksm_ps_guest1 < ksm_ps_guest2); 1701 } else { 1702 assert!(ksm_ps_guest1 == 0); 1703 assert!(ksm_ps_guest2 == 0); 1704 } 1705 }); 1706 1707 let _ = child1.kill(); 1708 let _ = child2.kill(); 1709 1710 let output = child1.wait_with_output().unwrap(); 1711 child2.wait().unwrap(); 1712 1713 handle_child_output(r, &output); 1714 } 1715 1716 fn _get_vmm_overhead(pid: u32, guest_memory_size: u32) -> HashMap<String, u32> { 1717 let smaps = fs::File::open(format!("/proc/{}/smaps", pid)).unwrap(); 1718 let reader = io::BufReader::new(smaps); 1719 1720 let mut skip_map: bool = false; 1721 let mut region_name: String = "".to_string(); 1722 let mut region_maps = HashMap::new(); 1723 for line in reader.lines() { 1724 let l = line.unwrap(); 1725 1726 if l.contains('-') { 1727 let values: Vec<&str> = l.split_whitespace().collect(); 1728 region_name = values.last().unwrap().trim().to_string(); 1729 if region_name == "0" { 1730 region_name = "anonymous".to_string() 1731 } 1732 } 1733 1734 // Each section begins with something that looks like: 1735 // Size: 2184 kB 1736 if l.starts_with("Size:") { 1737 let values: Vec<&str> = l.split_whitespace().collect(); 1738 let map_size = values[1].parse::<u32>().unwrap(); 1739 // We skip the assigned guest RAM map, its RSS is only 1740 // dependent on the guest actual memory usage. 1741 // Everything else can be added to the VMM overhead. 1742 skip_map = map_size >= guest_memory_size; 1743 continue; 1744 } 1745 1746 // If this is a map we're taking into account, then we only 1747 // count the RSS. The sum of all counted RSS is the VMM overhead. 1748 if !skip_map && l.starts_with("Rss:") { 1749 let values: Vec<&str> = l.split_whitespace().collect(); 1750 let value = values[1].trim().parse::<u32>().unwrap(); 1751 *region_maps.entry(region_name.clone()).or_insert(0) += value; 1752 } 1753 } 1754 1755 region_maps 1756 } 1757 1758 fn get_vmm_overhead(pid: u32, guest_memory_size: u32) -> u32 { 1759 let mut total = 0; 1760 1761 for (region_name, value) in &_get_vmm_overhead(pid, guest_memory_size) { 1762 eprintln!("{}: {}", region_name, value); 1763 total += value; 1764 } 1765 1766 total 1767 } 1768 1769 // 10MB is our maximum accepted overhead. 1770 const MAXIMUM_VMM_OVERHEAD_KB: u32 = 10 * 1024; 1771 1772 #[derive(PartialEq, PartialOrd)] 1773 struct Counters { 1774 rx_bytes: u64, 1775 rx_frames: u64, 1776 tx_bytes: u64, 1777 tx_frames: u64, 1778 read_bytes: u64, 1779 write_bytes: u64, 1780 read_ops: u64, 1781 write_ops: u64, 1782 } 1783 1784 fn get_counters(api_socket: &str) -> Counters { 1785 // Get counters 1786 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "counters", None); 1787 assert!(cmd_success); 1788 1789 let counters: HashMap<&str, HashMap<&str, u64>> = 1790 serde_json::from_slice(&cmd_output).unwrap_or_default(); 1791 1792 let rx_bytes = *counters.get("_net2").unwrap().get("rx_bytes").unwrap(); 1793 let rx_frames = *counters.get("_net2").unwrap().get("rx_frames").unwrap(); 1794 let tx_bytes = *counters.get("_net2").unwrap().get("tx_bytes").unwrap(); 1795 let tx_frames = *counters.get("_net2").unwrap().get("tx_frames").unwrap(); 1796 1797 let read_bytes = *counters.get("_disk0").unwrap().get("read_bytes").unwrap(); 1798 let write_bytes = *counters.get("_disk0").unwrap().get("write_bytes").unwrap(); 1799 let read_ops = *counters.get("_disk0").unwrap().get("read_ops").unwrap(); 1800 let write_ops = *counters.get("_disk0").unwrap().get("write_ops").unwrap(); 1801 1802 Counters { 1803 rx_bytes, 1804 rx_frames, 1805 tx_bytes, 1806 tx_frames, 1807 read_bytes, 1808 write_bytes, 1809 read_ops, 1810 write_ops, 1811 } 1812 } 1813 1814 fn pty_read(mut pty: std::fs::File) -> Receiver<String> { 1815 let (tx, rx) = mpsc::channel::<String>(); 1816 thread::spawn(move || loop { 1817 thread::sleep(std::time::Duration::new(1, 0)); 1818 let mut buf = [0; 512]; 1819 match pty.read(&mut buf) { 1820 Ok(_) => { 1821 let output = std::str::from_utf8(&buf).unwrap().to_string(); 1822 match tx.send(output) { 1823 Ok(_) => (), 1824 Err(_) => break, 1825 } 1826 } 1827 Err(_) => break, 1828 } 1829 }); 1830 rx 1831 } 1832 1833 fn get_pty_path(api_socket: &str, pty_type: &str) -> PathBuf { 1834 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "info", None); 1835 assert!(cmd_success); 1836 let info: serde_json::Value = serde_json::from_slice(&cmd_output).unwrap_or_default(); 1837 assert_eq!("Pty", info["config"][pty_type]["mode"]); 1838 PathBuf::from( 1839 info["config"][pty_type]["file"] 1840 .as_str() 1841 .expect("Missing pty path"), 1842 ) 1843 } 1844 1845 // VFIO test network setup. 1846 // We reserve a different IP class for it: 172.18.0.0/24. 1847 fn setup_vfio_network_interfaces() { 1848 // 'vfio-br0' 1849 assert!(exec_host_command_status("sudo ip link add name vfio-br0 type bridge").success()); 1850 assert!(exec_host_command_status("sudo ip link set vfio-br0 up").success()); 1851 assert!(exec_host_command_status("sudo ip addr add 172.18.0.1/24 dev vfio-br0").success()); 1852 // 'vfio-tap0' 1853 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap0 mode tap").success()); 1854 assert!(exec_host_command_status("sudo ip link set vfio-tap0 master vfio-br0").success()); 1855 assert!(exec_host_command_status("sudo ip link set vfio-tap0 up").success()); 1856 // 'vfio-tap1' 1857 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap1 mode tap").success()); 1858 assert!(exec_host_command_status("sudo ip link set vfio-tap1 master vfio-br0").success()); 1859 assert!(exec_host_command_status("sudo ip link set vfio-tap1 up").success()); 1860 // 'vfio-tap2' 1861 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap2 mode tap").success()); 1862 assert!(exec_host_command_status("sudo ip link set vfio-tap2 master vfio-br0").success()); 1863 assert!(exec_host_command_status("sudo ip link set vfio-tap2 up").success()); 1864 // 'vfio-tap3' 1865 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap3 mode tap").success()); 1866 assert!(exec_host_command_status("sudo ip link set vfio-tap3 master vfio-br0").success()); 1867 assert!(exec_host_command_status("sudo ip link set vfio-tap3 up").success()); 1868 } 1869 1870 // Tear VFIO test network down 1871 fn cleanup_vfio_network_interfaces() { 1872 assert!(exec_host_command_status("sudo ip link del vfio-br0").success()); 1873 assert!(exec_host_command_status("sudo ip link del vfio-tap0").success()); 1874 assert!(exec_host_command_status("sudo ip link del vfio-tap1").success()); 1875 assert!(exec_host_command_status("sudo ip link del vfio-tap2").success()); 1876 assert!(exec_host_command_status("sudo ip link del vfio-tap3").success()); 1877 } 1878 1879 fn balloon_size(api_socket: &str) -> u64 { 1880 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "info", None); 1881 assert!(cmd_success); 1882 1883 let info: serde_json::Value = serde_json::from_slice(&cmd_output).unwrap_or_default(); 1884 let total_mem = &info["config"]["memory"]["size"] 1885 .to_string() 1886 .parse::<u64>() 1887 .unwrap(); 1888 let actual_mem = &info["memory_actual_size"] 1889 .to_string() 1890 .parse::<u64>() 1891 .unwrap(); 1892 total_mem - actual_mem 1893 } 1894 1895 mod parallel { 1896 use crate::tests::*; 1897 1898 #[test] 1899 #[cfg(target_arch = "x86_64")] 1900 fn test_simple_launch() { 1901 let bionic = UbuntuDiskConfig::new(BIONIC_IMAGE_NAME.to_string()); 1902 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1903 1904 vec![Box::new(bionic), Box::new(focal)] 1905 .drain(..) 1906 .for_each(|disk_config| { 1907 let guest = Guest::new(disk_config); 1908 1909 let mut child = GuestCommand::new(&guest) 1910 .args(&["--cpus", "boot=1"]) 1911 .args(&["--memory", "size=512M"]) 1912 .args(&["--kernel", guest.fw_path.as_str()]) 1913 .default_disks() 1914 .default_net() 1915 .args(&["--serial", "tty", "--console", "off"]) 1916 .capture_output() 1917 .spawn() 1918 .unwrap(); 1919 1920 let r = std::panic::catch_unwind(|| { 1921 guest.wait_vm_boot(Some(120)).unwrap(); 1922 1923 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 1924 assert_eq!(guest.get_initial_apicid().unwrap_or(1), 0); 1925 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 1926 assert!(guest.get_entropy().unwrap_or_default() >= 900); 1927 assert_eq!(guest.get_pci_bridge_class().unwrap_or_default(), "0x060000"); 1928 }); 1929 1930 let _ = child.kill(); 1931 let output = child.wait_with_output().unwrap(); 1932 1933 handle_child_output(r, &output); 1934 }); 1935 } 1936 1937 #[test] 1938 #[cfg(all(target_arch = "aarch64", feature = "acpi"))] 1939 fn test_edk2_acpi_launch() { 1940 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1941 1942 vec![Box::new(focal)].drain(..).for_each(|disk_config| { 1943 let guest = Guest::new(disk_config); 1944 1945 let mut child = GuestCommand::new(&guest) 1946 .args(&["--cpus", "boot=1"]) 1947 .args(&["--memory", "size=512M"]) 1948 .args(&["--kernel", edk2_path().to_str().unwrap()]) 1949 .default_disks() 1950 .default_net() 1951 .args(&["--serial", "tty", "--console", "off"]) 1952 .capture_output() 1953 .spawn() 1954 .unwrap(); 1955 1956 let r = std::panic::catch_unwind(|| { 1957 guest.wait_vm_boot(Some(120)).unwrap(); 1958 1959 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 1960 assert!(guest.get_total_memory().unwrap_or_default() > 400_000); 1961 assert!(guest.get_entropy().unwrap_or_default() >= 900); 1962 assert_eq!(guest.get_pci_bridge_class().unwrap_or_default(), "0x060000"); 1963 }); 1964 1965 let _ = child.kill(); 1966 let output = child.wait_with_output().unwrap(); 1967 1968 handle_child_output(r, &output); 1969 }); 1970 } 1971 1972 #[test] 1973 fn test_multi_cpu() { 1974 let bionic = UbuntuDiskConfig::new(BIONIC_IMAGE_NAME.to_string()); 1975 let guest = Guest::new(Box::new(bionic)); 1976 let mut cmd = GuestCommand::new(&guest); 1977 cmd.args(&["--cpus", "boot=2,max=4"]) 1978 .args(&["--memory", "size=512M"]) 1979 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 1980 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1981 .capture_output() 1982 .default_disks() 1983 .default_net(); 1984 1985 let mut child = cmd.spawn().unwrap(); 1986 1987 let r = std::panic::catch_unwind(|| { 1988 guest.wait_vm_boot(Some(120)).unwrap(); 1989 1990 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 1991 1992 #[cfg(target_arch = "x86_64")] 1993 assert_eq!( 1994 guest 1995 .ssh_command( 1996 r#"dmesg | grep "smpboot: Allowing" | sed "s/\[\ *[0-9.]*\] //""# 1997 ) 1998 .unwrap() 1999 .trim(), 2000 "smpboot: Allowing 4 CPUs, 2 hotplug CPUs" 2001 ); 2002 #[cfg(target_arch = "aarch64")] 2003 assert_eq!( 2004 guest 2005 .ssh_command( 2006 r#"dmesg | grep "smp: Brought up" | sed "s/\[\ *[0-9.]*\] //""# 2007 ) 2008 .unwrap() 2009 .trim(), 2010 "smp: Brought up 1 node, 2 CPUs" 2011 ); 2012 }); 2013 2014 let _ = child.kill(); 2015 let output = child.wait_with_output().unwrap(); 2016 2017 handle_child_output(r, &output); 2018 } 2019 2020 #[test] 2021 fn test_cpu_topology_421() { 2022 test_cpu_topology(4, 2, 1, false); 2023 } 2024 2025 #[test] 2026 fn test_cpu_topology_142() { 2027 test_cpu_topology(1, 4, 2, false); 2028 } 2029 2030 #[test] 2031 fn test_cpu_topology_262() { 2032 test_cpu_topology(2, 6, 2, false); 2033 } 2034 2035 #[test] 2036 #[cfg(all(target_arch = "aarch64", feature = "acpi"))] 2037 fn test_cpu_topology_421_fw() { 2038 test_cpu_topology(4, 2, 1, true); 2039 } 2040 2041 #[test] 2042 #[cfg(all(target_arch = "aarch64", feature = "acpi"))] 2043 fn test_cpu_topology_142_fw() { 2044 test_cpu_topology(1, 4, 2, true); 2045 } 2046 2047 #[test] 2048 #[cfg(all(target_arch = "aarch64", feature = "acpi"))] 2049 fn test_cpu_topology_262_fw() { 2050 test_cpu_topology(2, 6, 2, true); 2051 } 2052 2053 #[test] 2054 #[cfg(target_arch = "x86_64")] 2055 fn test_cpu_physical_bits() { 2056 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2057 let guest = Guest::new(Box::new(focal)); 2058 let max_phys_bits: u8 = 36; 2059 let mut child = GuestCommand::new(&guest) 2060 .args(&["--cpus", &format!("max_phys_bits={}", max_phys_bits)]) 2061 .args(&["--memory", "size=512M"]) 2062 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2063 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2064 .default_disks() 2065 .default_net() 2066 .capture_output() 2067 .spawn() 2068 .unwrap(); 2069 2070 let r = std::panic::catch_unwind(|| { 2071 guest.wait_vm_boot(None).unwrap(); 2072 2073 assert!( 2074 guest 2075 .ssh_command("lscpu | grep \"Address sizes:\" | cut -f 2 -d \":\" | sed \"s# *##\" | cut -f 1 -d \" \"") 2076 .unwrap() 2077 .trim() 2078 .parse::<u8>() 2079 .unwrap_or(max_phys_bits + 1) <= max_phys_bits, 2080 ); 2081 }); 2082 2083 let _ = child.kill(); 2084 let output = child.wait_with_output().unwrap(); 2085 2086 handle_child_output(r, &output); 2087 } 2088 2089 #[test] 2090 fn test_large_vm() { 2091 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2092 let guest = Guest::new(Box::new(focal)); 2093 let mut cmd = GuestCommand::new(&guest); 2094 cmd.args(&["--cpus", "boot=48"]) 2095 .args(&["--memory", "size=5120M"]) 2096 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2097 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2098 .args(&["--serial", "tty"]) 2099 .args(&["--console", "off"]) 2100 .capture_output() 2101 .default_disks() 2102 .default_net(); 2103 2104 let mut child = cmd.spawn().unwrap(); 2105 2106 guest.wait_vm_boot(None).unwrap(); 2107 2108 let r = std::panic::catch_unwind(|| { 2109 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 48); 2110 assert_eq!( 2111 guest 2112 .ssh_command( 2113 "lscpu | grep \"On-line\" | cut -f 2 -d \":\" | sed \"s# *##\"" 2114 ) 2115 .unwrap() 2116 .trim(), 2117 "0-47" 2118 ); 2119 2120 assert!(guest.get_total_memory().unwrap_or_default() > 5_000_000); 2121 }); 2122 2123 let _ = child.kill(); 2124 let output = child.wait_with_output().unwrap(); 2125 2126 handle_child_output(r, &output); 2127 } 2128 2129 #[test] 2130 fn test_huge_memory() { 2131 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2132 let guest = Guest::new(Box::new(focal)); 2133 let mut cmd = GuestCommand::new(&guest); 2134 cmd.args(&["--cpus", "boot=1"]) 2135 .args(&["--memory", "size=128G"]) 2136 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2137 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2138 .capture_output() 2139 .default_disks() 2140 .default_net(); 2141 2142 let mut child = cmd.spawn().unwrap(); 2143 2144 guest.wait_vm_boot(Some(120)).unwrap(); 2145 2146 let r = std::panic::catch_unwind(|| { 2147 assert!(guest.get_total_memory().unwrap_or_default() > 128_000_000); 2148 }); 2149 2150 let _ = child.kill(); 2151 let output = child.wait_with_output().unwrap(); 2152 2153 handle_child_output(r, &output); 2154 } 2155 2156 #[test] 2157 fn test_power_button() { 2158 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2159 let guest = Guest::new(Box::new(focal)); 2160 let mut cmd = GuestCommand::new(&guest); 2161 let api_socket = temp_api_path(&guest.tmp_dir); 2162 2163 cmd.args(&["--cpus", "boot=1"]) 2164 .args(&["--memory", "size=512M"]) 2165 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2166 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2167 .capture_output() 2168 .default_disks() 2169 .default_net() 2170 .args(&["--api-socket", &api_socket]); 2171 2172 let child = cmd.spawn().unwrap(); 2173 2174 let r = std::panic::catch_unwind(|| { 2175 guest.wait_vm_boot(None).unwrap(); 2176 assert!(remote_command(&api_socket, "power-button", None)); 2177 }); 2178 2179 let output = child.wait_with_output().unwrap(); 2180 assert!(output.status.success()); 2181 handle_child_output(r, &output); 2182 } 2183 2184 #[test] 2185 #[cfg(target_arch = "x86_64")] 2186 fn test_user_defined_memory_regions() { 2187 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2188 let guest = Guest::new(Box::new(focal)); 2189 let api_socket = temp_api_path(&guest.tmp_dir); 2190 2191 let kernel_path = direct_kernel_boot_path(); 2192 2193 let mut child = GuestCommand::new(&guest) 2194 .args(&["--cpus", "boot=1"]) 2195 .args(&["--memory", "size=0,hotplug_method=virtio-mem"]) 2196 .args(&[ 2197 "--memory-zone", 2198 "id=mem0,size=1G,hotplug_size=2G", 2199 "id=mem1,size=1G,file=/dev/shm", 2200 "id=mem2,size=1G,host_numa_node=0,hotplug_size=2G", 2201 ]) 2202 .args(&["--kernel", kernel_path.to_str().unwrap()]) 2203 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2204 .args(&["--api-socket", &api_socket]) 2205 .capture_output() 2206 .default_disks() 2207 .default_net() 2208 .spawn() 2209 .unwrap(); 2210 2211 let r = std::panic::catch_unwind(|| { 2212 guest.wait_vm_boot(None).unwrap(); 2213 2214 assert!(guest.get_total_memory().unwrap_or_default() > 2_880_000); 2215 2216 guest.enable_memory_hotplug(); 2217 2218 resize_zone_command(&api_socket, "mem0", "3G"); 2219 thread::sleep(std::time::Duration::new(5, 0)); 2220 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2221 resize_zone_command(&api_socket, "mem2", "3G"); 2222 thread::sleep(std::time::Duration::new(5, 0)); 2223 assert!(guest.get_total_memory().unwrap_or_default() > 6_720_000); 2224 resize_zone_command(&api_socket, "mem0", "2G"); 2225 thread::sleep(std::time::Duration::new(5, 0)); 2226 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 2227 resize_zone_command(&api_socket, "mem2", "2G"); 2228 thread::sleep(std::time::Duration::new(5, 0)); 2229 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2230 2231 guest.reboot_linux(0, None); 2232 2233 // Check the amount of RAM after reboot 2234 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2235 assert!(guest.get_total_memory().unwrap_or_default() < 5_760_000); 2236 2237 // Check if we can still resize down to the initial 'boot'size 2238 resize_zone_command(&api_socket, "mem0", "1G"); 2239 thread::sleep(std::time::Duration::new(5, 0)); 2240 assert!(guest.get_total_memory().unwrap_or_default() < 4_800_000); 2241 resize_zone_command(&api_socket, "mem2", "1G"); 2242 thread::sleep(std::time::Duration::new(5, 0)); 2243 assert!(guest.get_total_memory().unwrap_or_default() < 3_840_000); 2244 }); 2245 2246 let _ = child.kill(); 2247 let output = child.wait_with_output().unwrap(); 2248 2249 handle_child_output(r, &output); 2250 } 2251 2252 #[test] 2253 #[cfg(feature = "acpi")] 2254 fn test_guest_numa_nodes() { 2255 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2256 let guest = Guest::new(Box::new(focal)); 2257 let api_socket = temp_api_path(&guest.tmp_dir); 2258 2259 #[cfg(target_arch = "x86_64")] 2260 let kernel_path = direct_kernel_boot_path(); 2261 #[cfg(target_arch = "aarch64")] 2262 let kernel_path = edk2_path(); 2263 2264 let mut child = GuestCommand::new(&guest) 2265 .args(&["--cpus", "boot=6,max=12"]) 2266 .args(&["--memory", "size=0,hotplug_method=virtio-mem"]) 2267 .args(&[ 2268 "--memory-zone", 2269 "id=mem0,size=1G,hotplug_size=3G", 2270 "id=mem1,size=2G,hotplug_size=3G", 2271 "id=mem2,size=3G,hotplug_size=3G", 2272 ]) 2273 .args(&[ 2274 "--numa", 2275 "guest_numa_id=0,cpus=0-2:9,distances=1@15:2@20,memory_zones=mem0", 2276 "guest_numa_id=1,cpus=3-4:6-8,distances=0@20:2@25,memory_zones=mem1", 2277 "guest_numa_id=2,cpus=5:10-11,distances=0@25:1@30,memory_zones=mem2", 2278 ]) 2279 .args(&["--kernel", kernel_path.to_str().unwrap()]) 2280 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2281 .args(&["--api-socket", &api_socket]) 2282 .capture_output() 2283 .default_disks() 2284 .default_net() 2285 .spawn() 2286 .unwrap(); 2287 2288 let r = std::panic::catch_unwind(|| { 2289 guest.wait_vm_boot(None).unwrap(); 2290 2291 // Check each NUMA node has been assigned the right amount of 2292 // memory. 2293 assert!(guest.get_numa_node_memory(0).unwrap_or_default() > 960_000); 2294 assert!(guest.get_numa_node_memory(1).unwrap_or_default() > 1_920_000); 2295 assert!(guest.get_numa_node_memory(2).unwrap_or_default() > 2_880_000); 2296 2297 // Check each NUMA node has been assigned the right CPUs set. 2298 guest.check_numa_node_cpus(0, vec![0, 1, 2]).unwrap(); 2299 guest.check_numa_node_cpus(1, vec![3, 4]).unwrap(); 2300 guest.check_numa_node_cpus(2, vec![5]).unwrap(); 2301 2302 // Check each NUMA node has been assigned the right distances. 2303 assert!(guest.check_numa_node_distances(0, "10 15 20").unwrap()); 2304 assert!(guest.check_numa_node_distances(1, "20 10 25").unwrap()); 2305 assert!(guest.check_numa_node_distances(2, "25 30 10").unwrap()); 2306 2307 // AArch64 currently does not support hotplug, and therefore we only 2308 // test hotplug-related function on x86_64 here. 2309 #[cfg(target_arch = "x86_64")] 2310 { 2311 guest.enable_memory_hotplug(); 2312 2313 // Resize every memory zone and check each associated NUMA node 2314 // has been assigned the right amount of memory. 2315 resize_zone_command(&api_socket, "mem0", "4G"); 2316 thread::sleep(std::time::Duration::new(5, 0)); 2317 assert!(guest.get_numa_node_memory(0).unwrap_or_default() > 3_840_000); 2318 resize_zone_command(&api_socket, "mem1", "4G"); 2319 thread::sleep(std::time::Duration::new(5, 0)); 2320 assert!(guest.get_numa_node_memory(1).unwrap_or_default() > 3_840_000); 2321 resize_zone_command(&api_socket, "mem2", "4G"); 2322 thread::sleep(std::time::Duration::new(5, 0)); 2323 assert!(guest.get_numa_node_memory(2).unwrap_or_default() > 3_840_000); 2324 2325 // Resize to the maximum amount of CPUs and check each NUMA 2326 // node has been assigned the right CPUs set. 2327 resize_command(&api_socket, Some(12), None, None); 2328 guest.check_numa_node_cpus(0, vec![0, 1, 2, 9]).unwrap(); 2329 guest.check_numa_node_cpus(1, vec![3, 4, 6, 7, 8]).unwrap(); 2330 guest.check_numa_node_cpus(2, vec![5, 10, 11]).unwrap(); 2331 } 2332 }); 2333 2334 let _ = child.kill(); 2335 let output = child.wait_with_output().unwrap(); 2336 2337 handle_child_output(r, &output); 2338 } 2339 2340 #[test] 2341 fn test_pci_msi() { 2342 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2343 let guest = Guest::new(Box::new(focal)); 2344 let mut cmd = GuestCommand::new(&guest); 2345 cmd.args(&["--cpus", "boot=1"]) 2346 .args(&["--memory", "size=512M"]) 2347 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2348 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2349 .capture_output() 2350 .default_disks() 2351 .default_net(); 2352 2353 let mut child = cmd.spawn().unwrap(); 2354 2355 guest.wait_vm_boot(None).unwrap(); 2356 2357 #[cfg(target_arch = "x86_64")] 2358 let grep_cmd = "grep -c PCI-MSI /proc/interrupts"; 2359 #[cfg(target_arch = "aarch64")] 2360 let grep_cmd = "grep -c ITS-MSI /proc/interrupts"; 2361 2362 let r = std::panic::catch_unwind(|| { 2363 assert_eq!( 2364 guest 2365 .ssh_command(grep_cmd) 2366 .unwrap() 2367 .trim() 2368 .parse::<u32>() 2369 .unwrap_or_default(), 2370 12 2371 ); 2372 }); 2373 2374 let _ = child.kill(); 2375 let output = child.wait_with_output().unwrap(); 2376 2377 handle_child_output(r, &output); 2378 } 2379 2380 #[test] 2381 fn test_direct_kernel_boot() { 2382 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2383 let guest = Guest::new(Box::new(focal)); 2384 2385 let kernel_path = direct_kernel_boot_path(); 2386 2387 let mut child = GuestCommand::new(&guest) 2388 .args(&["--cpus", "boot=1"]) 2389 .args(&["--memory", "size=512M"]) 2390 .args(&["--kernel", kernel_path.to_str().unwrap()]) 2391 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2392 .default_disks() 2393 .default_net() 2394 .capture_output() 2395 .spawn() 2396 .unwrap(); 2397 2398 let r = std::panic::catch_unwind(|| { 2399 guest.wait_vm_boot(None).unwrap(); 2400 2401 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 2402 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 2403 assert!(guest.get_entropy().unwrap_or_default() >= 900); 2404 2405 let grep_cmd = if cfg!(target_arch = "x86_64") { 2406 "grep -c PCI-MSI /proc/interrupts" 2407 } else { 2408 "grep -c ITS-MSI /proc/interrupts" 2409 }; 2410 assert_eq!( 2411 guest 2412 .ssh_command(grep_cmd) 2413 .unwrap() 2414 .trim() 2415 .parse::<u32>() 2416 .unwrap_or_default(), 2417 12 2418 ); 2419 }); 2420 2421 let _ = child.kill(); 2422 let output = child.wait_with_output().unwrap(); 2423 2424 handle_child_output(r, &output); 2425 } 2426 2427 fn _test_virtio_block(image_name: &str, disable_io_uring: bool) { 2428 let focal = UbuntuDiskConfig::new(image_name.to_string()); 2429 let guest = Guest::new(Box::new(focal)); 2430 2431 let mut workload_path = dirs::home_dir().unwrap(); 2432 workload_path.push("workloads"); 2433 2434 let mut blk_file_path = workload_path; 2435 blk_file_path.push("blk.img"); 2436 2437 let kernel_path = direct_kernel_boot_path(); 2438 2439 let mut cloud_child = GuestCommand::new(&guest) 2440 .args(&["--cpus", "boot=4"]) 2441 .args(&["--memory", "size=512M,shared=on"]) 2442 .args(&["--kernel", kernel_path.to_str().unwrap()]) 2443 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2444 .args(&[ 2445 "--disk", 2446 format!( 2447 "path={}", 2448 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 2449 ) 2450 .as_str(), 2451 format!( 2452 "path={}", 2453 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2454 ) 2455 .as_str(), 2456 format!( 2457 "path={},readonly=on,direct=on,num_queues=4,_disable_io_uring={}", 2458 blk_file_path.to_str().unwrap(), 2459 disable_io_uring 2460 ) 2461 .as_str(), 2462 ]) 2463 .default_net() 2464 .capture_output() 2465 .spawn() 2466 .unwrap(); 2467 2468 let r = std::panic::catch_unwind(|| { 2469 guest.wait_vm_boot(None).unwrap(); 2470 2471 // Check both if /dev/vdc exists and if the block size is 16M. 2472 assert_eq!( 2473 guest 2474 .ssh_command("lsblk | grep vdc | grep -c 16M") 2475 .unwrap() 2476 .trim() 2477 .parse::<u32>() 2478 .unwrap_or_default(), 2479 1 2480 ); 2481 2482 // Check both if /dev/vdc exists and if this block is RO. 2483 assert_eq!( 2484 guest 2485 .ssh_command("lsblk | grep vdc | awk '{print $5}'") 2486 .unwrap() 2487 .trim() 2488 .parse::<u32>() 2489 .unwrap_or_default(), 2490 1 2491 ); 2492 2493 // Check if the number of queues is 4. 2494 assert_eq!( 2495 guest 2496 .ssh_command("ls -ll /sys/block/vdc/mq | grep ^d | wc -l") 2497 .unwrap() 2498 .trim() 2499 .parse::<u32>() 2500 .unwrap_or_default(), 2501 4 2502 ); 2503 }); 2504 2505 let _ = cloud_child.kill(); 2506 let output = cloud_child.wait_with_output().unwrap(); 2507 2508 handle_child_output(r, &output); 2509 } 2510 2511 #[test] 2512 fn test_virtio_block() { 2513 _test_virtio_block(FOCAL_IMAGE_NAME, false) 2514 } 2515 2516 #[test] 2517 fn test_virtio_block_disable_io_uring() { 2518 _test_virtio_block(FOCAL_IMAGE_NAME, true) 2519 } 2520 2521 #[test] 2522 fn test_virtio_block_qcow2() { 2523 _test_virtio_block(FOCAL_IMAGE_NAME_QCOW2, false) 2524 } 2525 2526 #[test] 2527 fn test_virtio_block_vhd() { 2528 let mut workload_path = dirs::home_dir().unwrap(); 2529 workload_path.push("workloads"); 2530 2531 let mut raw_file_path = workload_path.clone(); 2532 let mut vhd_file_path = workload_path; 2533 raw_file_path.push(FOCAL_IMAGE_NAME); 2534 vhd_file_path.push(FOCAL_IMAGE_NAME_VHD); 2535 2536 // Generate VHD file from RAW file 2537 std::process::Command::new("qemu-img") 2538 .arg("convert") 2539 .arg("-p") 2540 .args(&["-f", "raw"]) 2541 .args(&["-O", "vpc"]) 2542 .args(&["-o", "subformat=fixed"]) 2543 .arg(raw_file_path.to_str().unwrap()) 2544 .arg(vhd_file_path.to_str().unwrap()) 2545 .output() 2546 .expect("Expect generating VHD image from RAW image"); 2547 2548 _test_virtio_block(FOCAL_IMAGE_NAME_VHD, false) 2549 } 2550 2551 #[test] 2552 fn test_vhost_user_net_default() { 2553 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, false, false) 2554 } 2555 2556 #[test] 2557 fn test_vhost_user_net_named_tap() { 2558 test_vhost_user_net( 2559 Some("mytap0"), 2560 2, 2561 &prepare_vhost_user_net_daemon, 2562 false, 2563 false, 2564 ) 2565 } 2566 2567 #[test] 2568 fn test_vhost_user_net_existing_tap() { 2569 test_vhost_user_net( 2570 Some("vunet-tap0"), 2571 2, 2572 &prepare_vhost_user_net_daemon, 2573 false, 2574 false, 2575 ) 2576 } 2577 2578 #[test] 2579 fn test_vhost_user_net_multiple_queues() { 2580 test_vhost_user_net(None, 4, &prepare_vhost_user_net_daemon, false, false) 2581 } 2582 2583 #[test] 2584 fn test_vhost_user_net_tap_multiple_queues() { 2585 test_vhost_user_net( 2586 Some("vunet-tap1"), 2587 4, 2588 &prepare_vhost_user_net_daemon, 2589 false, 2590 false, 2591 ) 2592 } 2593 2594 #[test] 2595 fn test_vhost_user_net_host_mac() { 2596 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, true, false) 2597 } 2598 2599 #[test] 2600 fn test_vhost_user_net_client_mode() { 2601 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, false, true) 2602 } 2603 2604 #[test] 2605 fn test_vhost_user_blk_default() { 2606 test_vhost_user_blk(2, false, false, Some(&prepare_vubd)) 2607 } 2608 2609 #[test] 2610 fn test_vhost_user_blk_readonly() { 2611 test_vhost_user_blk(1, true, false, Some(&prepare_vubd)) 2612 } 2613 2614 #[test] 2615 fn test_vhost_user_blk_direct() { 2616 test_vhost_user_blk(1, false, true, Some(&prepare_vubd)) 2617 } 2618 2619 #[test] 2620 fn test_boot_from_vhost_user_blk_default() { 2621 test_boot_from_vhost_user_blk(1, false, false, Some(&prepare_vubd)) 2622 } 2623 2624 #[test] 2625 #[cfg(target_arch = "x86_64")] 2626 fn test_split_irqchip() { 2627 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2628 let guest = Guest::new(Box::new(focal)); 2629 2630 let mut child = GuestCommand::new(&guest) 2631 .args(&["--cpus", "boot=1"]) 2632 .args(&["--memory", "size=512M"]) 2633 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2634 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2635 .default_disks() 2636 .default_net() 2637 .capture_output() 2638 .spawn() 2639 .unwrap(); 2640 2641 let r = std::panic::catch_unwind(|| { 2642 guest.wait_vm_boot(None).unwrap(); 2643 2644 assert_eq!( 2645 guest 2646 .ssh_command("grep -c IO-APIC.*timer /proc/interrupts || true") 2647 .unwrap() 2648 .trim() 2649 .parse::<u32>() 2650 .unwrap_or(1), 2651 0 2652 ); 2653 assert_eq!( 2654 guest 2655 .ssh_command("grep -c IO-APIC.*cascade /proc/interrupts || true") 2656 .unwrap() 2657 .trim() 2658 .parse::<u32>() 2659 .unwrap_or(1), 2660 0 2661 ); 2662 }); 2663 2664 let _ = child.kill(); 2665 let output = child.wait_with_output().unwrap(); 2666 2667 handle_child_output(r, &output); 2668 } 2669 2670 #[test] 2671 fn test_virtio_fs_dax_on_default_cache_size() { 2672 test_virtio_fs(true, None, "none", &prepare_virtiofsd, false) 2673 } 2674 2675 #[test] 2676 fn test_virtio_fs_dax_on_cache_size_1_gib() { 2677 test_virtio_fs(true, Some(0x4000_0000), "none", &prepare_virtiofsd, false) 2678 } 2679 2680 #[test] 2681 fn test_virtio_fs_dax_off() { 2682 test_virtio_fs(false, None, "none", &prepare_virtiofsd, false) 2683 } 2684 2685 #[test] 2686 fn test_virtio_fs_dax_on_default_cache_size_w_virtiofsd_rs_daemon() { 2687 test_virtio_fs(true, None, "none", &prepare_virtofsd_rs_daemon, false) 2688 } 2689 2690 #[test] 2691 fn test_virtio_fs_dax_on_cache_size_1_gib_w_virtiofsd_rs_daemon() { 2692 test_virtio_fs( 2693 true, 2694 Some(0x4000_0000), 2695 "none", 2696 &prepare_virtofsd_rs_daemon, 2697 false, 2698 ) 2699 } 2700 2701 #[test] 2702 fn test_virtio_fs_dax_off_w_virtiofsd_rs_daemon() { 2703 test_virtio_fs(false, None, "none", &prepare_virtofsd_rs_daemon, false) 2704 } 2705 2706 #[test] 2707 #[cfg(target_arch = "x86_64")] 2708 fn test_virtio_fs_hotplug_dax_on() { 2709 test_virtio_fs(true, None, "none", &prepare_virtiofsd, true) 2710 } 2711 2712 #[test] 2713 #[cfg(target_arch = "x86_64")] 2714 fn test_virtio_fs_hotplug_dax_off() { 2715 test_virtio_fs(false, None, "none", &prepare_virtiofsd, true) 2716 } 2717 2718 #[test] 2719 #[cfg(target_arch = "x86_64")] 2720 fn test_virtio_fs_hotplug_dax_on_w_virtiofsd_rs_daemon() { 2721 test_virtio_fs(true, None, "none", &prepare_virtofsd_rs_daemon, true) 2722 } 2723 2724 #[test] 2725 #[cfg(target_arch = "x86_64")] 2726 fn test_virtio_fs_hotplug_dax_off_w_virtiofsd_rs_daemon() { 2727 test_virtio_fs(false, None, "none", &prepare_virtofsd_rs_daemon, true) 2728 } 2729 2730 #[test] 2731 #[cfg(target_arch = "x86_64")] 2732 fn test_virtio_pmem_persist_writes() { 2733 test_virtio_pmem(false, false) 2734 } 2735 2736 #[test] 2737 #[cfg(target_arch = "x86_64")] 2738 fn test_virtio_pmem_discard_writes() { 2739 test_virtio_pmem(true, false) 2740 } 2741 2742 #[test] 2743 #[cfg(target_arch = "x86_64")] 2744 fn test_virtio_pmem_with_size() { 2745 test_virtio_pmem(true, true) 2746 } 2747 2748 #[test] 2749 fn test_boot_from_virtio_pmem() { 2750 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2751 let guest = Guest::new(Box::new(focal)); 2752 2753 let kernel_path = direct_kernel_boot_path(); 2754 2755 let mut child = GuestCommand::new(&guest) 2756 .args(&["--cpus", "boot=1"]) 2757 .args(&["--memory", "size=512M"]) 2758 .args(&["--kernel", kernel_path.to_str().unwrap()]) 2759 .args(&[ 2760 "--disk", 2761 format!( 2762 "path={}", 2763 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2764 ) 2765 .as_str(), 2766 ]) 2767 .default_net() 2768 .args(&[ 2769 "--pmem", 2770 format!( 2771 "file={},size={}", 2772 guest.disk_config.disk(DiskType::OperatingSystem).unwrap(), 2773 fs::metadata(&guest.disk_config.disk(DiskType::OperatingSystem).unwrap()) 2774 .unwrap() 2775 .len() 2776 ) 2777 .as_str(), 2778 ]) 2779 .args(&[ 2780 "--cmdline", 2781 DIRECT_KERNEL_BOOT_CMDLINE 2782 .replace("vda1", "pmem0p1") 2783 .as_str(), 2784 ]) 2785 .capture_output() 2786 .spawn() 2787 .unwrap(); 2788 2789 let r = std::panic::catch_unwind(|| { 2790 guest.wait_vm_boot(None).unwrap(); 2791 2792 // Simple checks to validate the VM booted properly 2793 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 2794 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 2795 }); 2796 2797 let _ = child.kill(); 2798 let output = child.wait_with_output().unwrap(); 2799 2800 handle_child_output(r, &output); 2801 } 2802 2803 #[test] 2804 fn test_multiple_network_interfaces() { 2805 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2806 let guest = Guest::new(Box::new(focal)); 2807 2808 let kernel_path = direct_kernel_boot_path(); 2809 2810 let mut child = GuestCommand::new(&guest) 2811 .args(&["--cpus", "boot=1"]) 2812 .args(&["--memory", "size=512M"]) 2813 .args(&["--kernel", kernel_path.to_str().unwrap()]) 2814 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2815 .default_disks() 2816 .args(&[ 2817 "--net", 2818 guest.default_net_string().as_str(), 2819 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 2820 "tap=mytap1,mac=fe:1f:9e:e1:60:f2,ip=192.168.4.1,mask=255.255.255.0", 2821 ]) 2822 .capture_output() 2823 .spawn() 2824 .unwrap(); 2825 2826 let r = std::panic::catch_unwind(|| { 2827 guest.wait_vm_boot(None).unwrap(); 2828 2829 let tap_count = exec_host_command_output("ip link | grep -c mytap1"); 2830 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 2831 2832 // 3 network interfaces + default localhost ==> 4 interfaces 2833 assert_eq!( 2834 guest 2835 .ssh_command("ip -o link | wc -l") 2836 .unwrap() 2837 .trim() 2838 .parse::<u32>() 2839 .unwrap_or_default(), 2840 4 2841 ); 2842 }); 2843 2844 let _ = child.kill(); 2845 let output = child.wait_with_output().unwrap(); 2846 2847 handle_child_output(r, &output); 2848 } 2849 2850 #[test] 2851 fn test_serial_off() { 2852 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2853 let guest = Guest::new(Box::new(focal)); 2854 let mut child = GuestCommand::new(&guest) 2855 .args(&["--cpus", "boot=1"]) 2856 .args(&["--memory", "size=512M"]) 2857 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2858 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2859 .default_disks() 2860 .default_net() 2861 .args(&["--serial", "off"]) 2862 .capture_output() 2863 .spawn() 2864 .unwrap(); 2865 2866 let r = std::panic::catch_unwind(|| { 2867 guest.wait_vm_boot(None).unwrap(); 2868 2869 // Test that there is no ttyS0 2870 assert_eq!( 2871 guest 2872 .ssh_command(GREP_SERIAL_IRQ_CMD) 2873 .unwrap() 2874 .trim() 2875 .parse::<u32>() 2876 .unwrap_or(1), 2877 0 2878 ); 2879 }); 2880 2881 let _ = child.kill(); 2882 let output = child.wait_with_output().unwrap(); 2883 2884 handle_child_output(r, &output); 2885 } 2886 2887 #[test] 2888 fn test_serial_null() { 2889 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2890 let guest = Guest::new(Box::new(focal)); 2891 let mut cmd = GuestCommand::new(&guest); 2892 #[cfg(target_arch = "x86_64")] 2893 let console_str: &str = "console=ttyS0"; 2894 #[cfg(target_arch = "aarch64")] 2895 let console_str: &str = "console=ttyAMA0"; 2896 2897 cmd.args(&["--cpus", "boot=1"]) 2898 .args(&["--memory", "size=512M"]) 2899 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2900 .args(&[ 2901 "--cmdline", 2902 DIRECT_KERNEL_BOOT_CMDLINE 2903 .replace("console=hvc0 ", console_str) 2904 .as_str(), 2905 ]) 2906 .default_disks() 2907 .default_net() 2908 .args(&["--serial", "null"]) 2909 .args(&["--console", "off"]) 2910 .capture_output(); 2911 2912 let mut child = cmd.spawn().unwrap(); 2913 2914 let r = std::panic::catch_unwind(|| { 2915 guest.wait_vm_boot(None).unwrap(); 2916 2917 // Test that there is a ttyS0 2918 assert_eq!( 2919 guest 2920 .ssh_command(GREP_SERIAL_IRQ_CMD) 2921 .unwrap() 2922 .trim() 2923 .parse::<u32>() 2924 .unwrap_or_default(), 2925 1 2926 ); 2927 }); 2928 2929 let _ = child.kill(); 2930 let output = child.wait_with_output().unwrap(); 2931 handle_child_output(r, &output); 2932 2933 let r = std::panic::catch_unwind(|| { 2934 assert!(!String::from_utf8_lossy(&output.stdout).contains(CONSOLE_TEST_STRING)); 2935 }); 2936 2937 handle_child_output(r, &output); 2938 } 2939 2940 #[test] 2941 fn test_serial_tty() { 2942 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2943 let guest = Guest::new(Box::new(focal)); 2944 2945 let kernel_path = direct_kernel_boot_path(); 2946 2947 #[cfg(target_arch = "x86_64")] 2948 let console_str: &str = "console=ttyS0"; 2949 #[cfg(target_arch = "aarch64")] 2950 let console_str: &str = "console=ttyAMA0"; 2951 2952 let mut child = GuestCommand::new(&guest) 2953 .args(&["--cpus", "boot=1"]) 2954 .args(&["--memory", "size=512M"]) 2955 .args(&["--kernel", kernel_path.to_str().unwrap()]) 2956 .args(&[ 2957 "--cmdline", 2958 DIRECT_KERNEL_BOOT_CMDLINE 2959 .replace("console=hvc0 ", console_str) 2960 .as_str(), 2961 ]) 2962 .default_disks() 2963 .default_net() 2964 .args(&["--serial", "tty"]) 2965 .args(&["--console", "off"]) 2966 .capture_output() 2967 .spawn() 2968 .unwrap(); 2969 2970 let r = std::panic::catch_unwind(|| { 2971 guest.wait_vm_boot(None).unwrap(); 2972 2973 // Test that there is a ttyS0 2974 assert_eq!( 2975 guest 2976 .ssh_command(GREP_SERIAL_IRQ_CMD) 2977 .unwrap() 2978 .trim() 2979 .parse::<u32>() 2980 .unwrap_or_default(), 2981 1 2982 ); 2983 }); 2984 2985 // This sleep is needed to wait for the login prompt 2986 thread::sleep(std::time::Duration::new(2, 0)); 2987 2988 let _ = child.kill(); 2989 let output = child.wait_with_output().unwrap(); 2990 handle_child_output(r, &output); 2991 2992 let r = std::panic::catch_unwind(|| { 2993 assert!(String::from_utf8_lossy(&output.stdout).contains(CONSOLE_TEST_STRING)); 2994 }); 2995 2996 handle_child_output(r, &output); 2997 } 2998 2999 #[test] 3000 fn test_serial_file() { 3001 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3002 let guest = Guest::new(Box::new(focal)); 3003 3004 let serial_path = guest.tmp_dir.as_path().join("/tmp/serial-output"); 3005 #[cfg(target_arch = "x86_64")] 3006 let console_str: &str = "console=ttyS0"; 3007 #[cfg(target_arch = "aarch64")] 3008 let console_str: &str = "console=ttyAMA0"; 3009 3010 let mut child = GuestCommand::new(&guest) 3011 .args(&["--cpus", "boot=1"]) 3012 .args(&["--memory", "size=512M"]) 3013 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3014 .args(&[ 3015 "--cmdline", 3016 DIRECT_KERNEL_BOOT_CMDLINE 3017 .replace("console=hvc0 ", console_str) 3018 .as_str(), 3019 ]) 3020 .default_disks() 3021 .default_net() 3022 .args(&[ 3023 "--serial", 3024 format!("file={}", serial_path.to_str().unwrap()).as_str(), 3025 ]) 3026 .capture_output() 3027 .spawn() 3028 .unwrap(); 3029 3030 let r = std::panic::catch_unwind(|| { 3031 guest.wait_vm_boot(None).unwrap(); 3032 3033 // Test that there is a ttyS0 3034 assert_eq!( 3035 guest 3036 .ssh_command(GREP_SERIAL_IRQ_CMD) 3037 .unwrap() 3038 .trim() 3039 .parse::<u32>() 3040 .unwrap_or_default(), 3041 1 3042 ); 3043 3044 guest.ssh_command("sudo shutdown -h now").unwrap(); 3045 }); 3046 3047 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 3048 let _ = child.kill(); 3049 let output = child.wait_with_output().unwrap(); 3050 handle_child_output(r, &output); 3051 3052 let r = std::panic::catch_unwind(|| { 3053 // Check that the cloud-hypervisor binary actually terminated 3054 assert!(output.status.success()); 3055 3056 // Do this check after shutdown of the VM as an easy way to ensure 3057 // all writes are flushed to disk 3058 let mut f = std::fs::File::open(serial_path).unwrap(); 3059 let mut buf = String::new(); 3060 f.read_to_string(&mut buf).unwrap(); 3061 assert!(buf.contains(CONSOLE_TEST_STRING)); 3062 }); 3063 3064 handle_child_output(r, &output); 3065 } 3066 3067 #[test] 3068 fn test_pty_interaction() { 3069 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3070 let guest = Guest::new(Box::new(focal)); 3071 let api_socket = temp_api_path(&guest.tmp_dir); 3072 let serial_option = if cfg!(target_arch = "x86_64") { 3073 " console=ttyS0" 3074 } else { 3075 " console=ttyAMA0" 3076 }; 3077 let cmdline = DIRECT_KERNEL_BOOT_CMDLINE.to_owned() + serial_option; 3078 3079 let mut child = GuestCommand::new(&guest) 3080 .args(&["--cpus", "boot=1"]) 3081 .args(&["--memory", "size=512M"]) 3082 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3083 .args(&["--cmdline", &cmdline]) 3084 .default_disks() 3085 .default_net() 3086 .args(&["--serial", "null"]) 3087 .args(&["--console", "pty"]) 3088 .args(&["--api-socket", &api_socket]) 3089 .spawn() 3090 .unwrap(); 3091 3092 let r = std::panic::catch_unwind(|| { 3093 guest.wait_vm_boot(None).unwrap(); 3094 // Get pty fd for console 3095 let console_path = get_pty_path(&api_socket, "console"); 3096 // TODO: Get serial pty test working 3097 let mut cf = std::fs::OpenOptions::new() 3098 .write(true) 3099 .read(true) 3100 .open(console_path) 3101 .unwrap(); 3102 3103 // Some dumb sleeps but we don't want to write 3104 // before the console is up and we don't want 3105 // to try and write the next line before the 3106 // login process is ready. 3107 thread::sleep(std::time::Duration::new(5, 0)); 3108 assert_eq!(cf.write(b"cloud\n").unwrap(), 6); 3109 thread::sleep(std::time::Duration::new(2, 0)); 3110 assert_eq!(cf.write(b"cloud123\n").unwrap(), 9); 3111 thread::sleep(std::time::Duration::new(2, 0)); 3112 assert_eq!(cf.write(b"echo test_pty_console\n").unwrap(), 22); 3113 thread::sleep(std::time::Duration::new(2, 0)); 3114 3115 // read pty and ensure they have a login shell 3116 // some fairly hacky workarounds to avoid looping 3117 // forever in case the channel is blocked getting output 3118 let ptyc = pty_read(cf); 3119 let mut empty = 0; 3120 let mut prev = String::new(); 3121 loop { 3122 thread::sleep(std::time::Duration::new(2, 0)); 3123 match ptyc.try_recv() { 3124 Ok(line) => { 3125 empty = 0; 3126 prev = prev + &line; 3127 if prev.contains("test_pty_console") { 3128 break; 3129 } 3130 } 3131 Err(mpsc::TryRecvError::Empty) => { 3132 empty += 1; 3133 if empty > 5 { 3134 panic!("No login on pty"); 3135 } 3136 } 3137 _ => panic!("No login on pty"), 3138 } 3139 } 3140 3141 guest.ssh_command("sudo shutdown -h now").unwrap(); 3142 }); 3143 3144 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 3145 let _ = child.kill(); 3146 let output = child.wait_with_output().unwrap(); 3147 handle_child_output(r, &output); 3148 3149 let r = std::panic::catch_unwind(|| { 3150 // Check that the cloud-hypervisor binary actually terminated 3151 assert!(output.status.success()) 3152 }); 3153 handle_child_output(r, &output); 3154 } 3155 3156 #[test] 3157 fn test_virtio_console() { 3158 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3159 let guest = Guest::new(Box::new(focal)); 3160 3161 let kernel_path = direct_kernel_boot_path(); 3162 3163 let mut child = GuestCommand::new(&guest) 3164 .args(&["--cpus", "boot=1"]) 3165 .args(&["--memory", "size=512M"]) 3166 .args(&["--kernel", kernel_path.to_str().unwrap()]) 3167 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3168 .default_disks() 3169 .default_net() 3170 .args(&["--console", "tty"]) 3171 .args(&["--serial", "null"]) 3172 .capture_output() 3173 .spawn() 3174 .unwrap(); 3175 3176 let text = String::from("On a branch floating down river a cricket, singing."); 3177 let cmd = format!("echo {} | sudo tee /dev/hvc0", text); 3178 3179 let r = std::panic::catch_unwind(|| { 3180 guest.wait_vm_boot(None).unwrap(); 3181 3182 #[cfg(feature = "acpi")] 3183 assert!(guest 3184 .does_device_vendor_pair_match("0x1043", "0x1af4") 3185 .unwrap_or_default()); 3186 3187 guest.ssh_command(&cmd).unwrap(); 3188 }); 3189 3190 let _ = child.kill(); 3191 let output = child.wait_with_output().unwrap(); 3192 handle_child_output(r, &output); 3193 3194 let r = std::panic::catch_unwind(|| { 3195 assert!(String::from_utf8_lossy(&output.stdout).contains(&text)); 3196 }); 3197 3198 handle_child_output(r, &output); 3199 } 3200 3201 #[test] 3202 fn test_console_file() { 3203 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3204 let guest = Guest::new(Box::new(focal)); 3205 3206 let console_path = guest.tmp_dir.as_path().join("/tmp/console-output"); 3207 let mut child = GuestCommand::new(&guest) 3208 .args(&["--cpus", "boot=1"]) 3209 .args(&["--memory", "size=512M"]) 3210 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3211 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3212 .default_disks() 3213 .default_net() 3214 .args(&[ 3215 "--console", 3216 format!("file={}", console_path.to_str().unwrap()).as_str(), 3217 ]) 3218 .capture_output() 3219 .spawn() 3220 .unwrap(); 3221 3222 guest.wait_vm_boot(None).unwrap(); 3223 3224 guest.ssh_command("sudo shutdown -h now").unwrap(); 3225 3226 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 3227 let _ = child.kill(); 3228 let output = child.wait_with_output().unwrap(); 3229 3230 let r = std::panic::catch_unwind(|| { 3231 // Check that the cloud-hypervisor binary actually terminated 3232 assert!(output.status.success()); 3233 3234 // Do this check after shutdown of the VM as an easy way to ensure 3235 // all writes are flushed to disk 3236 let mut f = std::fs::File::open(console_path).unwrap(); 3237 let mut buf = String::new(); 3238 f.read_to_string(&mut buf).unwrap(); 3239 3240 if !buf.contains(CONSOLE_TEST_STRING) { 3241 eprintln!( 3242 "\n\n==== Console file output ====\n\n{}\n\n==== End console file output ====", 3243 buf 3244 ); 3245 } 3246 assert!(buf.contains(CONSOLE_TEST_STRING)); 3247 }); 3248 3249 handle_child_output(r, &output); 3250 } 3251 3252 #[test] 3253 #[cfg(target_arch = "x86_64")] 3254 // The VFIO integration test starts cloud-hypervisor guest with 3 TAP 3255 // backed networking interfaces, bound through a simple bridge on the host. 3256 // So if the nested cloud-hypervisor succeeds in getting a directly 3257 // assigned interface from its cloud-hypervisor host, we should be able to 3258 // ssh into it, and verify that it's running with the right kernel command 3259 // line (We tag the command line from cloud-hypervisor for that purpose). 3260 // The third device is added to validate that hotplug works correctly since 3261 // it is being added to the L2 VM through hotplugging mechanism. 3262 // Also, we pass-through a vitio-blk device to the L2 VM to test the 32-bit 3263 // vfio device support 3264 fn test_vfio() { 3265 setup_vfio_network_interfaces(); 3266 3267 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3268 let guest = Guest::new_from_ip_range(Box::new(focal), "172.18", 0); 3269 3270 let mut workload_path = dirs::home_dir().unwrap(); 3271 workload_path.push("workloads"); 3272 3273 let kernel_path = direct_kernel_boot_path(); 3274 3275 let mut vfio_path = workload_path.clone(); 3276 vfio_path.push("vfio"); 3277 3278 let mut cloud_init_vfio_base_path = vfio_path.clone(); 3279 cloud_init_vfio_base_path.push("cloudinit.img"); 3280 3281 // We copy our cloudinit into the vfio mount point, for the nested 3282 // cloud-hypervisor guest to use. 3283 rate_limited_copy( 3284 &guest.disk_config.disk(DiskType::CloudInit).unwrap(), 3285 &cloud_init_vfio_base_path, 3286 ) 3287 .expect("copying of cloud-init disk failed"); 3288 3289 let mut vfio_disk_path = workload_path.clone(); 3290 vfio_disk_path.push("vfio.img"); 3291 3292 // Create the vfio disk image 3293 let output = Command::new("mkfs.ext4") 3294 .arg("-d") 3295 .arg(vfio_path.to_str().unwrap()) 3296 .arg(vfio_disk_path.to_str().unwrap()) 3297 .arg("2g") 3298 .output() 3299 .unwrap(); 3300 if !output.status.success() { 3301 eprintln!("{}", String::from_utf8_lossy(&output.stderr)); 3302 panic!("mkfs.ext4 command generated an error"); 3303 } 3304 3305 let mut blk_file_path = workload_path; 3306 blk_file_path.push("blk.img"); 3307 3308 let vfio_tap0 = "vfio-tap0"; 3309 let vfio_tap1 = "vfio-tap1"; 3310 let vfio_tap2 = "vfio-tap2"; 3311 let vfio_tap3 = "vfio-tap3"; 3312 3313 let mut child = GuestCommand::new(&guest) 3314 .args(&["--cpus", "boot=4"]) 3315 .args(&["--memory", "size=2G,hugepages=on,shared=on"]) 3316 .args(&["--kernel", kernel_path.to_str().unwrap()]) 3317 .args(&[ 3318 "--disk", 3319 format!( 3320 "path={}", 3321 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 3322 ) 3323 .as_str(), 3324 format!( 3325 "path={}", 3326 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3327 ) 3328 .as_str(), 3329 format!("path={}", vfio_disk_path.to_str().unwrap()).as_str(), 3330 format!("path={},iommu=on", blk_file_path.to_str().unwrap()).as_str(), 3331 ]) 3332 .args(&[ 3333 "--cmdline", 3334 format!( 3335 "{} kvm-intel.nested=1 vfio_iommu_type1.allow_unsafe_interrupts", 3336 DIRECT_KERNEL_BOOT_CMDLINE 3337 ) 3338 .as_str(), 3339 ]) 3340 .args(&[ 3341 "--net", 3342 format!("tap={},mac={}", vfio_tap0, guest.network.guest_mac).as_str(), 3343 format!( 3344 "tap={},mac={},iommu=on", 3345 vfio_tap1, guest.network.l2_guest_mac1 3346 ) 3347 .as_str(), 3348 format!( 3349 "tap={},mac={},iommu=on", 3350 vfio_tap2, guest.network.l2_guest_mac2 3351 ) 3352 .as_str(), 3353 format!( 3354 "tap={},mac={},iommu=on", 3355 vfio_tap3, guest.network.l2_guest_mac3 3356 ) 3357 .as_str(), 3358 ]) 3359 .capture_output() 3360 .spawn() 3361 .unwrap(); 3362 3363 thread::sleep(std::time::Duration::new(30, 0)); 3364 3365 let r = std::panic::catch_unwind(|| { 3366 guest.ssh_command_l1("sudo systemctl start vfio").unwrap(); 3367 thread::sleep(std::time::Duration::new(120, 0)); 3368 3369 // We booted our cloud hypervisor L2 guest with a "VFIOTAG" tag 3370 // added to its kernel command line. 3371 // Let's ssh into it and verify that it's there. If it is it means 3372 // we're in the right guest (The L2 one) because the QEMU L1 guest 3373 // does not have this command line tag. 3374 assert_eq!( 3375 guest 3376 .ssh_command_l2_1("grep -c VFIOTAG /proc/cmdline") 3377 .unwrap() 3378 .trim() 3379 .parse::<u32>() 3380 .unwrap_or_default(), 3381 1 3382 ); 3383 3384 // Let's also verify from the second virtio-net device passed to 3385 // the L2 VM. 3386 assert_eq!( 3387 guest 3388 .ssh_command_l2_2("grep -c VFIOTAG /proc/cmdline") 3389 .unwrap() 3390 .trim() 3391 .parse::<u32>() 3392 .unwrap_or_default(), 3393 1 3394 ); 3395 3396 // Check the amount of PCI devices appearing in L2 VM. 3397 assert_eq!( 3398 guest 3399 .ssh_command_l2_1("ls /sys/bus/pci/devices | wc -l") 3400 .unwrap() 3401 .trim() 3402 .parse::<u32>() 3403 .unwrap_or_default(), 3404 8, 3405 ); 3406 3407 // Check both if /dev/vdc exists and if the block size is 16M in L2 VM 3408 assert_eq!( 3409 guest 3410 .ssh_command_l2_1("lsblk | grep vdc | grep -c 16M") 3411 .unwrap() 3412 .trim() 3413 .parse::<u32>() 3414 .unwrap_or_default(), 3415 1 3416 ); 3417 3418 // Hotplug an extra virtio-net device through L2 VM. 3419 guest.ssh_command_l1( 3420 "echo 0000:00:09.0 | sudo tee /sys/bus/pci/devices/0000:00:09.0/driver/unbind", 3421 ).unwrap(); 3422 guest 3423 .ssh_command_l1( 3424 "echo 0000:00:09.0 | sudo tee /sys/bus/pci/drivers/vfio-pci/bind", 3425 ) 3426 .unwrap(); 3427 let vfio_hotplug_output = guest 3428 .ssh_command_l1( 3429 "sudo /mnt/ch-remote \ 3430 --api-socket=/tmp/ch_api.sock \ 3431 add-device path=/sys/bus/pci/devices/0000:00:09.0,id=vfio123", 3432 ) 3433 .unwrap(); 3434 assert!( 3435 vfio_hotplug_output.contains("{\"id\":\"vfio123\",\"bdf\":\"0000:00:08.0\"}") 3436 ); 3437 3438 thread::sleep(std::time::Duration::new(10, 0)); 3439 3440 // Let's also verify from the third virtio-net device passed to 3441 // the L2 VM. This third device has been hotplugged through the L2 3442 // VM, so this is our way to validate hotplug works for VFIO PCI. 3443 assert_eq!( 3444 guest 3445 .ssh_command_l2_3("grep -c VFIOTAG /proc/cmdline") 3446 .unwrap() 3447 .trim() 3448 .parse::<u32>() 3449 .unwrap_or_default(), 3450 1 3451 ); 3452 3453 // Check the amount of PCI devices appearing in L2 VM. 3454 // There should be one more device than before, raising the count 3455 // up to 9 PCI devices. 3456 assert_eq!( 3457 guest 3458 .ssh_command_l2_1("ls /sys/bus/pci/devices | wc -l") 3459 .unwrap() 3460 .trim() 3461 .parse::<u32>() 3462 .unwrap_or_default(), 3463 9, 3464 ); 3465 3466 // Let's now verify that we can correctly remove the virtio-net 3467 // device through the "remove-device" command responsible for 3468 // unplugging VFIO devices. 3469 guest 3470 .ssh_command_l1( 3471 "sudo /mnt/ch-remote \ 3472 --api-socket=/tmp/ch_api.sock \ 3473 remove-device vfio123", 3474 ) 3475 .unwrap(); 3476 thread::sleep(std::time::Duration::new(10, 0)); 3477 3478 // Check the amount of PCI devices appearing in L2 VM is back down 3479 // to 8 devices. 3480 assert_eq!( 3481 guest 3482 .ssh_command_l2_1("ls /sys/bus/pci/devices | wc -l") 3483 .unwrap() 3484 .trim() 3485 .parse::<u32>() 3486 .unwrap_or_default(), 3487 8, 3488 ); 3489 3490 // Perform memory hotplug in L2 and validate the memory is showing 3491 // up as expected. In order to check, we will use the virtio-net 3492 // device already passed through L2 as a VFIO device, this will 3493 // verify that VFIO devices are functional with memory hotplug. 3494 assert!(guest.get_total_memory_l2().unwrap_or_default() > 480_000); 3495 guest.ssh_command_l2_1( 3496 "sudo bash -c 'echo online > /sys/devices/system/memory/auto_online_blocks'", 3497 ).unwrap(); 3498 guest 3499 .ssh_command_l1( 3500 "sudo /mnt/ch-remote \ 3501 --api-socket=/tmp/ch_api.sock \ 3502 resize --memory=1073741824", 3503 ) 3504 .unwrap(); 3505 assert!(guest.get_total_memory_l2().unwrap_or_default() > 960_000); 3506 }); 3507 3508 let _ = child.kill(); 3509 let output = child.wait_with_output().unwrap(); 3510 3511 cleanup_vfio_network_interfaces(); 3512 3513 handle_child_output(r, &output); 3514 } 3515 3516 #[test] 3517 fn test_direct_kernel_boot_noacpi() { 3518 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3519 let guest = Guest::new(Box::new(focal)); 3520 3521 let kernel_path = direct_kernel_boot_path(); 3522 3523 let mut child = GuestCommand::new(&guest) 3524 .args(&["--cpus", "boot=1"]) 3525 .args(&["--memory", "size=512M"]) 3526 .args(&["--kernel", kernel_path.to_str().unwrap()]) 3527 .args(&[ 3528 "--cmdline", 3529 format!("{} acpi=off", DIRECT_KERNEL_BOOT_CMDLINE).as_str(), 3530 ]) 3531 .default_disks() 3532 .default_net() 3533 .capture_output() 3534 .spawn() 3535 .unwrap(); 3536 3537 let r = std::panic::catch_unwind(|| { 3538 guest.wait_vm_boot(None).unwrap(); 3539 3540 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 3541 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 3542 assert!(guest.get_entropy().unwrap_or_default() >= 900); 3543 }); 3544 3545 let _ = child.kill(); 3546 let output = child.wait_with_output().unwrap(); 3547 3548 handle_child_output(r, &output); 3549 } 3550 3551 #[test] 3552 fn test_reboot() { 3553 let bionic = UbuntuDiskConfig::new(BIONIC_IMAGE_NAME.to_string()); 3554 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3555 3556 vec![Box::new(bionic), Box::new(focal)] 3557 .drain(..) 3558 .for_each(|disk_config| { 3559 let guest = Guest::new(disk_config); 3560 3561 let mut cmd = GuestCommand::new(&guest); 3562 cmd.args(&["--cpus", "boot=1"]) 3563 .args(&["--memory", "size=512M"]) 3564 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3565 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3566 .default_disks() 3567 .default_net() 3568 .capture_output(); 3569 3570 let mut child = cmd.spawn().unwrap(); 3571 3572 let r = std::panic::catch_unwind(|| { 3573 guest.wait_vm_boot(Some(120)).unwrap(); 3574 3575 let fd_count_1 = get_fd_count(child.id()); 3576 guest.reboot_linux(0, Some(120)); 3577 let fd_count_2 = get_fd_count(child.id()); 3578 assert_eq!(fd_count_1, fd_count_2); 3579 3580 guest.ssh_command("sudo shutdown -h now").unwrap(); 3581 }); 3582 3583 let _ = child.wait_timeout(std::time::Duration::from_secs(40)); 3584 let _ = child.kill(); 3585 let output = child.wait_with_output().unwrap(); 3586 handle_child_output(r, &output); 3587 3588 let r = std::panic::catch_unwind(|| { 3589 // Check that the cloud-hypervisor binary actually terminated 3590 assert!(output.status.success()); 3591 }); 3592 3593 handle_child_output(r, &output); 3594 }); 3595 } 3596 3597 #[test] 3598 fn test_virtio_vsock() { 3599 _test_virtio_vsock(false) 3600 } 3601 3602 #[test] 3603 #[cfg(target_arch = "x86_64")] 3604 fn test_virtio_vsock_hotplug() { 3605 _test_virtio_vsock(true); 3606 } 3607 3608 #[test] 3609 // Start cloud-hypervisor with no VM parameters, only the API server running. 3610 // From the API: Create a VM, boot it and check that it looks as expected. 3611 fn test_api_create_boot() { 3612 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3613 let guest = Guest::new(Box::new(focal)); 3614 3615 let api_socket = temp_api_path(&guest.tmp_dir); 3616 3617 let mut child = GuestCommand::new(&guest) 3618 .args(&["--api-socket", &api_socket]) 3619 .capture_output() 3620 .spawn() 3621 .unwrap(); 3622 3623 thread::sleep(std::time::Duration::new(1, 0)); 3624 3625 // Verify API server is running 3626 curl_command(&api_socket, "GET", "http://localhost/api/v1/vmm.ping", None); 3627 3628 // Create the VM first 3629 let cpu_count: u8 = 4; 3630 let http_body = guest.api_create_body(cpu_count); 3631 curl_command( 3632 &api_socket, 3633 "PUT", 3634 "http://localhost/api/v1/vm.create", 3635 Some(&http_body), 3636 ); 3637 3638 // Then boot it 3639 curl_command(&api_socket, "PUT", "http://localhost/api/v1/vm.boot", None); 3640 thread::sleep(std::time::Duration::new(20, 0)); 3641 3642 let r = std::panic::catch_unwind(|| { 3643 // Check that the VM booted as expected 3644 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 3645 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 3646 assert!(guest.get_entropy().unwrap_or_default() >= 900); 3647 }); 3648 3649 let _ = child.kill(); 3650 let output = child.wait_with_output().unwrap(); 3651 3652 handle_child_output(r, &output); 3653 } 3654 3655 #[test] 3656 // Start cloud-hypervisor with no VM parameters, only the API server running. 3657 // From the API: Create a VM, boot it and check that it looks as expected. 3658 // Then we pause the VM, check that it's no longer available. 3659 // Finally we resume the VM and check that it's available. 3660 fn test_api_pause_resume() { 3661 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3662 let guest = Guest::new(Box::new(focal)); 3663 3664 let api_socket = temp_api_path(&guest.tmp_dir); 3665 3666 let mut child = GuestCommand::new(&guest) 3667 .args(&["--api-socket", &api_socket]) 3668 .capture_output() 3669 .spawn() 3670 .unwrap(); 3671 3672 thread::sleep(std::time::Duration::new(1, 0)); 3673 3674 // Verify API server is running 3675 curl_command(&api_socket, "GET", "http://localhost/api/v1/vmm.ping", None); 3676 3677 // Create the VM first 3678 let cpu_count: u8 = 4; 3679 let http_body = guest.api_create_body(cpu_count); 3680 curl_command( 3681 &api_socket, 3682 "PUT", 3683 "http://localhost/api/v1/vm.create", 3684 Some(&http_body), 3685 ); 3686 3687 // Then boot it 3688 curl_command(&api_socket, "PUT", "http://localhost/api/v1/vm.boot", None); 3689 thread::sleep(std::time::Duration::new(20, 0)); 3690 3691 let r = std::panic::catch_unwind(|| { 3692 // Check that the VM booted as expected 3693 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 3694 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 3695 assert!(guest.get_entropy().unwrap_or_default() >= 900); 3696 3697 // We now pause the VM 3698 assert!(remote_command(&api_socket, "pause", None)); 3699 3700 // Check pausing again fails 3701 assert!(!remote_command(&api_socket, "pause", None)); 3702 3703 thread::sleep(std::time::Duration::new(2, 0)); 3704 3705 // SSH into the VM should fail 3706 assert!(ssh_command_ip( 3707 "grep -c processor /proc/cpuinfo", 3708 &guest.network.guest_ip, 3709 2, 3710 5 3711 ) 3712 .is_err()); 3713 3714 // Resume the VM 3715 assert!(remote_command(&api_socket, "resume", None)); 3716 3717 // Check resuming again fails 3718 assert!(!remote_command(&api_socket, "resume", None)); 3719 3720 thread::sleep(std::time::Duration::new(2, 0)); 3721 3722 // Now we should be able to SSH back in and get the right number of CPUs 3723 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 3724 }); 3725 3726 let _ = child.kill(); 3727 let output = child.wait_with_output().unwrap(); 3728 3729 handle_child_output(r, &output); 3730 } 3731 3732 #[test] 3733 #[cfg(target_arch = "x86_64")] 3734 // This test validates that it can find the virtio-iommu device at first. 3735 // It also verifies that both disks and the network card are attached to 3736 // the virtual IOMMU by looking at /sys/kernel/iommu_groups directory. 3737 // The last interesting part of this test is that it exercises the network 3738 // interface attached to the virtual IOMMU since this is the one used to 3739 // send all commands through SSH. 3740 fn test_virtio_iommu() { 3741 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3742 let guest = Guest::new(Box::new(focal)); 3743 3744 let kernel_path = direct_kernel_boot_path(); 3745 3746 let mut child = GuestCommand::new(&guest) 3747 .args(&["--cpus", "boot=1"]) 3748 .args(&["--memory", "size=512M"]) 3749 .args(&["--kernel", kernel_path.to_str().unwrap()]) 3750 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3751 .args(&[ 3752 "--disk", 3753 format!( 3754 "path={},iommu=on", 3755 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 3756 ) 3757 .as_str(), 3758 format!( 3759 "path={},iommu=on", 3760 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3761 ) 3762 .as_str(), 3763 ]) 3764 .args(&["--net", guest.default_net_string_w_iommu().as_str()]) 3765 .capture_output() 3766 .spawn() 3767 .unwrap(); 3768 3769 let r = std::panic::catch_unwind(|| { 3770 guest.wait_vm_boot(None).unwrap(); 3771 3772 // Verify the virtio-iommu device is present. 3773 assert!(guest 3774 .does_device_vendor_pair_match("0x1057", "0x1af4") 3775 .unwrap_or_default()); 3776 3777 // Verify the first disk is located under IOMMU group 0. 3778 assert_eq!( 3779 guest 3780 .ssh_command("ls /sys/kernel/iommu_groups/0/devices") 3781 .unwrap() 3782 .trim(), 3783 "0000:00:02.0" 3784 ); 3785 3786 // Verify the second disk is located under IOMMU group 1. 3787 assert_eq!( 3788 guest 3789 .ssh_command("ls /sys/kernel/iommu_groups/1/devices") 3790 .unwrap() 3791 .trim(), 3792 "0000:00:03.0" 3793 ); 3794 3795 // Verify the network card is located under IOMMU group 2. 3796 assert_eq!( 3797 guest 3798 .ssh_command("ls /sys/kernel/iommu_groups/2/devices") 3799 .unwrap() 3800 .trim(), 3801 "0000:00:04.0" 3802 ); 3803 }); 3804 3805 let _ = child.kill(); 3806 let output = child.wait_with_output().unwrap(); 3807 3808 handle_child_output(r, &output); 3809 } 3810 3811 #[test] 3812 #[cfg(target_arch = "x86_64")] 3813 // We cannot force the software running in the guest to reprogram the BAR 3814 // with some different addresses, but we have a reliable way of testing it 3815 // with a standard Linux kernel. 3816 // By removing a device from the PCI tree, and then rescanning the tree, 3817 // Linux consistently chooses to reorganize the PCI device BARs to other 3818 // locations in the guest address space. 3819 // This test creates a dedicated PCI network device to be checked as being 3820 // properly probed first, then removing it, and adding it again by doing a 3821 // rescan. 3822 fn test_pci_bar_reprogramming() { 3823 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3824 let guest = Guest::new(Box::new(focal)); 3825 let mut child = GuestCommand::new(&guest) 3826 .args(&["--cpus", "boot=1"]) 3827 .args(&["--memory", "size=512M"]) 3828 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3829 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3830 .default_disks() 3831 .args(&[ 3832 "--net", 3833 guest.default_net_string().as_str(), 3834 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 3835 ]) 3836 .capture_output() 3837 .spawn() 3838 .unwrap(); 3839 3840 let r = 3841 std::panic::catch_unwind(|| { 3842 guest.wait_vm_boot(None).unwrap(); 3843 3844 // 2 network interfaces + default localhost ==> 3 interfaces 3845 assert_eq!( 3846 guest 3847 .ssh_command("ip -o link | wc -l") 3848 .unwrap() 3849 .trim() 3850 .parse::<u32>() 3851 .unwrap_or_default(), 3852 3 3853 ); 3854 3855 let init_bar_addr = guest.ssh_command( 3856 "sudo awk '{print $1; exit}' /sys/bus/pci/devices/0000:00:05.0/resource", 3857 ).unwrap(); 3858 3859 // Remove the PCI device 3860 guest 3861 .ssh_command("echo 1 | sudo tee /sys/bus/pci/devices/0000:00:05.0/remove") 3862 .unwrap(); 3863 3864 // Only 1 network interface left + default localhost ==> 2 interfaces 3865 assert_eq!( 3866 guest 3867 .ssh_command("ip -o link | wc -l") 3868 .unwrap() 3869 .trim() 3870 .parse::<u32>() 3871 .unwrap_or_default(), 3872 2 3873 ); 3874 3875 // Remove the PCI device 3876 guest 3877 .ssh_command("echo 1 | sudo tee /sys/bus/pci/rescan") 3878 .unwrap(); 3879 3880 // Back to 2 network interface + default localhost ==> 3 interfaces 3881 assert_eq!( 3882 guest 3883 .ssh_command("ip -o link | wc -l") 3884 .unwrap() 3885 .trim() 3886 .parse::<u32>() 3887 .unwrap_or_default(), 3888 3 3889 ); 3890 3891 let new_bar_addr = guest.ssh_command( 3892 "sudo awk '{print $1; exit}' /sys/bus/pci/devices/0000:00:05.0/resource", 3893 ).unwrap(); 3894 3895 // Let's compare the BAR addresses for our virtio-net device. 3896 // They should be different as we expect the BAR reprogramming 3897 // to have happened. 3898 assert_ne!(init_bar_addr, new_bar_addr); 3899 }); 3900 3901 let _ = child.kill(); 3902 let output = child.wait_with_output().unwrap(); 3903 3904 handle_child_output(r, &output); 3905 } 3906 3907 #[test] 3908 fn test_memory_mergeable_off() { 3909 test_memory_mergeable(false) 3910 } 3911 3912 #[test] 3913 #[cfg(target_arch = "x86_64")] 3914 fn test_cpu_hotplug() { 3915 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3916 let guest = Guest::new(Box::new(focal)); 3917 let api_socket = temp_api_path(&guest.tmp_dir); 3918 3919 let kernel_path = direct_kernel_boot_path(); 3920 3921 let mut child = GuestCommand::new(&guest) 3922 .args(&["--cpus", "boot=2,max=4"]) 3923 .args(&["--memory", "size=512M"]) 3924 .args(&["--kernel", kernel_path.to_str().unwrap()]) 3925 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3926 .default_disks() 3927 .default_net() 3928 .args(&["--api-socket", &api_socket]) 3929 .capture_output() 3930 .spawn() 3931 .unwrap(); 3932 3933 let r = std::panic::catch_unwind(|| { 3934 guest.wait_vm_boot(None).unwrap(); 3935 3936 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 3937 3938 // Resize the VM 3939 let desired_vcpus = 4; 3940 resize_command(&api_socket, Some(desired_vcpus), None, None); 3941 3942 guest 3943 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 3944 .unwrap(); 3945 guest 3946 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 3947 .unwrap(); 3948 thread::sleep(std::time::Duration::new(10, 0)); 3949 assert_eq!( 3950 guest.get_cpu_count().unwrap_or_default(), 3951 u32::from(desired_vcpus) 3952 ); 3953 3954 guest.reboot_linux(0, None); 3955 3956 assert_eq!( 3957 guest.get_cpu_count().unwrap_or_default(), 3958 u32::from(desired_vcpus) 3959 ); 3960 3961 // Resize the VM 3962 let desired_vcpus = 2; 3963 resize_command(&api_socket, Some(desired_vcpus), None, None); 3964 3965 thread::sleep(std::time::Duration::new(10, 0)); 3966 assert_eq!( 3967 guest.get_cpu_count().unwrap_or_default(), 3968 u32::from(desired_vcpus) 3969 ); 3970 3971 // Resize the VM back up to 4 3972 let desired_vcpus = 4; 3973 resize_command(&api_socket, Some(desired_vcpus), None, None); 3974 3975 guest 3976 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 3977 .unwrap(); 3978 guest 3979 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 3980 .unwrap(); 3981 thread::sleep(std::time::Duration::new(10, 0)); 3982 assert_eq!( 3983 guest.get_cpu_count().unwrap_or_default(), 3984 u32::from(desired_vcpus) 3985 ); 3986 }); 3987 3988 let _ = child.kill(); 3989 let output = child.wait_with_output().unwrap(); 3990 3991 handle_child_output(r, &output); 3992 } 3993 3994 #[test] 3995 #[cfg(target_arch = "x86_64")] 3996 fn test_memory_hotplug() { 3997 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3998 let guest = Guest::new(Box::new(focal)); 3999 let api_socket = temp_api_path(&guest.tmp_dir); 4000 4001 let kernel_path = direct_kernel_boot_path(); 4002 4003 let mut child = GuestCommand::new(&guest) 4004 .args(&["--cpus", "boot=2,max=4"]) 4005 .args(&["--memory", "size=512M,hotplug_size=8192M"]) 4006 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4007 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4008 .default_disks() 4009 .default_net() 4010 .args(&["--balloon", "size=0"]) 4011 .args(&["--api-socket", &api_socket]) 4012 .capture_output() 4013 .spawn() 4014 .unwrap(); 4015 4016 let r = std::panic::catch_unwind(|| { 4017 guest.wait_vm_boot(None).unwrap(); 4018 4019 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4020 4021 guest.enable_memory_hotplug(); 4022 4023 // Add RAM to the VM 4024 let desired_ram = 1024 << 20; 4025 resize_command(&api_socket, None, Some(desired_ram), None); 4026 4027 thread::sleep(std::time::Duration::new(10, 0)); 4028 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4029 4030 // Use balloon to remove RAM from the VM 4031 let desired_balloon = 512 << 20; 4032 resize_command(&api_socket, None, None, Some(desired_balloon)); 4033 4034 thread::sleep(std::time::Duration::new(10, 0)); 4035 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4036 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 4037 4038 guest.reboot_linux(0, None); 4039 4040 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 4041 4042 // Use balloon add RAM to the VM 4043 let desired_balloon = 0; 4044 resize_command(&api_socket, None, None, Some(desired_balloon)); 4045 4046 thread::sleep(std::time::Duration::new(10, 0)); 4047 4048 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4049 4050 guest.enable_memory_hotplug(); 4051 4052 // Add RAM to the VM 4053 let desired_ram = 2048 << 20; 4054 resize_command(&api_socket, None, Some(desired_ram), None); 4055 4056 thread::sleep(std::time::Duration::new(10, 0)); 4057 assert!(guest.get_total_memory().unwrap_or_default() > 1_920_000); 4058 4059 // Remove RAM to the VM (only applies after reboot) 4060 let desired_ram = 1024 << 20; 4061 resize_command(&api_socket, None, Some(desired_ram), None); 4062 4063 guest.reboot_linux(1, None); 4064 4065 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4066 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 4067 }); 4068 4069 let _ = child.kill(); 4070 let output = child.wait_with_output().unwrap(); 4071 4072 handle_child_output(r, &output); 4073 } 4074 4075 #[test] 4076 #[cfg(target_arch = "x86_64")] 4077 fn test_virtio_mem() { 4078 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4079 let guest = Guest::new(Box::new(focal)); 4080 let api_socket = temp_api_path(&guest.tmp_dir); 4081 4082 let kernel_path = direct_kernel_boot_path(); 4083 4084 let mut child = GuestCommand::new(&guest) 4085 .args(&["--cpus", "boot=2,max=4"]) 4086 .args(&[ 4087 "--memory", 4088 "size=512M,hotplug_method=virtio-mem,hotplug_size=8192M", 4089 ]) 4090 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4091 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4092 .default_disks() 4093 .default_net() 4094 .args(&["--api-socket", &api_socket]) 4095 .capture_output() 4096 .spawn() 4097 .unwrap(); 4098 4099 let r = std::panic::catch_unwind(|| { 4100 guest.wait_vm_boot(None).unwrap(); 4101 4102 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4103 4104 guest.enable_memory_hotplug(); 4105 4106 // Add RAM to the VM 4107 let desired_ram = 1024 << 20; 4108 resize_command(&api_socket, None, Some(desired_ram), None); 4109 4110 thread::sleep(std::time::Duration::new(10, 0)); 4111 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4112 4113 // Add RAM to the VM 4114 let desired_ram = 2048 << 20; 4115 resize_command(&api_socket, None, Some(desired_ram), None); 4116 4117 thread::sleep(std::time::Duration::new(10, 0)); 4118 assert!(guest.get_total_memory().unwrap_or_default() > 1_920_000); 4119 4120 // Remove RAM from the VM 4121 let desired_ram = 1024 << 20; 4122 resize_command(&api_socket, None, Some(desired_ram), None); 4123 4124 thread::sleep(std::time::Duration::new(10, 0)); 4125 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4126 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 4127 4128 guest.reboot_linux(0, None); 4129 4130 // Check the amount of memory after reboot is 1GiB 4131 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4132 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 4133 4134 // Check we can still resize to 512MiB 4135 let desired_ram = 512 << 20; 4136 resize_command(&api_socket, None, Some(desired_ram), None); 4137 thread::sleep(std::time::Duration::new(10, 0)); 4138 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4139 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 4140 }); 4141 4142 let _ = child.kill(); 4143 let output = child.wait_with_output().unwrap(); 4144 4145 handle_child_output(r, &output); 4146 } 4147 4148 #[test] 4149 #[cfg(target_arch = "x86_64")] 4150 // Test both vCPU and memory resizing together 4151 fn test_resize() { 4152 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4153 let guest = Guest::new(Box::new(focal)); 4154 let api_socket = temp_api_path(&guest.tmp_dir); 4155 4156 let kernel_path = direct_kernel_boot_path(); 4157 4158 let mut child = GuestCommand::new(&guest) 4159 .args(&["--cpus", "boot=2,max=4"]) 4160 .args(&["--memory", "size=512M,hotplug_size=8192M"]) 4161 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4162 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4163 .default_disks() 4164 .default_net() 4165 .args(&["--api-socket", &api_socket]) 4166 .capture_output() 4167 .spawn() 4168 .unwrap(); 4169 4170 let r = std::panic::catch_unwind(|| { 4171 guest.wait_vm_boot(None).unwrap(); 4172 4173 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 4174 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4175 4176 guest.enable_memory_hotplug(); 4177 4178 // Resize the VM 4179 let desired_vcpus = 4; 4180 let desired_ram = 1024 << 20; 4181 resize_command(&api_socket, Some(desired_vcpus), Some(desired_ram), None); 4182 4183 guest 4184 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 4185 .unwrap(); 4186 guest 4187 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 4188 .unwrap(); 4189 thread::sleep(std::time::Duration::new(10, 0)); 4190 assert_eq!( 4191 guest.get_cpu_count().unwrap_or_default(), 4192 u32::from(desired_vcpus) 4193 ); 4194 4195 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4196 }); 4197 4198 let _ = child.kill(); 4199 let output = child.wait_with_output().unwrap(); 4200 4201 handle_child_output(r, &output); 4202 } 4203 4204 #[test] 4205 fn test_memory_overhead() { 4206 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4207 let guest = Guest::new(Box::new(focal)); 4208 4209 let kernel_path = direct_kernel_boot_path(); 4210 4211 let guest_memory_size_kb = 512 * 1024; 4212 4213 let mut child = GuestCommand::new(&guest) 4214 .args(&["--cpus", "boot=1"]) 4215 .args(&[ 4216 "--memory", 4217 format!("size={}K", guest_memory_size_kb).as_str(), 4218 ]) 4219 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4220 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4221 .default_disks() 4222 .capture_output() 4223 .spawn() 4224 .unwrap(); 4225 4226 thread::sleep(std::time::Duration::new(20, 0)); 4227 4228 let r = std::panic::catch_unwind(|| { 4229 // On AArch64 when acpi is enabled, there is a 4 MiB gap between the RAM 4230 // that the VMM gives and the guest can see. 4231 // This is a temporary solution, will be fixed in future. 4232 #[cfg(all(target_arch = "aarch64", feature = "acpi"))] 4233 let guest_memory_size_kb = guest_memory_size_kb - 4 * 1024; 4234 4235 let overhead = get_vmm_overhead(child.id(), guest_memory_size_kb); 4236 eprintln!( 4237 "Guest memory overhead: {} vs {}", 4238 overhead, MAXIMUM_VMM_OVERHEAD_KB 4239 ); 4240 assert!(overhead <= MAXIMUM_VMM_OVERHEAD_KB); 4241 }); 4242 4243 let _ = child.kill(); 4244 let output = child.wait_with_output().unwrap(); 4245 4246 handle_child_output(r, &output); 4247 } 4248 4249 #[test] 4250 #[cfg(target_arch = "x86_64")] 4251 fn test_disk_hotplug() { 4252 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4253 let guest = Guest::new(Box::new(focal)); 4254 4255 let kernel_path = direct_kernel_boot_path(); 4256 4257 let api_socket = temp_api_path(&guest.tmp_dir); 4258 4259 let mut child = GuestCommand::new(&guest) 4260 .args(&["--api-socket", &api_socket]) 4261 .args(&["--cpus", "boot=1"]) 4262 .args(&["--memory", "size=512M"]) 4263 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4264 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4265 .default_disks() 4266 .default_net() 4267 .capture_output() 4268 .spawn() 4269 .unwrap(); 4270 4271 let r = std::panic::catch_unwind(|| { 4272 guest.wait_vm_boot(None).unwrap(); 4273 4274 // Check /dev/vdc is not there 4275 assert_eq!( 4276 guest 4277 .ssh_command("lsblk | grep -c vdc.*16M || true") 4278 .unwrap() 4279 .trim() 4280 .parse::<u32>() 4281 .unwrap_or(1), 4282 0 4283 ); 4284 4285 // Now let's add the extra disk. 4286 let mut blk_file_path = dirs::home_dir().unwrap(); 4287 blk_file_path.push("workloads"); 4288 blk_file_path.push("blk.img"); 4289 let (cmd_success, cmd_output) = remote_command_w_output( 4290 &api_socket, 4291 "add-disk", 4292 Some(format!("path={},id=test0", blk_file_path.to_str().unwrap()).as_str()), 4293 ); 4294 assert!(cmd_success); 4295 assert!(String::from_utf8_lossy(&cmd_output) 4296 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 4297 4298 thread::sleep(std::time::Duration::new(10, 0)); 4299 4300 // Check that /dev/vdc exists and the block size is 16M. 4301 assert_eq!( 4302 guest 4303 .ssh_command("lsblk | grep vdc | grep -c 16M") 4304 .unwrap() 4305 .trim() 4306 .parse::<u32>() 4307 .unwrap_or_default(), 4308 1 4309 ); 4310 // And check the block device can be read. 4311 guest 4312 .ssh_command("sudo dd if=/dev/vdc of=/dev/null bs=1M iflag=direct count=16") 4313 .unwrap(); 4314 4315 // Let's remove it the extra disk. 4316 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 4317 thread::sleep(std::time::Duration::new(5, 0)); 4318 // And check /dev/vdc is not there 4319 assert_eq!( 4320 guest 4321 .ssh_command("lsblk | grep -c vdc.*16M || true") 4322 .unwrap() 4323 .trim() 4324 .parse::<u32>() 4325 .unwrap_or(1), 4326 0 4327 ); 4328 4329 // And add it back to validate unplug did work correctly. 4330 let (cmd_success, cmd_output) = remote_command_w_output( 4331 &api_socket, 4332 "add-disk", 4333 Some(format!("path={},id=test0", blk_file_path.to_str().unwrap()).as_str()), 4334 ); 4335 assert!(cmd_success); 4336 assert!(String::from_utf8_lossy(&cmd_output) 4337 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 4338 4339 thread::sleep(std::time::Duration::new(10, 0)); 4340 4341 // Check that /dev/vdc exists and the block size is 16M. 4342 assert_eq!( 4343 guest 4344 .ssh_command("lsblk | grep vdc | grep -c 16M") 4345 .unwrap() 4346 .trim() 4347 .parse::<u32>() 4348 .unwrap_or_default(), 4349 1 4350 ); 4351 // And check the block device can be read. 4352 guest 4353 .ssh_command("sudo dd if=/dev/vdc of=/dev/null bs=1M iflag=direct count=16") 4354 .unwrap(); 4355 4356 // Reboot the VM. 4357 guest.reboot_linux(0, None); 4358 4359 // Check still there after reboot 4360 assert_eq!( 4361 guest 4362 .ssh_command("lsblk | grep vdc | grep -c 16M") 4363 .unwrap() 4364 .trim() 4365 .parse::<u32>() 4366 .unwrap_or_default(), 4367 1 4368 ); 4369 4370 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 4371 4372 thread::sleep(std::time::Duration::new(20, 0)); 4373 4374 // Check device has gone away 4375 assert_eq!( 4376 guest 4377 .ssh_command("lsblk | grep -c vdc.*16M || true") 4378 .unwrap() 4379 .trim() 4380 .parse::<u32>() 4381 .unwrap_or(1), 4382 0 4383 ); 4384 4385 guest.reboot_linux(1, None); 4386 4387 // Check device still absent 4388 assert_eq!( 4389 guest 4390 .ssh_command("lsblk | grep -c vdc.*16M || true") 4391 .unwrap() 4392 .trim() 4393 .parse::<u32>() 4394 .unwrap_or(1), 4395 0 4396 ); 4397 }); 4398 4399 let _ = child.kill(); 4400 let output = child.wait_with_output().unwrap(); 4401 4402 handle_child_output(r, &output); 4403 } 4404 4405 #[test] 4406 #[cfg(target_arch = "x86_64")] 4407 fn test_virtio_balloon() { 4408 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4409 let guest = Guest::new(Box::new(focal)); 4410 4411 let kernel_path = direct_kernel_boot_path(); 4412 4413 let api_socket = temp_api_path(&guest.tmp_dir); 4414 4415 //Let's start a 4G guest with balloon occupied 2G memory 4416 let mut child = GuestCommand::new(&guest) 4417 .args(&["--api-socket", &api_socket]) 4418 .args(&["--cpus", "boot=1"]) 4419 .args(&["--memory", "size=4G"]) 4420 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4421 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4422 .args(&["--balloon", "size=2G,deflate_on_oom=on"]) 4423 .default_disks() 4424 .default_net() 4425 .capture_output() 4426 .spawn() 4427 .unwrap(); 4428 4429 let r = std::panic::catch_unwind(|| { 4430 guest.wait_vm_boot(None).unwrap(); 4431 4432 // Wait for balloon memory's initialization and check its size. 4433 // The virtio-balloon driver might take a few seconds to report the 4434 // balloon effective size back to the VMM. 4435 thread::sleep(std::time::Duration::new(20, 0)); 4436 4437 let orig_balloon = balloon_size(&api_socket); 4438 println!("The original balloon memory size is {} bytes", orig_balloon); 4439 assert!(orig_balloon == 2147483648); 4440 4441 // Two steps to verify if the 'deflate_on_oom' parameter works. 4442 // 1st: run a command in guest to eat up memory heavily, which 4443 // will consume much more memory than $(total_mem - balloon_size) 4444 // to trigger an oom. 4445 guest 4446 .ssh_command("sudo stress --vm 25 --vm-keep --vm-bytes 1G --timeout 20") 4447 .unwrap(); 4448 4449 // 2nd: check balloon_mem's value to verify balloon has been automatically deflated 4450 let deflated_balloon = balloon_size(&api_socket); 4451 println!( 4452 "After deflating, balloon memory size is {} bytes", 4453 deflated_balloon 4454 ); 4455 // Verify the balloon size deflated 4456 assert!(deflated_balloon < 2147483648); 4457 }); 4458 4459 let _ = child.kill(); 4460 let output = child.wait_with_output().unwrap(); 4461 4462 handle_child_output(r, &output); 4463 } 4464 4465 #[test] 4466 #[cfg(target_arch = "x86_64")] 4467 fn test_pmem_hotplug() { 4468 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4469 let guest = Guest::new(Box::new(focal)); 4470 4471 let kernel_path = direct_kernel_boot_path(); 4472 4473 let api_socket = temp_api_path(&guest.tmp_dir); 4474 4475 let mut child = GuestCommand::new(&guest) 4476 .args(&["--api-socket", &api_socket]) 4477 .args(&["--cpus", "boot=1"]) 4478 .args(&["--memory", "size=512M"]) 4479 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4480 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4481 .default_disks() 4482 .default_net() 4483 .capture_output() 4484 .spawn() 4485 .unwrap(); 4486 4487 let r = std::panic::catch_unwind(|| { 4488 guest.wait_vm_boot(None).unwrap(); 4489 4490 // Check /dev/pmem0 is not there 4491 assert_eq!( 4492 guest 4493 .ssh_command("lsblk | grep -c pmem0 || true") 4494 .unwrap() 4495 .trim() 4496 .parse::<u32>() 4497 .unwrap_or(1), 4498 0 4499 ); 4500 4501 let pmem_temp_file = TempFile::new().unwrap(); 4502 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 4503 let (cmd_success, cmd_output) = remote_command_w_output( 4504 &api_socket, 4505 "add-pmem", 4506 Some(&format!( 4507 "file={},id=test0", 4508 pmem_temp_file.as_path().to_str().unwrap() 4509 )), 4510 ); 4511 assert!(cmd_success); 4512 assert!(String::from_utf8_lossy(&cmd_output) 4513 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 4514 4515 // Check that /dev/pmem0 exists and the block size is 128M 4516 assert_eq!( 4517 guest 4518 .ssh_command("lsblk | grep pmem0 | grep -c 128M") 4519 .unwrap() 4520 .trim() 4521 .parse::<u32>() 4522 .unwrap_or_default(), 4523 1 4524 ); 4525 4526 guest.reboot_linux(0, None); 4527 4528 // Check still there after reboot 4529 assert_eq!( 4530 guest 4531 .ssh_command("lsblk | grep pmem0 | grep -c 128M") 4532 .unwrap() 4533 .trim() 4534 .parse::<u32>() 4535 .unwrap_or_default(), 4536 1 4537 ); 4538 4539 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 4540 4541 thread::sleep(std::time::Duration::new(20, 0)); 4542 4543 // Check device has gone away 4544 assert_eq!( 4545 guest 4546 .ssh_command("lsblk | grep -c pmem0.*128M || true") 4547 .unwrap() 4548 .trim() 4549 .parse::<u32>() 4550 .unwrap_or(1), 4551 0 4552 ); 4553 4554 guest.reboot_linux(1, None); 4555 4556 // Check still absent after reboot 4557 assert_eq!( 4558 guest 4559 .ssh_command("lsblk | grep -c pmem0.*128M || true") 4560 .unwrap() 4561 .trim() 4562 .parse::<u32>() 4563 .unwrap_or(1), 4564 0 4565 ); 4566 }); 4567 4568 let _ = child.kill(); 4569 let output = child.wait_with_output().unwrap(); 4570 4571 handle_child_output(r, &output); 4572 } 4573 4574 #[test] 4575 #[cfg(target_arch = "x86_64")] 4576 fn test_net_hotplug() { 4577 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4578 let guest = Guest::new(Box::new(focal)); 4579 4580 let kernel_path = direct_kernel_boot_path(); 4581 4582 let api_socket = temp_api_path(&guest.tmp_dir); 4583 4584 // Boot without network 4585 let mut child = GuestCommand::new(&guest) 4586 .args(&["--api-socket", &api_socket]) 4587 .args(&["--cpus", "boot=1"]) 4588 .args(&["--memory", "size=512M"]) 4589 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4590 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4591 .default_disks() 4592 .capture_output() 4593 .spawn() 4594 .unwrap(); 4595 4596 thread::sleep(std::time::Duration::new(20, 0)); 4597 4598 let r = std::panic::catch_unwind(|| { 4599 // Add network 4600 let (cmd_success, cmd_output) = remote_command_w_output( 4601 &api_socket, 4602 "add-net", 4603 Some(guest.default_net_string().as_str()), 4604 ); 4605 assert!(cmd_success); 4606 assert!(String::from_utf8_lossy(&cmd_output) 4607 .contains("{\"id\":\"_net2\",\"bdf\":\"0000:00:05.0\"}")); 4608 4609 thread::sleep(std::time::Duration::new(5, 0)); 4610 4611 // 1 network interfaces + default localhost ==> 2 interfaces 4612 assert_eq!( 4613 guest 4614 .ssh_command("ip -o link | wc -l") 4615 .unwrap() 4616 .trim() 4617 .parse::<u32>() 4618 .unwrap_or_default(), 4619 2 4620 ); 4621 4622 // Remove network 4623 assert!(remote_command(&api_socket, "remove-device", Some("_net2"))); 4624 thread::sleep(std::time::Duration::new(5, 0)); 4625 4626 // Add network again 4627 let (cmd_success, cmd_output) = remote_command_w_output( 4628 &api_socket, 4629 "add-net", 4630 Some(guest.default_net_string().as_str()), 4631 ); 4632 assert!(cmd_success); 4633 assert!(String::from_utf8_lossy(&cmd_output) 4634 .contains("{\"id\":\"_net3\",\"bdf\":\"0000:00:05.0\"}")); 4635 4636 thread::sleep(std::time::Duration::new(5, 0)); 4637 4638 // 1 network interfaces + default localhost ==> 2 interfaces 4639 assert_eq!( 4640 guest 4641 .ssh_command("ip -o link | wc -l") 4642 .unwrap() 4643 .trim() 4644 .parse::<u32>() 4645 .unwrap_or_default(), 4646 2 4647 ); 4648 4649 guest.reboot_linux(0, None); 4650 4651 // Check still there after reboot 4652 // 1 network interfaces + default localhost ==> 2 interfaces 4653 assert_eq!( 4654 guest 4655 .ssh_command("ip -o link | wc -l") 4656 .unwrap() 4657 .trim() 4658 .parse::<u32>() 4659 .unwrap_or_default(), 4660 2 4661 ); 4662 }); 4663 4664 let _ = child.kill(); 4665 let output = child.wait_with_output().unwrap(); 4666 4667 handle_child_output(r, &output); 4668 } 4669 4670 #[test] 4671 fn test_initramfs() { 4672 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4673 let guest = Guest::new(Box::new(focal)); 4674 let mut workload_path = dirs::home_dir().unwrap(); 4675 workload_path.push("workloads"); 4676 4677 let mut kernels = vec![direct_kernel_boot_path()]; 4678 4679 #[cfg(target_arch = "x86_64")] 4680 { 4681 let mut pvh_kernel_path = workload_path.clone(); 4682 pvh_kernel_path.push("vmlinux"); 4683 kernels.push(pvh_kernel_path); 4684 } 4685 4686 let mut initramfs_path = workload_path; 4687 initramfs_path.push("alpine_initramfs.img"); 4688 4689 let test_string = String::from("axz34i9rylotd8n50wbv6kcj7f2qushme1pg"); 4690 let cmdline = format!("console=hvc0 quiet TEST_STRING={}", test_string); 4691 4692 kernels.iter().for_each(|k_path| { 4693 let mut child = GuestCommand::new(&guest) 4694 .args(&["--kernel", k_path.to_str().unwrap()]) 4695 .args(&["--initramfs", initramfs_path.to_str().unwrap()]) 4696 .args(&["--cmdline", &cmdline]) 4697 .capture_output() 4698 .spawn() 4699 .unwrap(); 4700 4701 thread::sleep(std::time::Duration::new(20, 0)); 4702 4703 let _ = child.kill(); 4704 let output = child.wait_with_output().unwrap(); 4705 4706 let r = std::panic::catch_unwind(|| { 4707 let s = String::from_utf8_lossy(&output.stdout); 4708 4709 assert_ne!(s.lines().position(|line| line == test_string), None); 4710 }); 4711 4712 handle_child_output(r, &output); 4713 }); 4714 } 4715 4716 // One thing to note about this test. The virtio-net device is heavily used 4717 // through each ssh command. There's no need to perform a dedicated test to 4718 // verify the migration went well for virtio-net. 4719 #[test] 4720 fn test_snapshot_restore() { 4721 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4722 let guest = Guest::new(Box::new(focal)); 4723 let kernel_path = direct_kernel_boot_path(); 4724 4725 let api_socket = temp_api_path(&guest.tmp_dir); 4726 4727 let net_id = "net123"; 4728 let net_params = format!( 4729 "id={},tap=,mac={},ip={},mask=255.255.255.0", 4730 net_id, guest.network.guest_mac, guest.network.host_ip 4731 ); 4732 4733 let cloudinit_params = format!( 4734 "path={},iommu=on", 4735 guest.disk_config.disk(DiskType::CloudInit).unwrap() 4736 ); 4737 4738 let socket = temp_vsock_path(&guest.tmp_dir); 4739 4740 let mut child = GuestCommand::new(&guest) 4741 .args(&["--api-socket", &api_socket]) 4742 .args(&["--cpus", "boot=4"]) 4743 .args(&["--memory", "size=4G"]) 4744 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4745 .args(&[ 4746 "--disk", 4747 format!( 4748 "path={}", 4749 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 4750 ) 4751 .as_str(), 4752 cloudinit_params.as_str(), 4753 ]) 4754 .args(&["--net", net_params.as_str()]) 4755 .args(&["--vsock", format!("cid=3,socket={}", socket).as_str()]) 4756 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4757 .capture_output() 4758 .spawn() 4759 .unwrap(); 4760 4761 let console_text = String::from("On a branch floating down river a cricket, singing."); 4762 let console_cmd = format!("echo {} | sudo tee /dev/hvc0", console_text); 4763 // Create the snapshot directory 4764 let snapshot_dir = temp_snapshot_dir_path(&guest.tmp_dir); 4765 4766 let r = std::panic::catch_unwind(|| { 4767 guest.wait_vm_boot(None).unwrap(); 4768 4769 // Check the number of vCPUs 4770 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 4); 4771 // Check the guest RAM 4772 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 4773 // Check block devices are readable 4774 guest 4775 .ssh_command("sudo dd if=/dev/vda of=/dev/null bs=1M iflag=direct count=1024") 4776 .unwrap(); 4777 guest 4778 .ssh_command("sudo dd if=/dev/vdb of=/dev/null bs=1M iflag=direct count=8") 4779 .unwrap(); 4780 // Check if the rng device is readable 4781 guest 4782 .ssh_command("sudo head -c 1000 /dev/hwrng > /dev/null") 4783 .unwrap(); 4784 // Check vsock 4785 guest.check_vsock(socket.as_str()); 4786 // Check if the console is usable 4787 4788 guest.ssh_command(&console_cmd).unwrap(); 4789 4790 // x86_64: We check that removing and adding back the virtio-net device 4791 // does not break the snapshot/restore support for virtio-pci. 4792 // This is an important thing to test as the hotplug will 4793 // trigger a PCI BAR reprogramming, which is a good way of 4794 // checking if the stored resources are correctly restored. 4795 // Unplug the virtio-net device 4796 // AArch64: Device hotplug is currently not supported, skipping here. 4797 #[cfg(target_arch = "x86_64")] 4798 { 4799 assert!(remote_command(&api_socket, "remove-device", Some(net_id),)); 4800 thread::sleep(std::time::Duration::new(10, 0)); 4801 4802 // Plug the virtio-net device again 4803 assert!(remote_command( 4804 &api_socket, 4805 "add-net", 4806 Some(net_params.as_str()), 4807 )); 4808 thread::sleep(std::time::Duration::new(10, 0)); 4809 } 4810 4811 // Pause the VM 4812 assert!(remote_command(&api_socket, "pause", None)); 4813 4814 // Take a snapshot from the VM 4815 assert!(remote_command( 4816 &api_socket, 4817 "snapshot", 4818 Some(format!("file://{}", snapshot_dir).as_str()), 4819 )); 4820 4821 // Wait to make sure the snapshot is completed 4822 thread::sleep(std::time::Duration::new(10, 0)); 4823 }); 4824 4825 // Shutdown the source VM and check console output 4826 let _ = child.kill(); 4827 let output = child.wait_with_output().unwrap(); 4828 handle_child_output(r, &output); 4829 4830 let r = std::panic::catch_unwind(|| { 4831 assert!(String::from_utf8_lossy(&output.stdout).contains(&console_text)); 4832 }); 4833 4834 handle_child_output(r, &output); 4835 4836 // Remove the vsock socket file. 4837 Command::new("rm") 4838 .arg("-f") 4839 .arg(socket.as_str()) 4840 .output() 4841 .unwrap(); 4842 4843 // Restore the VM from the snapshot 4844 let mut child = GuestCommand::new(&guest) 4845 .args(&["--api-socket", &api_socket]) 4846 .args(&[ 4847 "--restore", 4848 format!("source_url=file://{}", snapshot_dir).as_str(), 4849 ]) 4850 .capture_output() 4851 .spawn() 4852 .unwrap(); 4853 4854 // Wait for the VM to be restored 4855 thread::sleep(std::time::Duration::new(10, 0)); 4856 4857 let r = std::panic::catch_unwind(|| { 4858 // Resume the VM 4859 assert!(remote_command(&api_socket, "resume", None)); 4860 4861 // Perform same checks to validate VM has been properly restored 4862 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 4); 4863 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 4864 guest 4865 .ssh_command("sudo dd if=/dev/vda of=/dev/null bs=1M iflag=direct count=1024") 4866 .unwrap(); 4867 guest 4868 .ssh_command("sudo dd if=/dev/vdb of=/dev/null bs=1M iflag=direct count=8") 4869 .unwrap(); 4870 guest 4871 .ssh_command("sudo head -c 1000 /dev/hwrng > /dev/null") 4872 .unwrap(); 4873 guest.check_vsock(socket.as_str()); 4874 guest.ssh_command(&console_cmd).unwrap(); 4875 // Shutdown the target VM and check console output 4876 }); 4877 let _ = child.kill(); 4878 let output = child.wait_with_output().unwrap(); 4879 handle_child_output(r, &output); 4880 4881 let r = std::panic::catch_unwind(|| { 4882 assert!(String::from_utf8_lossy(&output.stdout).contains(&console_text)); 4883 }); 4884 4885 handle_child_output(r, &output); 4886 } 4887 4888 #[test] 4889 fn test_counters() { 4890 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4891 let guest = Guest::new(Box::new(focal)); 4892 let api_socket = temp_api_path(&guest.tmp_dir); 4893 4894 let mut cmd = GuestCommand::new(&guest); 4895 cmd.args(&["--cpus", "boot=1"]) 4896 .args(&["--memory", "size=512M"]) 4897 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 4898 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4899 .default_disks() 4900 .args(&["--net", guest.default_net_string().as_str()]) 4901 .args(&["--api-socket", &api_socket]) 4902 .capture_output(); 4903 4904 let mut child = cmd.spawn().unwrap(); 4905 4906 let r = std::panic::catch_unwind(|| { 4907 guest.wait_vm_boot(None).unwrap(); 4908 4909 let orig_counters = get_counters(&api_socket); 4910 guest 4911 .ssh_command("dd if=/dev/zero of=test count=8 bs=1M") 4912 .unwrap(); 4913 4914 let new_counters = get_counters(&api_socket); 4915 4916 // Check that all the counters have increased 4917 assert!(new_counters > orig_counters); 4918 }); 4919 4920 let _ = child.kill(); 4921 let output = child.wait_with_output().unwrap(); 4922 4923 handle_child_output(r, &output); 4924 } 4925 4926 #[test] 4927 fn test_watchdog() { 4928 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4929 let guest = Guest::new(Box::new(focal)); 4930 let api_socket = temp_api_path(&guest.tmp_dir); 4931 4932 let kernel_path = direct_kernel_boot_path(); 4933 4934 let mut cmd = GuestCommand::new(&guest); 4935 cmd.args(&["--cpus", "boot=1"]) 4936 .args(&["--memory", "size=512M"]) 4937 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4938 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4939 .default_disks() 4940 .args(&["--net", guest.default_net_string().as_str()]) 4941 .args(&["--watchdog"]) 4942 .args(&["--api-socket", &api_socket]) 4943 .capture_output(); 4944 4945 let mut child = cmd.spawn().unwrap(); 4946 4947 let r = std::panic::catch_unwind(|| { 4948 guest.wait_vm_boot(None).unwrap(); 4949 4950 // Check for PCI device 4951 assert!(guest 4952 .does_device_vendor_pair_match("0x1063", "0x1af4") 4953 .unwrap_or_default()); 4954 4955 // Enable systemd watchdog 4956 guest 4957 .ssh_command( 4958 "echo RuntimeWatchdogSec=15s | sudo tee -a /etc/systemd/system.conf", 4959 ) 4960 .unwrap(); 4961 4962 guest.ssh_command("sudo reboot").unwrap(); 4963 4964 guest.wait_vm_boot(None).unwrap(); 4965 4966 // Check that systemd has activated the watchdog 4967 assert_eq!( 4968 guest 4969 .ssh_command("sudo journalctl | grep -c -- \"Watchdog started\"") 4970 .unwrap() 4971 .trim() 4972 .parse::<u32>() 4973 .unwrap_or_default(), 4974 2 4975 ); 4976 4977 // Ensure that the current boot journal is written so reboot counts are valid 4978 guest.ssh_command("sudo journalctl --sync").unwrap(); 4979 4980 let boot_count = guest 4981 .ssh_command("sudo journalctl --list-boots | wc -l") 4982 .unwrap() 4983 .trim() 4984 .parse::<u32>() 4985 .unwrap_or_default(); 4986 assert_eq!(boot_count, 2); 4987 // Allow some normal time to elapse to check we don't get spurious reboots 4988 thread::sleep(std::time::Duration::new(40, 0)); 4989 4990 // Check no reboot 4991 let boot_count = guest 4992 .ssh_command("sudo journalctl --list-boots | wc -l") 4993 .unwrap() 4994 .trim() 4995 .parse::<u32>() 4996 .unwrap_or_default(); 4997 assert_eq!(boot_count, 2); 4998 4999 // Ensure that the current boot journal is written so reboot counts are valid 5000 guest.ssh_command("sudo journalctl --sync").unwrap(); 5001 5002 // Trigger a panic (sync first). We need to do this inside a screen with a delay so the SSH command returns. 5003 guest.ssh_command("screen -dmS reboot sh -c \"sleep 5; echo s | tee /proc/sysrq-trigger; echo c | sudo tee /proc/sysrq-trigger\"").unwrap(); 5004 5005 // Allow some time for the watchdog to trigger (max 30s) and reboot to happen 5006 guest.wait_vm_boot(Some(50)).unwrap(); 5007 5008 // Check that watchdog triggered reboot 5009 let boot_count = guest 5010 .ssh_command("sudo journalctl --list-boots | wc -l") 5011 .unwrap() 5012 .trim() 5013 .parse::<u32>() 5014 .unwrap_or_default(); 5015 assert_eq!(boot_count, 3); 5016 5017 #[cfg(target_arch = "x86_64")] 5018 { 5019 // Now pause the VM and remain offline for 30s 5020 assert!(remote_command(&api_socket, "pause", None)); 5021 thread::sleep(std::time::Duration::new(30, 0)); 5022 assert!(remote_command(&api_socket, "resume", None)); 5023 5024 // Check no reboot 5025 let boot_count = guest 5026 .ssh_command("sudo journalctl --list-boots | wc -l") 5027 .unwrap() 5028 .trim() 5029 .parse::<u32>() 5030 .unwrap_or_default(); 5031 assert_eq!(boot_count, 3); 5032 } 5033 }); 5034 5035 let _ = child.kill(); 5036 let output = child.wait_with_output().unwrap(); 5037 5038 handle_child_output(r, &output); 5039 } 5040 5041 #[test] 5042 fn test_tap_from_fd() { 5043 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5044 let guest = Guest::new(Box::new(focal)); 5045 let kernel_path = direct_kernel_boot_path(); 5046 5047 // Create a TAP interface with multi-queue enabled 5048 let num_queue_pairs: usize = 2; 5049 5050 use std::str::FromStr; 5051 let taps = net_util::open_tap( 5052 Some("chtap0"), 5053 Some(std::net::Ipv4Addr::from_str(&guest.network.host_ip).unwrap()), 5054 None, 5055 &mut None, 5056 num_queue_pairs, 5057 Some(libc::O_RDWR | libc::O_NONBLOCK), 5058 ) 5059 .unwrap(); 5060 5061 let mut child = GuestCommand::new(&guest) 5062 .args(&["--cpus", &format!("boot={}", num_queue_pairs)]) 5063 .args(&["--memory", "size=512M"]) 5064 .args(&["--kernel", kernel_path.to_str().unwrap()]) 5065 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5066 .default_disks() 5067 .args(&[ 5068 "--net", 5069 &format!( 5070 "fd={}:{},mac={},num_queues={}", 5071 taps[0].as_raw_fd(), 5072 taps[1].as_raw_fd(), 5073 guest.network.guest_mac, 5074 num_queue_pairs * 2 5075 ), 5076 ]) 5077 .capture_output() 5078 .spawn() 5079 .unwrap(); 5080 5081 let r = std::panic::catch_unwind(|| { 5082 guest.wait_vm_boot(None).unwrap(); 5083 5084 assert_eq!( 5085 guest 5086 .ssh_command("ip -o link | wc -l") 5087 .unwrap() 5088 .trim() 5089 .parse::<u32>() 5090 .unwrap_or_default(), 5091 2 5092 ); 5093 5094 guest.reboot_linux(0, None); 5095 5096 assert_eq!( 5097 guest 5098 .ssh_command("ip -o link | wc -l") 5099 .unwrap() 5100 .trim() 5101 .parse::<u32>() 5102 .unwrap_or_default(), 5103 2 5104 ); 5105 }); 5106 5107 let _ = child.kill(); 5108 let output = child.wait_with_output().unwrap(); 5109 5110 handle_child_output(r, &output); 5111 } 5112 5113 // By design, a guest VM won't be able to connect to the host 5114 // machine when using a macvtap network interface (while it can 5115 // communicate externally). As a workaround, this integration 5116 // test creates two macvtap interfaces in 'bridge' mode on the 5117 // same physical net interface, one for the guest and one for 5118 // the host. With additional setup on the IP address and the 5119 // routing table, it enables the communications between the 5120 // guest VM and the host machine. 5121 // Details: https://wiki.libvirt.org/page/TroubleshootMacvtapHostFail 5122 fn _test_macvtap(hotplug: bool, guest_macvtap_name: &str, host_macvtap_name: &str) { 5123 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5124 let guest = Guest::new(Box::new(focal)); 5125 let api_socket = temp_api_path(&guest.tmp_dir); 5126 let kernel_path = direct_kernel_boot_path(); 5127 let phy_net = "eth0"; 5128 5129 // Create a macvtap interface for the guest VM to use 5130 assert!(exec_host_command_status(&format!( 5131 "sudo ip link add link {} name {} type macvtap mod bridge", 5132 phy_net, guest_macvtap_name 5133 )) 5134 .success()); 5135 assert!(exec_host_command_status(&format!( 5136 "sudo ip link set {} address {} up", 5137 guest_macvtap_name, guest.network.guest_mac 5138 )) 5139 .success()); 5140 assert!( 5141 exec_host_command_status(&format!("sudo ip link show {}", guest_macvtap_name)) 5142 .success() 5143 ); 5144 5145 let tap_index = 5146 fs::read_to_string(&format!("/sys/class/net/{}/ifindex", guest_macvtap_name)) 5147 .unwrap(); 5148 let tap_device = format!("/dev/tap{}", tap_index.trim()); 5149 5150 assert!( 5151 exec_host_command_status(&format!("sudo chown $UID.$UID {}", tap_device)).success() 5152 ); 5153 5154 let cstr_tap_device = std::ffi::CString::new(tap_device).unwrap(); 5155 let tap_fd = unsafe { libc::open(cstr_tap_device.as_ptr(), libc::O_RDWR) }; 5156 assert!(tap_fd > 0); 5157 5158 // Create a macvtap on the same physical net interface for 5159 // the host machine to use 5160 assert!(exec_host_command_status(&format!( 5161 "sudo ip link add link {} name {} type macvtap mod bridge", 5162 phy_net, host_macvtap_name 5163 )) 5164 .success()); 5165 // Use default mask "255.255.255.0" 5166 assert!(exec_host_command_status(&format!( 5167 "sudo ip address add {}/24 dev {}", 5168 guest.network.host_ip, host_macvtap_name 5169 )) 5170 .success()); 5171 assert!(exec_host_command_status(&format!( 5172 "sudo ip link set dev {} up", 5173 host_macvtap_name 5174 )) 5175 .success()); 5176 5177 let mut guest_command = GuestCommand::new(&guest); 5178 guest_command 5179 .args(&["--cpus", "boot=1"]) 5180 .args(&["--memory", "size=512M"]) 5181 .args(&["--kernel", kernel_path.to_str().unwrap()]) 5182 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5183 .default_disks() 5184 .args(&["--api-socket", &api_socket]); 5185 5186 let net_params = format!("fd={},mac={}", tap_fd, guest.network.guest_mac); 5187 5188 if !hotplug { 5189 guest_command.args(&["--net", &net_params]); 5190 } 5191 5192 let mut child = guest_command.capture_output().spawn().unwrap(); 5193 5194 if hotplug { 5195 // Give some time to the VMM process to listen to the API 5196 // socket. This is the only requirement to avoid the following 5197 // call to ch-remote from failing. 5198 thread::sleep(std::time::Duration::new(10, 0)); 5199 // Hotplug the virtio-net device 5200 let (cmd_success, cmd_output) = 5201 remote_command_w_output(&api_socket, "add-net", Some(&net_params)); 5202 assert!(cmd_success); 5203 assert!(String::from_utf8_lossy(&cmd_output) 5204 .contains("{\"id\":\"_net2\",\"bdf\":\"0000:00:05.0\"}")); 5205 } 5206 5207 // The functional connectivity provided by the virtio-net device 5208 // gets tested through wait_vm_boot() as it expects to receive a 5209 // HTTP request, and through the SSH command as well. 5210 let r = std::panic::catch_unwind(|| { 5211 guest.wait_vm_boot(None).unwrap(); 5212 5213 assert_eq!( 5214 guest 5215 .ssh_command("ip -o link | wc -l") 5216 .unwrap() 5217 .trim() 5218 .parse::<u32>() 5219 .unwrap_or_default(), 5220 2 5221 ); 5222 }); 5223 5224 let _ = child.kill(); 5225 5226 exec_host_command_status(&format!("sudo ip link del {}", guest_macvtap_name)); 5227 exec_host_command_status(&format!("sudo ip link del {}", host_macvtap_name)); 5228 5229 let output = child.wait_with_output().unwrap(); 5230 5231 handle_child_output(r, &output); 5232 } 5233 5234 #[test] 5235 fn test_macvtap() { 5236 _test_macvtap(false, "guestmacvtap0", "hostmacvtap0") 5237 } 5238 5239 #[test] 5240 #[cfg(target_arch = "x86_64")] 5241 fn test_macvtap_hotplug() { 5242 _test_macvtap(true, "guestmacvtap1", "hostmacvtap1") 5243 } 5244 5245 #[test] 5246 fn test_ovs_dpdk() { 5247 // Create OVS-DPDK bridge and ports 5248 assert!(exec_host_command_status( 5249 "ovs-vsctl add-br ovsbr0 -- set bridge ovsbr0 datapath_type=netdev", 5250 ) 5251 .success()); 5252 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user1 -- set Interface vhost-user1 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient1").success()); 5253 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user2 -- set Interface vhost-user2 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient2").success()); 5254 assert!(exec_host_command_status("ip link set up dev ovsbr0").success()); 5255 assert!(exec_host_command_status("service openvswitch-switch restart").success()); 5256 5257 let focal1 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5258 let guest1 = Guest::new(Box::new(focal1)); 5259 let mut child1 = GuestCommand::new(&guest1) 5260 .args(&["--cpus", "boot=2"]) 5261 .args(&["--memory", "size=0,shared=on"]) 5262 .args(&["--memory-zone", "id=mem0,size=1G,shared=on,host_numa_node=0"]) 5263 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 5264 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5265 .default_disks() 5266 .args(&["--net", guest1.default_net_string().as_str(), "vhost_user=true,socket=/tmp/dpdkvhostclient1,num_queues=2,queue_size=256,vhost_mode=server"]) 5267 .capture_output() 5268 .spawn() 5269 .unwrap(); 5270 5271 #[cfg(target_arch = "x86_64")] 5272 let guest_net_iface = "ens5"; 5273 #[cfg(target_arch = "aarch64")] 5274 let guest_net_iface = "enp0s5"; 5275 5276 let r = std::panic::catch_unwind(|| { 5277 guest1.wait_vm_boot(None).unwrap(); 5278 5279 guest1 5280 .ssh_command(&format!( 5281 "sudo ip addr add 172.100.0.1/24 dev {}", 5282 guest_net_iface 5283 )) 5284 .unwrap(); 5285 guest1 5286 .ssh_command(&format!("sudo ip link set up dev {}", guest_net_iface)) 5287 .unwrap(); 5288 5289 let guest_ip = guest1.network.guest_ip.clone(); 5290 thread::spawn(move || { 5291 ssh_command_ip( 5292 "nc -l 12345", 5293 &guest_ip, 5294 DEFAULT_SSH_RETRIES, 5295 DEFAULT_SSH_TIMEOUT, 5296 ) 5297 .unwrap(); 5298 }); 5299 }); 5300 if r.is_err() { 5301 let _ = child1.kill(); 5302 let output = child1.wait_with_output().unwrap(); 5303 handle_child_output(r, &output); 5304 panic!("Test should already be failed/panicked"); // To explicitly mark this block never return 5305 } 5306 5307 let focal2 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5308 let guest2 = Guest::new(Box::new(focal2)); 5309 let api_socket = temp_api_path(&guest2.tmp_dir); 5310 5311 // Create the snapshot directory 5312 let snapshot_dir = temp_snapshot_dir_path(&guest2.tmp_dir); 5313 5314 let mut child2 = GuestCommand::new(&guest2) 5315 .args(&["--api-socket", &api_socket]) 5316 .args(&["--cpus", "boot=2"]) 5317 .args(&["--memory", "size=0,shared=on"]) 5318 .args(&["--memory-zone", "id=mem0,size=1G,shared=on,host_numa_node=0"]) 5319 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 5320 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5321 .default_disks() 5322 .args(&["--net", guest2.default_net_string().as_str(), "vhost_user=true,socket=/tmp/dpdkvhostclient2,num_queues=2,queue_size=256,vhost_mode=server"]) 5323 .capture_output() 5324 .spawn() 5325 .unwrap(); 5326 5327 let r = std::panic::catch_unwind(|| { 5328 guest2.wait_vm_boot(None).unwrap(); 5329 5330 guest2 5331 .ssh_command(&format!( 5332 "sudo ip addr add 172.100.0.2/24 dev {}", 5333 guest_net_iface 5334 )) 5335 .unwrap(); 5336 guest2 5337 .ssh_command(&format!("sudo ip link set up dev {}", guest_net_iface)) 5338 .unwrap(); 5339 5340 // Check the connection works properly between the two VMs 5341 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 5342 5343 // Remove one of the two ports from the OVS bridge 5344 assert!(exec_host_command_status("ovs-vsctl del-port vhost-user1").success()); 5345 5346 // Spawn a new netcat listener in the first VM 5347 let guest_ip = guest1.network.guest_ip.clone(); 5348 thread::spawn(move || { 5349 ssh_command_ip( 5350 "nc -l 12345", 5351 &guest_ip, 5352 DEFAULT_SSH_RETRIES, 5353 DEFAULT_SSH_TIMEOUT, 5354 ) 5355 .unwrap(); 5356 }); 5357 5358 // Wait for the server to be listening 5359 thread::sleep(std::time::Duration::new(5, 0)); 5360 5361 // Check the connection fails this time 5362 assert!(guest2.ssh_command("nc -vz 172.100.0.1 12345").is_err()); 5363 5364 // Add the OVS port back 5365 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user1 -- set Interface vhost-user1 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient1").success()); 5366 5367 // And finally check the connection is functional again 5368 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 5369 5370 // Pause the VM 5371 assert!(remote_command(&api_socket, "pause", None)); 5372 5373 // Take a snapshot from the VM 5374 assert!(remote_command( 5375 &api_socket, 5376 "snapshot", 5377 Some(format!("file://{}", snapshot_dir).as_str()), 5378 )); 5379 5380 // Wait to make sure the snapshot is completed 5381 thread::sleep(std::time::Duration::new(10, 0)); 5382 }); 5383 5384 // Shutdown the source VM 5385 let _ = child2.kill(); 5386 let output = child2.wait_with_output().unwrap(); 5387 handle_child_output(r, &output); 5388 5389 // Remove the vhost-user socket file. 5390 Command::new("rm") 5391 .arg("-f") 5392 .arg("/tmp/dpdkvhostclient2") 5393 .output() 5394 .unwrap(); 5395 5396 // Restore the VM from the snapshot 5397 let mut child2 = GuestCommand::new(&guest2) 5398 .args(&["--api-socket", &api_socket]) 5399 .args(&[ 5400 "--restore", 5401 format!("source_url=file://{}", snapshot_dir).as_str(), 5402 ]) 5403 .capture_output() 5404 .spawn() 5405 .unwrap(); 5406 5407 // Wait for the VM to be restored 5408 thread::sleep(std::time::Duration::new(10, 0)); 5409 5410 let r = std::panic::catch_unwind(|| { 5411 // Resume the VM 5412 assert!(remote_command(&api_socket, "resume", None)); 5413 5414 // Spawn a new netcat listener in the first VM 5415 let guest_ip = guest1.network.guest_ip.clone(); 5416 thread::spawn(move || { 5417 ssh_command_ip( 5418 "nc -l 12345", 5419 &guest_ip, 5420 DEFAULT_SSH_RETRIES, 5421 DEFAULT_SSH_TIMEOUT, 5422 ) 5423 .unwrap(); 5424 }); 5425 5426 // Wait for the server to be listening 5427 thread::sleep(std::time::Duration::new(5, 0)); 5428 5429 // And check the connection is still functional after restore 5430 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 5431 }); 5432 5433 let _ = child1.kill(); 5434 let _ = child2.kill(); 5435 5436 let output = child1.wait_with_output().unwrap(); 5437 child2.wait().unwrap(); 5438 5439 handle_child_output(r, &output); 5440 } 5441 } 5442 5443 mod sequential { 5444 use crate::tests::*; 5445 5446 #[test] 5447 fn test_memory_mergeable_on() { 5448 test_memory_mergeable(true) 5449 } 5450 } 5451 5452 #[cfg(target_arch = "x86_64")] 5453 mod windows { 5454 use crate::tests::*; 5455 5456 lazy_static! { 5457 static ref NEXT_DISK_ID: Mutex<u8> = Mutex::new(1); 5458 } 5459 5460 struct WindowsGuest { 5461 guest: Guest, 5462 auth: PasswordAuth, 5463 } 5464 5465 trait FsType { 5466 const FS_FAT: u8; 5467 const FS_NTFS: u8; 5468 } 5469 impl FsType for WindowsGuest { 5470 const FS_FAT: u8 = 0; 5471 const FS_NTFS: u8 = 1; 5472 } 5473 5474 impl WindowsGuest { 5475 fn new() -> Self { 5476 let disk = WindowsDiskConfig::new(WINDOWS_IMAGE_NAME.to_string()); 5477 let guest = Guest::new(Box::new(disk)); 5478 let auth = PasswordAuth { 5479 username: String::from("administrator"), 5480 password: String::from("Admin123"), 5481 }; 5482 5483 WindowsGuest { guest, auth } 5484 } 5485 5486 fn guest(&self) -> &Guest { 5487 &self.guest 5488 } 5489 5490 fn ssh_cmd(&self, cmd: &str) -> String { 5491 ssh_command_ip_with_auth( 5492 cmd, 5493 &self.auth, 5494 &self.guest.network.guest_ip, 5495 DEFAULT_SSH_RETRIES, 5496 DEFAULT_SSH_TIMEOUT, 5497 ) 5498 .unwrap() 5499 } 5500 5501 fn cpu_count(&self) -> u8 { 5502 self.ssh_cmd("powershell -Command \"(Get-CimInstance win32_computersystem).NumberOfLogicalProcessors\"") 5503 .trim() 5504 .parse::<u8>() 5505 .unwrap_or(0) 5506 } 5507 5508 fn ram_size(&self) -> usize { 5509 self.ssh_cmd("powershell -Command \"(Get-CimInstance win32_computersystem).TotalPhysicalMemory\"") 5510 .trim() 5511 .parse::<usize>() 5512 .unwrap_or(0) 5513 } 5514 5515 fn netdev_count(&self) -> u8 { 5516 self.ssh_cmd("powershell -Command \"netsh int ipv4 show interfaces | Select-String ethernet | Measure-Object -Line | Format-Table -HideTableHeaders\"") 5517 .trim() 5518 .parse::<u8>() 5519 .unwrap_or(0) 5520 } 5521 5522 fn disk_count(&self) -> u8 { 5523 self.ssh_cmd("powershell -Command \"Get-Disk | Measure-Object -Line | Format-Table -HideTableHeaders\"") 5524 .trim() 5525 .parse::<u8>() 5526 .unwrap_or(0) 5527 } 5528 5529 fn reboot(&self) { 5530 let _ = self.ssh_cmd("shutdown /r /t 0"); 5531 } 5532 5533 fn shutdown(&self) { 5534 let _ = self.ssh_cmd("shutdown /s /t 0"); 5535 } 5536 5537 fn run_dnsmasq(&self) -> std::process::Child { 5538 let listen_address = format!("--listen-address={}", self.guest.network.host_ip); 5539 let dhcp_host = format!( 5540 "--dhcp-host={},{}", 5541 self.guest.network.guest_mac, self.guest.network.guest_ip 5542 ); 5543 let dhcp_range = format!( 5544 "--dhcp-range=eth,{},{}", 5545 self.guest.network.guest_ip, self.guest.network.guest_ip 5546 ); 5547 5548 Command::new("dnsmasq") 5549 .arg("--no-daemon") 5550 .arg("--log-queries") 5551 .arg(listen_address.as_str()) 5552 .arg("--except-interface=lo") 5553 .arg("--bind-dynamic") // Allow listening to host_ip while the interface is not ready yet. 5554 .arg("--conf-file=/dev/null") 5555 .arg(dhcp_host.as_str()) 5556 .arg(dhcp_range.as_str()) 5557 .spawn() 5558 .unwrap() 5559 } 5560 5561 // TODO Cleanup image file explicitly after test, if there's some space issues. 5562 fn disk_new(&self, fs: u8, sz: usize) -> String { 5563 let mut guard = NEXT_DISK_ID.lock().unwrap(); 5564 let id = *guard; 5565 *guard = id + 1; 5566 5567 let img = PathBuf::from(format!("/tmp/test-hotplug-{}.raw", id)); 5568 let _ = fs::remove_file(&img); 5569 5570 // Create an image file 5571 let out = Command::new("qemu-img") 5572 .args(&[ 5573 "create", 5574 "-f", 5575 "raw", 5576 img.to_str().unwrap(), 5577 format!("{}m", sz).as_str(), 5578 ]) 5579 .output() 5580 .expect("qemu-img command failed") 5581 .stdout; 5582 println!("{:?}", out); 5583 5584 // Associate image to a loop device 5585 let out = Command::new("losetup") 5586 .args(&["--show", "-f", img.to_str().unwrap()]) 5587 .output() 5588 .expect("failed to create loop device") 5589 .stdout; 5590 let _tmp = String::from_utf8_lossy(&out); 5591 let loop_dev = _tmp.trim(); 5592 println!("{:?}", out); 5593 5594 // Create a partition table 5595 // echo 'type=7' | sudo sfdisk "${LOOP}" 5596 let mut child = Command::new("sfdisk") 5597 .args(&[loop_dev]) 5598 .stdin(Stdio::piped()) 5599 .spawn() 5600 .unwrap(); 5601 let stdin = child.stdin.as_mut().expect("failed to open stdin"); 5602 let _ = stdin 5603 .write_all("type=7".as_bytes()) 5604 .expect("failed to write stdin"); 5605 let out = child.wait_with_output().expect("sfdisk failed").stdout; 5606 println!("{:?}", out); 5607 5608 // Disengage the loop device 5609 let out = Command::new("losetup") 5610 .args(&["-d", loop_dev]) 5611 .output() 5612 .expect("loop device not found") 5613 .stdout; 5614 println!("{:?}", out); 5615 5616 // Re-associate loop device pointing to the partition only 5617 let out = Command::new("losetup") 5618 .args(&[ 5619 "--show", 5620 "--offset", 5621 (512 * 2048).to_string().as_str(), 5622 "-f", 5623 img.to_str().unwrap(), 5624 ]) 5625 .output() 5626 .expect("failed to create loop device") 5627 .stdout; 5628 let _tmp = String::from_utf8_lossy(&out); 5629 let loop_dev = _tmp.trim(); 5630 println!("{:?}", out); 5631 5632 // Create filesystem. 5633 let fs_cmd = match fs { 5634 WindowsGuest::FS_FAT => "mkfs.msdos", 5635 WindowsGuest::FS_NTFS => "mkfs.ntfs", 5636 _ => panic!("Unknown filesystem type '{}'", fs), 5637 }; 5638 let out = Command::new(fs_cmd) 5639 .args(&[&loop_dev]) 5640 .output() 5641 .unwrap_or_else(|_| panic!("{} failed", fs_cmd)) 5642 .stdout; 5643 println!("{:?}", out); 5644 5645 // Disengage the loop device 5646 let out = Command::new("losetup") 5647 .args(&["-d", loop_dev]) 5648 .output() 5649 .unwrap_or_else(|_| panic!("loop device '{}' not found", loop_dev)) 5650 .stdout; 5651 println!("{:?}", out); 5652 5653 img.to_str().unwrap().to_string() 5654 } 5655 5656 fn disks_set_rw(&self) { 5657 let _ = self.ssh_cmd("powershell -Command \"Get-Disk | Where-Object IsOffline -eq $True | Set-Disk -IsReadOnly $False\""); 5658 } 5659 5660 fn disks_online(&self) { 5661 let _ = self.ssh_cmd("powershell -Command \"Get-Disk | Where-Object IsOffline -eq $True | Set-Disk -IsOffline $False\""); 5662 } 5663 5664 fn disk_file_put(&self, fname: &str, data: &str) { 5665 let _ = self.ssh_cmd(&format!( 5666 "powershell -Command \"'{}' | Set-Content -Path {}\"", 5667 data, fname 5668 )); 5669 } 5670 5671 fn disk_file_read(&self, fname: &str) -> String { 5672 self.ssh_cmd(&format!( 5673 "powershell -Command \"Get-Content -Path {}\"", 5674 fname 5675 )) 5676 } 5677 5678 fn wait_for_boot(&self) -> bool { 5679 let cmd = "dir /b c:\\ | find \"Windows\""; 5680 let tmo_max = 180; 5681 // The timeout increase by n*1+n*2+n*3+..., therefore the initial 5682 // interval must be small. 5683 let tmo_int = 2; 5684 let out = ssh_command_ip_with_auth( 5685 cmd, 5686 &self.auth, 5687 &self.guest.network.guest_ip, 5688 { 5689 let mut ret = 1; 5690 let mut tmo_acc = 0; 5691 loop { 5692 tmo_acc += tmo_int * ret; 5693 if tmo_acc >= tmo_max { 5694 break; 5695 } 5696 ret += 1; 5697 } 5698 ret 5699 }, 5700 tmo_int, 5701 ) 5702 .unwrap(); 5703 5704 if "Windows" == out.trim() { 5705 return true; 5706 } 5707 5708 false 5709 } 5710 } 5711 5712 fn vcpu_threads_count(pid: u32) -> u8 { 5713 // ps -T -p 12345 | grep vcpu | wc -l 5714 let out = Command::new("ps") 5715 .args(&["-T", "-p", format!("{}", pid).as_str()]) 5716 .output() 5717 .expect("ps command failed") 5718 .stdout; 5719 return String::from_utf8_lossy(&out).matches("vcpu").count() as u8; 5720 } 5721 5722 fn netdev_ctrl_threads_count(pid: u32) -> u8 { 5723 // ps -T -p 12345 | grep "_net[0-9]*_ctrl" | wc -l 5724 let out = Command::new("ps") 5725 .args(&["-T", "-p", format!("{}", pid).as_str()]) 5726 .output() 5727 .expect("ps command failed") 5728 .stdout; 5729 let mut n = 0; 5730 String::from_utf8_lossy(&out) 5731 .split_whitespace() 5732 .for_each(|s| n += (s.starts_with("_net") && s.ends_with("_ctrl")) as u8); // _net1_ctrl 5733 n 5734 } 5735 5736 fn disk_ctrl_threads_count(pid: u32) -> u8 { 5737 // ps -T -p 15782 | grep "_disk[0-9]*_q0" | wc -l 5738 let out = Command::new("ps") 5739 .args(&["-T", "-p", format!("{}", pid).as_str()]) 5740 .output() 5741 .expect("ps command failed") 5742 .stdout; 5743 let mut n = 0; 5744 String::from_utf8_lossy(&out) 5745 .split_whitespace() 5746 .for_each(|s| n += (s.starts_with("_disk") && s.ends_with("_q0")) as u8); // _disk0_q0, don't care about multiple queues as they're related to the same hdd 5747 n 5748 } 5749 5750 #[test] 5751 fn test_windows_guest() { 5752 let windows_guest = WindowsGuest::new(); 5753 5754 let mut ovmf_path = dirs::home_dir().unwrap(); 5755 ovmf_path.push("workloads"); 5756 ovmf_path.push(OVMF_NAME); 5757 5758 let mut child = GuestCommand::new(windows_guest.guest()) 5759 .args(&["--cpus", "boot=2,kvm_hyperv=on"]) 5760 .args(&["--memory", "size=4G"]) 5761 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 5762 .args(&["--serial", "tty"]) 5763 .args(&["--console", "off"]) 5764 .default_disks() 5765 .default_net() 5766 .capture_output() 5767 .spawn() 5768 .unwrap(); 5769 5770 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 5771 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 5772 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 5773 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 5774 5775 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 5776 5777 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 5778 5779 let r = std::panic::catch_unwind(|| { 5780 // Wait to make sure Windows boots up 5781 assert!(windows_guest.wait_for_boot()); 5782 5783 windows_guest.shutdown(); 5784 }); 5785 5786 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 5787 let _ = child.kill(); 5788 let output = child.wait_with_output().unwrap(); 5789 5790 let _ = child_dnsmasq.kill(); 5791 let _ = child_dnsmasq.wait(); 5792 5793 handle_child_output(r, &output); 5794 } 5795 5796 #[test] 5797 fn test_windows_guest_multiple_queues() { 5798 let windows_guest = WindowsGuest::new(); 5799 5800 let mut ovmf_path = dirs::home_dir().unwrap(); 5801 ovmf_path.push("workloads"); 5802 ovmf_path.push(OVMF_NAME); 5803 5804 let mut child = GuestCommand::new(windows_guest.guest()) 5805 .args(&["--cpus", "boot=4,kvm_hyperv=on"]) 5806 .args(&["--memory", "size=4G"]) 5807 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 5808 .args(&["--serial", "tty"]) 5809 .args(&["--console", "off"]) 5810 .args(&[ 5811 "--disk", 5812 format!( 5813 "path={},num_queues=4", 5814 windows_guest 5815 .guest() 5816 .disk_config 5817 .disk(DiskType::OperatingSystem) 5818 .unwrap() 5819 ) 5820 .as_str(), 5821 ]) 5822 .args(&[ 5823 "--net", 5824 format!( 5825 "tap=,mac={},ip={},mask=255.255.255.0,num_queues=8", 5826 windows_guest.guest().network.guest_mac, 5827 windows_guest.guest().network.host_ip 5828 ) 5829 .as_str(), 5830 ]) 5831 .capture_output() 5832 .spawn() 5833 .unwrap(); 5834 5835 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 5836 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 5837 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 5838 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 5839 5840 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 5841 5842 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 5843 5844 let r = std::panic::catch_unwind(|| { 5845 // Wait to make sure Windows boots up 5846 assert!(windows_guest.wait_for_boot()); 5847 5848 windows_guest.shutdown(); 5849 }); 5850 5851 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 5852 let _ = child.kill(); 5853 let output = child.wait_with_output().unwrap(); 5854 5855 let _ = child_dnsmasq.kill(); 5856 let _ = child_dnsmasq.wait(); 5857 5858 handle_child_output(r, &output); 5859 } 5860 5861 #[test] 5862 #[cfg(not(feature = "mshv"))] 5863 fn test_windows_guest_snapshot_restore() { 5864 let windows_guest = WindowsGuest::new(); 5865 5866 let mut ovmf_path = dirs::home_dir().unwrap(); 5867 ovmf_path.push("workloads"); 5868 ovmf_path.push(OVMF_NAME); 5869 5870 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 5871 let api_socket = temp_api_path(&tmp_dir); 5872 5873 let mut child = GuestCommand::new(windows_guest.guest()) 5874 .args(&["--api-socket", &api_socket]) 5875 .args(&["--cpus", "boot=2,kvm_hyperv=on"]) 5876 .args(&["--memory", "size=4G"]) 5877 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 5878 .args(&["--serial", "tty"]) 5879 .args(&["--console", "off"]) 5880 .default_disks() 5881 .default_net() 5882 .capture_output() 5883 .spawn() 5884 .unwrap(); 5885 5886 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 5887 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 5888 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 5889 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 5890 5891 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 5892 5893 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 5894 5895 // Wait to make sure Windows boots up 5896 assert!(windows_guest.wait_for_boot()); 5897 5898 let snapshot_dir = temp_snapshot_dir_path(&tmp_dir); 5899 5900 // Pause the VM 5901 assert!(remote_command(&api_socket, "pause", None)); 5902 5903 // Take a snapshot from the VM 5904 assert!(remote_command( 5905 &api_socket, 5906 "snapshot", 5907 Some(format!("file://{}", snapshot_dir).as_str()), 5908 )); 5909 5910 // Wait to make sure the snapshot is completed 5911 thread::sleep(std::time::Duration::new(30, 0)); 5912 5913 let _ = child.kill(); 5914 child.wait().unwrap(); 5915 5916 // Restore the VM from the snapshot 5917 let mut child = GuestCommand::new(windows_guest.guest()) 5918 .args(&["--api-socket", &api_socket]) 5919 .args(&[ 5920 "--restore", 5921 format!("source_url=file://{}", snapshot_dir).as_str(), 5922 ]) 5923 .capture_output() 5924 .spawn() 5925 .unwrap(); 5926 5927 // Wait for the VM to be restored 5928 thread::sleep(std::time::Duration::new(20, 0)); 5929 5930 let r = std::panic::catch_unwind(|| { 5931 // Resume the VM 5932 assert!(remote_command(&api_socket, "resume", None)); 5933 5934 windows_guest.shutdown(); 5935 }); 5936 5937 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 5938 let _ = child.kill(); 5939 let output = child.wait_with_output().unwrap(); 5940 5941 let _ = child_dnsmasq.kill(); 5942 let _ = child_dnsmasq.wait(); 5943 5944 handle_child_output(r, &output); 5945 } 5946 5947 #[test] 5948 #[cfg(not(feature = "mshv"))] 5949 fn test_windows_guest_cpu_hotplug() { 5950 let windows_guest = WindowsGuest::new(); 5951 5952 let mut ovmf_path = dirs::home_dir().unwrap(); 5953 ovmf_path.push("workloads"); 5954 ovmf_path.push(OVMF_NAME); 5955 5956 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 5957 let api_socket = temp_api_path(&tmp_dir); 5958 5959 let mut child = GuestCommand::new(windows_guest.guest()) 5960 .args(&["--api-socket", &api_socket]) 5961 .args(&["--cpus", "boot=2,max=8,kvm_hyperv=on"]) 5962 .args(&["--memory", "size=4G"]) 5963 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 5964 .args(&["--serial", "tty"]) 5965 .args(&["--console", "off"]) 5966 .default_disks() 5967 .default_net() 5968 .capture_output() 5969 .spawn() 5970 .unwrap(); 5971 5972 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 5973 5974 let r = std::panic::catch_unwind(|| { 5975 // Wait to make sure Windows boots up 5976 assert!(windows_guest.wait_for_boot()); 5977 5978 let vcpu_num = 2; 5979 // Check the initial number of CPUs the guest sees 5980 assert_eq!(windows_guest.cpu_count(), vcpu_num); 5981 // Check the initial number of vcpu threads in the CH process 5982 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 5983 5984 let vcpu_num = 6; 5985 // Hotplug some CPUs 5986 resize_command(&api_socket, Some(vcpu_num), None, None); 5987 // Wait to make sure CPUs are added 5988 thread::sleep(std::time::Duration::new(10, 0)); 5989 // Check the guest sees the correct number 5990 assert_eq!(windows_guest.cpu_count(), vcpu_num); 5991 // Check the CH process has the correct number of vcpu threads 5992 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 5993 5994 let vcpu_num = 4; 5995 // Remove some CPUs. Note that Windows doesn't support hot-remove. 5996 resize_command(&api_socket, Some(vcpu_num), None, None); 5997 // Wait to make sure CPUs are removed 5998 thread::sleep(std::time::Duration::new(10, 0)); 5999 // Reboot to let Windows catch up 6000 windows_guest.reboot(); 6001 // Wait to make sure Windows completely rebooted 6002 thread::sleep(std::time::Duration::new(60, 0)); 6003 // Check the guest sees the correct number 6004 assert_eq!(windows_guest.cpu_count(), vcpu_num); 6005 // Check the CH process has the correct number of vcpu threads 6006 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 6007 6008 windows_guest.shutdown(); 6009 }); 6010 6011 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 6012 let _ = child.kill(); 6013 let output = child.wait_with_output().unwrap(); 6014 6015 let _ = child_dnsmasq.kill(); 6016 let _ = child_dnsmasq.wait(); 6017 6018 handle_child_output(r, &output); 6019 } 6020 6021 #[test] 6022 #[cfg(not(feature = "mshv"))] 6023 fn test_windows_guest_ram_hotplug() { 6024 let windows_guest = WindowsGuest::new(); 6025 6026 let mut ovmf_path = dirs::home_dir().unwrap(); 6027 ovmf_path.push("workloads"); 6028 ovmf_path.push(OVMF_NAME); 6029 6030 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 6031 let api_socket = temp_api_path(&tmp_dir); 6032 6033 let mut child = GuestCommand::new(windows_guest.guest()) 6034 .args(&["--api-socket", &api_socket]) 6035 .args(&["--cpus", "boot=2,kvm_hyperv=on"]) 6036 .args(&["--memory", "size=2G,hotplug_size=5G"]) 6037 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 6038 .args(&["--serial", "tty"]) 6039 .args(&["--console", "off"]) 6040 .default_disks() 6041 .default_net() 6042 .capture_output() 6043 .spawn() 6044 .unwrap(); 6045 6046 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 6047 6048 let r = std::panic::catch_unwind(|| { 6049 // Wait to make sure Windows boots up 6050 assert!(windows_guest.wait_for_boot()); 6051 6052 let ram_size = 2 * 1024 * 1024 * 1024; 6053 // Check the initial number of RAM the guest sees 6054 let current_ram_size = windows_guest.ram_size(); 6055 // This size seems to be reserved by the system and thus the 6056 // reported amount differs by this constant value. 6057 let reserved_ram_size = ram_size - current_ram_size; 6058 // Verify that there's not more than 4mb constant diff wasted 6059 // by the reserved ram. 6060 assert!(reserved_ram_size < 4 * 1024 * 1024); 6061 6062 let ram_size = 4 * 1024 * 1024 * 1024; 6063 // Hotplug some RAM 6064 resize_command(&api_socket, None, Some(ram_size), None); 6065 // Wait to make sure RAM has been added 6066 thread::sleep(std::time::Duration::new(10, 0)); 6067 // Check the guest sees the correct number 6068 assert_eq!(windows_guest.ram_size(), ram_size - reserved_ram_size); 6069 6070 let ram_size = 3 * 1024 * 1024 * 1024; 6071 // Unplug some RAM. Note that hot-remove most likely won't work. 6072 resize_command(&api_socket, None, Some(ram_size), None); 6073 // Wait to make sure RAM has been added 6074 thread::sleep(std::time::Duration::new(10, 0)); 6075 // Reboot to let Windows catch up 6076 windows_guest.reboot(); 6077 // Wait to make sure guest completely rebooted 6078 thread::sleep(std::time::Duration::new(60, 0)); 6079 // Check the guest sees the correct number 6080 assert_eq!(windows_guest.ram_size(), ram_size - reserved_ram_size); 6081 6082 windows_guest.shutdown(); 6083 }); 6084 6085 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 6086 let _ = child.kill(); 6087 let output = child.wait_with_output().unwrap(); 6088 6089 let _ = child_dnsmasq.kill(); 6090 let _ = child_dnsmasq.wait(); 6091 6092 handle_child_output(r, &output); 6093 } 6094 6095 #[test] 6096 #[cfg(not(feature = "mshv"))] 6097 fn test_windows_guest_netdev_hotplug() { 6098 let windows_guest = WindowsGuest::new(); 6099 6100 let mut ovmf_path = dirs::home_dir().unwrap(); 6101 ovmf_path.push("workloads"); 6102 ovmf_path.push(OVMF_NAME); 6103 6104 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 6105 let api_socket = temp_api_path(&tmp_dir); 6106 6107 let mut child = GuestCommand::new(windows_guest.guest()) 6108 .args(&["--api-socket", &api_socket]) 6109 .args(&["--cpus", "boot=2,kvm_hyperv=on"]) 6110 .args(&["--memory", "size=4G"]) 6111 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 6112 .args(&["--serial", "tty"]) 6113 .args(&["--console", "off"]) 6114 .default_disks() 6115 .default_net() 6116 .capture_output() 6117 .spawn() 6118 .unwrap(); 6119 6120 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 6121 6122 let r = std::panic::catch_unwind(|| { 6123 // Wait to make sure Windows boots up 6124 assert!(windows_guest.wait_for_boot()); 6125 6126 // Initially present network device 6127 let netdev_num = 1; 6128 assert_eq!(windows_guest.netdev_count(), netdev_num); 6129 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 6130 6131 // Hotplug network device 6132 let (cmd_success, cmd_output) = remote_command_w_output( 6133 &api_socket, 6134 "add-net", 6135 Some(windows_guest.guest().default_net_string().as_str()), 6136 ); 6137 assert!(cmd_success); 6138 assert!(String::from_utf8_lossy(&cmd_output).contains("\"id\":\"_net2\"")); 6139 thread::sleep(std::time::Duration::new(5, 0)); 6140 // Verify the device is on the system 6141 let netdev_num = 2; 6142 assert_eq!(windows_guest.netdev_count(), netdev_num); 6143 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 6144 6145 // Remove network device 6146 let cmd_success = remote_command(&api_socket, "remove-device", Some("_net2")); 6147 assert!(cmd_success); 6148 thread::sleep(std::time::Duration::new(5, 0)); 6149 // Verify the device has been removed 6150 let netdev_num = 1; 6151 assert_eq!(windows_guest.netdev_count(), netdev_num); 6152 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 6153 6154 windows_guest.shutdown(); 6155 }); 6156 6157 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 6158 let _ = child.kill(); 6159 let output = child.wait_with_output().unwrap(); 6160 6161 let _ = child_dnsmasq.kill(); 6162 let _ = child_dnsmasq.wait(); 6163 6164 handle_child_output(r, &output); 6165 } 6166 6167 #[test] 6168 #[cfg(not(feature = "mshv"))] 6169 fn test_windows_guest_disk_hotplug() { 6170 let windows_guest = WindowsGuest::new(); 6171 6172 let mut ovmf_path = dirs::home_dir().unwrap(); 6173 ovmf_path.push("workloads"); 6174 ovmf_path.push(OVMF_NAME); 6175 6176 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 6177 let api_socket = temp_api_path(&tmp_dir); 6178 6179 let mut child = GuestCommand::new(windows_guest.guest()) 6180 .args(&["--api-socket", &api_socket]) 6181 .args(&["--cpus", "boot=2,kvm_hyperv=on"]) 6182 .args(&["--memory", "size=4G"]) 6183 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 6184 .args(&["--serial", "tty"]) 6185 .args(&["--console", "off"]) 6186 .default_disks() 6187 .default_net() 6188 .capture_output() 6189 .spawn() 6190 .unwrap(); 6191 6192 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 6193 6194 let disk = windows_guest.disk_new(WindowsGuest::FS_FAT, 100); 6195 6196 let r = std::panic::catch_unwind(|| { 6197 // Wait to make sure Windows boots up 6198 assert!(windows_guest.wait_for_boot()); 6199 6200 // Initially present disk device 6201 let disk_num = 1; 6202 assert_eq!(windows_guest.disk_count(), disk_num); 6203 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 6204 6205 // Hotplug disk device 6206 let (cmd_success, cmd_output) = remote_command_w_output( 6207 &api_socket, 6208 "add-disk", 6209 Some(format!("path={},readonly=off", disk).as_str()), 6210 ); 6211 assert!(cmd_success); 6212 assert!(String::from_utf8_lossy(&cmd_output).contains("\"id\":\"_disk2\"")); 6213 thread::sleep(std::time::Duration::new(5, 0)); 6214 // Online disk device 6215 windows_guest.disks_set_rw(); 6216 windows_guest.disks_online(); 6217 // Verify the device is on the system 6218 let disk_num = 2; 6219 assert_eq!(windows_guest.disk_count(), disk_num); 6220 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 6221 6222 let data = "hello"; 6223 let fname = "d:\\world"; 6224 windows_guest.disk_file_put(fname, data); 6225 6226 // Unmount disk device 6227 let cmd_success = remote_command(&api_socket, "remove-device", Some("_disk2")); 6228 assert!(cmd_success); 6229 thread::sleep(std::time::Duration::new(5, 0)); 6230 // Verify the device has been removed 6231 let disk_num = 1; 6232 assert_eq!(windows_guest.disk_count(), disk_num); 6233 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 6234 6235 // Remount and check the file exists with the expected contents 6236 let (cmd_success, _cmd_output) = remote_command_w_output( 6237 &api_socket, 6238 "add-disk", 6239 Some(format!("path={},readonly=off", disk).as_str()), 6240 ); 6241 assert!(cmd_success); 6242 thread::sleep(std::time::Duration::new(5, 0)); 6243 let out = windows_guest.disk_file_read(fname); 6244 assert_eq!(data, out.trim()); 6245 6246 // Intentionally no unmount, it'll happen at shutdown. 6247 6248 windows_guest.shutdown(); 6249 }); 6250 6251 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 6252 let _ = child.kill(); 6253 let output = child.wait_with_output().unwrap(); 6254 6255 let _ = child_dnsmasq.kill(); 6256 let _ = child_dnsmasq.wait(); 6257 6258 handle_child_output(r, &output); 6259 } 6260 6261 #[test] 6262 #[cfg(not(feature = "mshv"))] 6263 fn test_windows_guest_disk_hotplug_multi() { 6264 let windows_guest = WindowsGuest::new(); 6265 6266 let mut ovmf_path = dirs::home_dir().unwrap(); 6267 ovmf_path.push("workloads"); 6268 ovmf_path.push(OVMF_NAME); 6269 6270 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 6271 let api_socket = temp_api_path(&tmp_dir); 6272 6273 let mut child = GuestCommand::new(windows_guest.guest()) 6274 .args(&["--api-socket", &api_socket]) 6275 .args(&["--cpus", "boot=2,kvm_hyperv=on"]) 6276 .args(&["--memory", "size=2G"]) 6277 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 6278 .args(&["--serial", "tty"]) 6279 .args(&["--console", "off"]) 6280 .default_disks() 6281 .default_net() 6282 .capture_output() 6283 .spawn() 6284 .unwrap(); 6285 6286 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 6287 6288 // Predefined data to used at various test stages 6289 let disk_test_data: [[String; 4]; 2] = [ 6290 [ 6291 "_disk2".to_string(), 6292 windows_guest.disk_new(WindowsGuest::FS_FAT, 123), 6293 "d:\\world".to_string(), 6294 "hello".to_string(), 6295 ], 6296 [ 6297 "_disk3".to_string(), 6298 windows_guest.disk_new(WindowsGuest::FS_NTFS, 333), 6299 "e:\\hello".to_string(), 6300 "world".to_string(), 6301 ], 6302 ]; 6303 6304 let r = std::panic::catch_unwind(|| { 6305 // Wait to make sure Windows boots up 6306 assert!(windows_guest.wait_for_boot()); 6307 6308 // Initially present disk device 6309 let disk_num = 1; 6310 assert_eq!(windows_guest.disk_count(), disk_num); 6311 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 6312 6313 for it in &disk_test_data { 6314 let disk_id = it[0].as_str(); 6315 let disk = it[1].as_str(); 6316 // Hotplug disk device 6317 let (cmd_success, cmd_output) = remote_command_w_output( 6318 &api_socket, 6319 "add-disk", 6320 Some(format!("path={},readonly=off", disk).as_str()), 6321 ); 6322 assert!(cmd_success); 6323 assert!(String::from_utf8_lossy(&cmd_output) 6324 .contains(format!("\"id\":\"{}\"", disk_id).as_str())); 6325 thread::sleep(std::time::Duration::new(5, 0)); 6326 // Online disk devices 6327 windows_guest.disks_set_rw(); 6328 windows_guest.disks_online(); 6329 } 6330 // Verify the devices are on the system 6331 let disk_num = (disk_test_data.len() + 1) as u8; 6332 assert_eq!(windows_guest.disk_count(), disk_num); 6333 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 6334 6335 // Put test data 6336 for it in &disk_test_data { 6337 let fname = it[2].as_str(); 6338 let data = it[3].as_str(); 6339 windows_guest.disk_file_put(fname, data); 6340 } 6341 6342 // Unmount disk devices 6343 for it in &disk_test_data { 6344 let disk_id = it[0].as_str(); 6345 let cmd_success = remote_command(&api_socket, "remove-device", Some(disk_id)); 6346 assert!(cmd_success); 6347 thread::sleep(std::time::Duration::new(5, 0)); 6348 } 6349 6350 // Verify the devices have been removed 6351 let disk_num = 1; 6352 assert_eq!(windows_guest.disk_count(), disk_num); 6353 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 6354 6355 // Remount 6356 for it in &disk_test_data { 6357 let disk = it[1].as_str(); 6358 let (cmd_success, _cmd_output) = remote_command_w_output( 6359 &api_socket, 6360 "add-disk", 6361 Some(format!("path={},readonly=off", disk).as_str()), 6362 ); 6363 assert!(cmd_success); 6364 thread::sleep(std::time::Duration::new(5, 0)); 6365 } 6366 6367 // Check the files exists with the expected contents 6368 for it in &disk_test_data { 6369 let fname = it[2].as_str(); 6370 let data = it[3].as_str(); 6371 let out = windows_guest.disk_file_read(fname); 6372 assert_eq!(data, out.trim()); 6373 } 6374 6375 // Intentionally no unmount, it'll happen at shutdown. 6376 6377 windows_guest.shutdown(); 6378 }); 6379 6380 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 6381 let _ = child.kill(); 6382 let output = child.wait_with_output().unwrap(); 6383 6384 let _ = child_dnsmasq.kill(); 6385 let _ = child_dnsmasq.wait(); 6386 6387 handle_child_output(r, &output); 6388 } 6389 6390 #[test] 6391 #[cfg(not(feature = "mshv"))] 6392 fn test_windows_guest_netdev_multi() { 6393 let windows_guest = WindowsGuest::new(); 6394 6395 let mut ovmf_path = dirs::home_dir().unwrap(); 6396 ovmf_path.push("workloads"); 6397 ovmf_path.push(OVMF_NAME); 6398 6399 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 6400 let api_socket = temp_api_path(&tmp_dir); 6401 6402 let mut child = GuestCommand::new(windows_guest.guest()) 6403 .args(&["--api-socket", &api_socket]) 6404 .args(&["--cpus", "boot=2,kvm_hyperv=on"]) 6405 .args(&["--memory", "size=4G"]) 6406 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 6407 .args(&["--serial", "tty"]) 6408 .args(&["--console", "off"]) 6409 .default_disks() 6410 // The multi net dev config is borrowed from test_multiple_network_interfaces 6411 .args(&[ 6412 "--net", 6413 windows_guest.guest().default_net_string().as_str(), 6414 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 6415 "tap=mytap42,mac=fe:1f:9e:e1:60:f2,ip=192.168.4.1,mask=255.255.255.0", 6416 ]) 6417 .capture_output() 6418 .spawn() 6419 .unwrap(); 6420 6421 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 6422 6423 let r = std::panic::catch_unwind(|| { 6424 // Wait to make sure Windows boots up 6425 assert!(windows_guest.wait_for_boot()); 6426 6427 let netdev_num = 3; 6428 assert_eq!(windows_guest.netdev_count(), netdev_num); 6429 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 6430 6431 let tap_count = exec_host_command_output("ip link | grep -c mytap42"); 6432 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 6433 6434 windows_guest.shutdown(); 6435 }); 6436 6437 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 6438 let _ = child.kill(); 6439 let output = child.wait_with_output().unwrap(); 6440 6441 let _ = child_dnsmasq.kill(); 6442 let _ = child_dnsmasq.wait(); 6443 6444 handle_child_output(r, &output); 6445 } 6446 } 6447 6448 #[cfg(target_arch = "x86_64")] 6449 mod sgx { 6450 use crate::tests::*; 6451 6452 #[test] 6453 fn test_sgx() { 6454 let focal = UbuntuDiskConfig::new(FOCAL_SGX_IMAGE_NAME.to_string()); 6455 let guest = Guest::new(Box::new(focal)); 6456 let mut workload_path = dirs::home_dir().unwrap(); 6457 workload_path.push("workloads"); 6458 6459 let mut kernel_path = workload_path; 6460 kernel_path.push("vmlinux_w_sgx"); 6461 6462 let mut child = GuestCommand::new(&guest) 6463 .args(&["--cpus", "boot=1"]) 6464 .args(&["--memory", "size=512M"]) 6465 .args(&["--kernel", kernel_path.to_str().unwrap()]) 6466 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6467 .default_disks() 6468 .default_net() 6469 .args(&["--sgx-epc", "id=epc0,size=64M"]) 6470 .capture_output() 6471 .spawn() 6472 .unwrap(); 6473 6474 let r = std::panic::catch_unwind(|| { 6475 guest.wait_vm_boot(None).unwrap(); 6476 6477 // Check if SGX is correctly detected in the guest. 6478 guest.check_sgx_support().unwrap(); 6479 6480 // Validate the SGX EPC section is 64MiB. 6481 assert_eq!( 6482 guest 6483 .ssh_command("cpuid -l 0x12 -s 2 | grep 'section size' | cut -d '=' -f 2") 6484 .unwrap() 6485 .trim(), 6486 "0x0000000004000000" 6487 ); 6488 6489 // Run a test relying on SGX enclaves and check if it runs 6490 // successfully. 6491 assert!(guest 6492 .ssh_command("cd /linux-sgx/SampleCode/LocalAttestation/bin/ && sudo ./app") 6493 .unwrap() 6494 .trim() 6495 .contains( 6496 "succeed to load enclaves.\nsucceed to \ 6497 establish secure channel.\nSucceed to exchange \ 6498 secure message...\nSucceed to close Session..." 6499 )); 6500 }); 6501 6502 let _ = child.kill(); 6503 let output = child.wait_with_output().unwrap(); 6504 6505 handle_child_output(r, &output); 6506 } 6507 } 6508 6509 #[cfg(target_arch = "x86_64")] 6510 mod vfio { 6511 use crate::tests::*; 6512 6513 fn test_nvidia_card_memory_hotplug(hotplug_method: &str) { 6514 let hirsute = UbuntuDiskConfig::new(HIRSUTE_NVIDIA_IMAGE_NAME.to_string()); 6515 let guest = Guest::new(Box::new(hirsute)); 6516 let api_socket = temp_api_path(&guest.tmp_dir); 6517 6518 let mut child = GuestCommand::new(&guest) 6519 .args(&["--cpus", "boot=4"]) 6520 .args(&[ 6521 "--memory", 6522 format!("size=4G,hotplug_size=4G,hotplug_method={}", hotplug_method).as_str(), 6523 ]) 6524 .args(&["--kernel", guest.fw_path.as_str()]) 6525 .args(&["--device", "path=/sys/bus/pci/devices/0000:31:00.0/"]) 6526 .args(&["--api-socket", &api_socket]) 6527 .default_disks() 6528 .default_net() 6529 .capture_output() 6530 .spawn() 6531 .unwrap(); 6532 6533 let r = std::panic::catch_unwind(|| { 6534 guest.wait_vm_boot(None).unwrap(); 6535 6536 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 6537 6538 guest.enable_memory_hotplug(); 6539 6540 // Add RAM to the VM 6541 let desired_ram = 6 << 30; 6542 resize_command(&api_socket, None, Some(desired_ram), None); 6543 thread::sleep(std::time::Duration::new(30, 0)); 6544 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 6545 6546 // Check the VFIO device works when RAM is increased to 6GiB 6547 guest.check_nvidia_gpu(); 6548 }); 6549 6550 let _ = child.kill(); 6551 let output = child.wait_with_output().unwrap(); 6552 6553 handle_child_output(r, &output); 6554 } 6555 6556 #[test] 6557 fn test_nvidia_card_memory_hotplug_acpi() { 6558 test_nvidia_card_memory_hotplug("acpi") 6559 } 6560 6561 #[test] 6562 fn test_nvidia_card_memory_hotplug_virtio_mem() { 6563 test_nvidia_card_memory_hotplug("virtio-mem") 6564 } 6565 6566 #[test] 6567 fn test_nvidia_card_pci_hotplug() { 6568 let hirsute = UbuntuDiskConfig::new(HIRSUTE_NVIDIA_IMAGE_NAME.to_string()); 6569 let guest = Guest::new(Box::new(hirsute)); 6570 let api_socket = temp_api_path(&guest.tmp_dir); 6571 6572 let mut child = GuestCommand::new(&guest) 6573 .args(&["--cpus", "boot=4"]) 6574 .args(&["--memory", "size=4G"]) 6575 .args(&["--kernel", guest.fw_path.as_str()]) 6576 .args(&["--api-socket", &api_socket]) 6577 .default_disks() 6578 .default_net() 6579 .capture_output() 6580 .spawn() 6581 .unwrap(); 6582 6583 let r = std::panic::catch_unwind(|| { 6584 guest.wait_vm_boot(None).unwrap(); 6585 6586 // Hotplug the card to the VM 6587 let (cmd_success, cmd_output) = remote_command_w_output( 6588 &api_socket, 6589 "add-device", 6590 Some("id=vfio0,path=/sys/bus/pci/devices/0000:31:00.0/"), 6591 ); 6592 assert!(cmd_success); 6593 assert!(String::from_utf8_lossy(&cmd_output) 6594 .contains("{\"id\":\"vfio0\",\"bdf\":\"0000:00:06.0\"}")); 6595 6596 thread::sleep(std::time::Duration::new(10, 0)); 6597 6598 // Check the VFIO device works after hotplug 6599 guest.check_nvidia_gpu(); 6600 }); 6601 6602 let _ = child.kill(); 6603 let output = child.wait_with_output().unwrap(); 6604 6605 handle_child_output(r, &output); 6606 } 6607 6608 #[test] 6609 fn test_nvidia_card_reboot() { 6610 let hirsute = UbuntuDiskConfig::new(HIRSUTE_NVIDIA_IMAGE_NAME.to_string()); 6611 let guest = Guest::new(Box::new(hirsute)); 6612 let api_socket = temp_api_path(&guest.tmp_dir); 6613 6614 let mut child = GuestCommand::new(&guest) 6615 .args(&["--cpus", "boot=4"]) 6616 .args(&["--memory", "size=4G"]) 6617 .args(&["--kernel", guest.fw_path.as_str()]) 6618 .args(&["--device", "path=/sys/bus/pci/devices/0000:31:00.0/"]) 6619 .args(&["--api-socket", &api_socket]) 6620 .default_disks() 6621 .default_net() 6622 .capture_output() 6623 .spawn() 6624 .unwrap(); 6625 6626 let r = std::panic::catch_unwind(|| { 6627 guest.wait_vm_boot(None).unwrap(); 6628 6629 // Check the VFIO device works after boot 6630 guest.check_nvidia_gpu(); 6631 6632 guest.reboot_linux(0, None); 6633 6634 // Check the VFIO device works after reboot 6635 guest.check_nvidia_gpu(); 6636 }); 6637 6638 let _ = child.kill(); 6639 let output = child.wait_with_output().unwrap(); 6640 6641 handle_child_output(r, &output); 6642 } 6643 } 6644 } 6645