1 // Copyright © 2020 Intel Corporation 2 // 3 // SPDX-License-Identifier: Apache-2.0 4 // 5 6 #[cfg(test)] 7 #[cfg(feature = "integration_tests")] 8 #[macro_use] 9 extern crate lazy_static; 10 11 #[cfg(test)] 12 #[cfg(feature = "integration_tests")] 13 extern crate test_infra; 14 15 #[cfg(test)] 16 #[cfg(feature = "integration_tests")] 17 mod tests { 18 use net_util::MacAddr; 19 use std::collections::HashMap; 20 use std::env; 21 use std::ffi::OsStr; 22 use std::fs; 23 use std::io; 24 use std::io::BufRead; 25 use std::io::Read; 26 use std::io::Write; 27 use std::os::unix::io::AsRawFd; 28 use std::path::PathBuf; 29 use std::process::{Child, Command, Stdio}; 30 use std::string::String; 31 use std::sync::mpsc::Receiver; 32 use std::sync::{mpsc, Mutex}; 33 use std::thread; 34 use test_infra::*; 35 use vmm_sys_util::{tempdir::TempDir, tempfile::TempFile}; 36 #[cfg_attr(target_arch = "aarch64", allow(unused_imports))] 37 use wait_timeout::ChildExt; 38 39 lazy_static! { 40 static ref NEXT_VM_ID: Mutex<u8> = Mutex::new(1); 41 } 42 43 struct Guest { 44 tmp_dir: TempDir, 45 disk_config: Box<dyn DiskConfig>, 46 fw_path: String, 47 network: GuestNetworkConfig, 48 } 49 50 // Safe to implement as we know we have no interior mutability 51 impl std::panic::RefUnwindSafe for Guest {} 52 53 #[cfg(target_arch = "x86_64")] 54 const BIONIC_IMAGE_NAME: &str = "bionic-server-cloudimg-amd64.raw"; 55 #[cfg(target_arch = "x86_64")] 56 const FOCAL_IMAGE_NAME: &str = "focal-server-cloudimg-amd64-custom-20210609-0.raw"; 57 #[cfg(target_arch = "x86_64")] 58 const FOCAL_SGX_IMAGE_NAME: &str = "focal-server-cloudimg-amd64-sgx.raw"; 59 #[cfg(target_arch = "x86_64")] 60 const HIRSUTE_NVIDIA_IMAGE_NAME: &str = "hirsute-server-cloudimg-amd64-nvidia.raw"; 61 #[cfg(target_arch = "aarch64")] 62 const BIONIC_IMAGE_NAME: &str = "bionic-server-cloudimg-arm64.raw"; 63 #[cfg(target_arch = "aarch64")] 64 const FOCAL_IMAGE_NAME: &str = "focal-server-cloudimg-arm64-custom.raw"; 65 #[cfg(target_arch = "aarch64")] 66 const FOCAL_IMAGE_NAME_QCOW2: &str = "focal-server-cloudimg-arm64-custom.qcow2"; 67 #[cfg(target_arch = "x86_64")] 68 const FOCAL_IMAGE_NAME_QCOW2: &str = "focal-server-cloudimg-amd64-custom-20210609-0.qcow2"; 69 #[cfg(target_arch = "aarch64")] 70 const FOCAL_IMAGE_NAME_VHD: &str = "focal-server-cloudimg-arm64-custom.vhd"; 71 #[cfg(target_arch = "x86_64")] 72 const FOCAL_IMAGE_NAME_VHD: &str = "focal-server-cloudimg-amd64-custom-20210609-0.vhd"; 73 #[cfg(target_arch = "x86_64")] 74 const WINDOWS_IMAGE_NAME: &str = "windows-server-2019.raw"; 75 #[cfg(target_arch = "x86_64")] 76 const OVMF_NAME: &str = "OVMF-4b47d0c6c8.fd"; 77 78 const DIRECT_KERNEL_BOOT_CMDLINE: &str = 79 "root=/dev/vda1 console=hvc0 rw systemd.journald.forward_to_console=1"; 80 81 #[cfg(target_arch = "x86_64")] 82 const GREP_SERIAL_IRQ_CMD: &str = "grep -c 'IO-APIC.*ttyS0' /proc/interrupts || true"; 83 #[cfg(target_arch = "aarch64")] 84 const GREP_SERIAL_IRQ_CMD: &str = "grep -c 'GICv3.*uart-pl011' /proc/interrupts || true"; 85 86 const PIPE_SIZE: i32 = 32 << 20; 87 88 const CONSOLE_TEST_STRING: &str = "Started OpenBSD Secure Shell server"; 89 90 fn clh_command(cmd: &str) -> String { 91 env::var("BUILD_TARGET").map_or( 92 format!("target/x86_64-unknown-linux-gnu/release/{}", cmd), 93 |target| format!("target/{}/release/{}", target, cmd), 94 ) 95 } 96 97 fn prepare_virtiofsd( 98 tmp_dir: &TempDir, 99 shared_dir: &str, 100 cache: &str, 101 ) -> (std::process::Child, String) { 102 let mut workload_path = dirs::home_dir().unwrap(); 103 workload_path.push("workloads"); 104 105 let mut virtiofsd_path = workload_path; 106 virtiofsd_path.push("virtiofsd"); 107 let virtiofsd_path = String::from(virtiofsd_path.to_str().unwrap()); 108 109 let virtiofsd_socket_path = 110 String::from(tmp_dir.as_path().join("virtiofs.sock").to_str().unwrap()); 111 112 // Start the daemon 113 let child = Command::new(virtiofsd_path.as_str()) 114 .args(&[format!("--socket-path={}", virtiofsd_socket_path).as_str()]) 115 .args(&["-o", format!("source={}", shared_dir).as_str()]) 116 .args(&["-o", format!("cache={}", cache).as_str()]) 117 .spawn() 118 .unwrap(); 119 120 thread::sleep(std::time::Duration::new(10, 0)); 121 122 (child, virtiofsd_socket_path) 123 } 124 125 fn prepare_virtofsd_rs_daemon( 126 tmp_dir: &TempDir, 127 shared_dir: &str, 128 _cache: &str, 129 ) -> (std::process::Child, String) { 130 let mut workload_path = dirs::home_dir().unwrap(); 131 workload_path.push("workloads"); 132 133 let mut virtiofsd_path = workload_path; 134 virtiofsd_path.push("virtiofsd-rs"); 135 let virtiofsd_path = String::from(virtiofsd_path.to_str().unwrap()); 136 137 let virtiofsd_socket_path = 138 String::from(tmp_dir.as_path().join("virtiofs.sock").to_str().unwrap()); 139 140 // Start the daemon 141 let child = Command::new(virtiofsd_path.as_str()) 142 .args(&["--shared-dir", shared_dir]) 143 .args(&["--socket", virtiofsd_socket_path.as_str()]) 144 .spawn() 145 .unwrap(); 146 147 thread::sleep(std::time::Duration::new(10, 0)); 148 149 (child, virtiofsd_socket_path) 150 } 151 152 fn prepare_vubd( 153 tmp_dir: &TempDir, 154 blk_img: &str, 155 num_queues: usize, 156 rdonly: bool, 157 direct: bool, 158 ) -> (std::process::Child, String) { 159 let mut workload_path = dirs::home_dir().unwrap(); 160 workload_path.push("workloads"); 161 162 let mut blk_file_path = workload_path; 163 blk_file_path.push(blk_img); 164 let blk_file_path = String::from(blk_file_path.to_str().unwrap()); 165 166 let vubd_socket_path = String::from(tmp_dir.as_path().join("vub.sock").to_str().unwrap()); 167 168 // Start the daemon 169 let child = Command::new(clh_command("vhost_user_block")) 170 .args(&[ 171 "--block-backend", 172 format!( 173 "path={},socket={},num_queues={},readonly={},direct={}", 174 blk_file_path, vubd_socket_path, num_queues, rdonly, direct 175 ) 176 .as_str(), 177 ]) 178 .spawn() 179 .unwrap(); 180 181 thread::sleep(std::time::Duration::new(10, 0)); 182 183 (child, vubd_socket_path) 184 } 185 186 fn temp_vsock_path(tmp_dir: &TempDir) -> String { 187 String::from(tmp_dir.as_path().join("vsock").to_str().unwrap()) 188 } 189 190 fn temp_api_path(tmp_dir: &TempDir) -> String { 191 String::from( 192 tmp_dir 193 .as_path() 194 .join("cloud-hypervisor.sock") 195 .to_str() 196 .unwrap(), 197 ) 198 } 199 200 // Creates the directory and returns the path. 201 fn temp_snapshot_dir_path(tmp_dir: &TempDir) -> String { 202 let snapshot_dir = String::from(tmp_dir.as_path().join("snapshot").to_str().unwrap()); 203 std::fs::create_dir(&snapshot_dir).unwrap(); 204 snapshot_dir 205 } 206 207 // Creates the path for direct kernel boot and return the path. 208 // For x86_64, this function returns the vmlinux kernel path. 209 // For AArch64, this function returns the PE kernel path. 210 fn direct_kernel_boot_path() -> PathBuf { 211 let mut workload_path = dirs::home_dir().unwrap(); 212 workload_path.push("workloads"); 213 214 let mut kernel_path = workload_path; 215 #[cfg(target_arch = "x86_64")] 216 kernel_path.push("vmlinux"); 217 #[cfg(target_arch = "aarch64")] 218 kernel_path.push("Image"); 219 220 kernel_path 221 } 222 223 #[cfg(target_arch = "aarch64")] 224 fn edk2_path() -> PathBuf { 225 let mut workload_path = dirs::home_dir().unwrap(); 226 workload_path.push("workloads"); 227 let mut edk2_path = workload_path; 228 edk2_path.push("CLOUDHV_EFI.fd"); 229 230 edk2_path 231 } 232 233 fn prepare_vhost_user_net_daemon( 234 tmp_dir: &TempDir, 235 ip: &str, 236 tap: Option<&str>, 237 num_queues: usize, 238 client_mode: bool, 239 ) -> (std::process::Command, String) { 240 let vunet_socket_path = 241 String::from(tmp_dir.as_path().join("vunet.sock").to_str().unwrap()); 242 243 // Start the daemon 244 let net_params = if let Some(tap_str) = tap { 245 format!( 246 "tap={},ip={},mask=255.255.255.0,socket={},num_queues={},queue_size=1024,client={}", 247 tap_str, ip, vunet_socket_path, num_queues, client_mode 248 ) 249 } else { 250 format!( 251 "ip={},mask=255.255.255.0,socket={},num_queues={},queue_size=1024,client={}", 252 ip, vunet_socket_path, num_queues, client_mode 253 ) 254 }; 255 256 let mut command = Command::new(clh_command("vhost_user_net")); 257 command.args(&["--net-backend", net_params.as_str()]); 258 259 (command, vunet_socket_path) 260 } 261 262 fn curl_command(api_socket: &str, method: &str, url: &str, http_body: Option<&str>) { 263 let mut curl_args: Vec<&str> = 264 ["--unix-socket", api_socket, "-i", "-X", method, url].to_vec(); 265 266 if let Some(body) = http_body { 267 curl_args.push("-H"); 268 curl_args.push("Accept: application/json"); 269 curl_args.push("-H"); 270 curl_args.push("Content-Type: application/json"); 271 curl_args.push("-d"); 272 curl_args.push(body); 273 } 274 275 let status = Command::new("curl") 276 .args(curl_args) 277 .status() 278 .expect("Failed to launch curl command"); 279 280 assert!(status.success()); 281 } 282 283 fn remote_command(api_socket: &str, command: &str, arg: Option<&str>) -> bool { 284 let mut cmd = Command::new(clh_command("ch-remote")); 285 cmd.args(&[&format!("--api-socket={}", api_socket), command]); 286 287 if let Some(arg) = arg { 288 cmd.arg(arg); 289 } 290 291 cmd.status().expect("Failed to launch ch-remote").success() 292 } 293 294 fn remote_command_w_output( 295 api_socket: &str, 296 command: &str, 297 arg: Option<&str>, 298 ) -> (bool, Vec<u8>) { 299 let mut cmd = Command::new(clh_command("ch-remote")); 300 cmd.args(&[&format!("--api-socket={}", api_socket), command]); 301 302 if let Some(arg) = arg { 303 cmd.arg(arg); 304 } 305 306 let output = cmd.output().expect("Failed to launch ch-remote"); 307 308 (output.status.success(), output.stdout) 309 } 310 311 fn resize_command( 312 api_socket: &str, 313 desired_vcpus: Option<u8>, 314 desired_ram: Option<usize>, 315 desired_balloon: Option<usize>, 316 ) -> bool { 317 let mut cmd = Command::new(clh_command("ch-remote")); 318 cmd.args(&[&format!("--api-socket={}", api_socket), "resize"]); 319 320 if let Some(desired_vcpus) = desired_vcpus { 321 cmd.arg(format!("--cpus={}", desired_vcpus)); 322 } 323 324 if let Some(desired_ram) = desired_ram { 325 cmd.arg(format!("--memory={}", desired_ram)); 326 } 327 328 if let Some(desired_balloon) = desired_balloon { 329 cmd.arg(format!("--balloon={}", desired_balloon)); 330 } 331 332 cmd.status().expect("Failed to launch ch-remote").success() 333 } 334 335 fn resize_zone_command(api_socket: &str, id: &str, desired_size: &str) -> bool { 336 let mut cmd = Command::new(clh_command("ch-remote")); 337 cmd.args(&[ 338 &format!("--api-socket={}", api_socket), 339 "resize-zone", 340 &format!("--id={}", id), 341 &format!("--size={}", desired_size), 342 ]); 343 344 cmd.status().expect("Failed to launch ch-remote").success() 345 } 346 347 #[derive(Debug)] 348 enum Error { 349 Parsing(std::num::ParseIntError), 350 SshCommand(SshCommandError), 351 WaitForBoot(WaitForBootError), 352 } 353 354 impl From<SshCommandError> for Error { 355 fn from(e: SshCommandError) -> Self { 356 Self::SshCommand(e) 357 } 358 } 359 360 impl Guest { 361 fn new_from_ip_range(mut disk_config: Box<dyn DiskConfig>, class: &str, id: u8) -> Self { 362 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 363 364 let mut workload_path = dirs::home_dir().unwrap(); 365 workload_path.push("workloads"); 366 367 let mut fw_path = workload_path; 368 #[cfg(target_arch = "aarch64")] 369 fw_path.push("Image"); 370 #[cfg(target_arch = "x86_64")] 371 fw_path.push("hypervisor-fw"); 372 let fw_path = String::from(fw_path.to_str().unwrap()); 373 let network = GuestNetworkConfig { 374 guest_ip: format!("{}.{}.2", class, id), 375 l2_guest_ip1: format!("{}.{}.3", class, id), 376 l2_guest_ip2: format!("{}.{}.4", class, id), 377 l2_guest_ip3: format!("{}.{}.5", class, id), 378 host_ip: format!("{}.{}.1", class, id), 379 guest_mac: format!("12:34:56:78:90:{:02x}", id), 380 l2_guest_mac1: format!("de:ad:be:ef:12:{:02x}", id), 381 l2_guest_mac2: format!("de:ad:be:ef:34:{:02x}", id), 382 l2_guest_mac3: format!("de:ad:be:ef:56:{:02x}", id), 383 tcp_listener_port: DEFAULT_TCP_LISTENER_PORT + id as u16, 384 }; 385 386 disk_config.prepare_files(&tmp_dir, &network); 387 388 Guest { 389 tmp_dir, 390 disk_config, 391 fw_path, 392 network, 393 } 394 } 395 396 fn new(disk_config: Box<dyn DiskConfig>) -> Self { 397 let mut guard = NEXT_VM_ID.lock().unwrap(); 398 let id = *guard; 399 *guard = id + 1; 400 401 Self::new_from_ip_range(disk_config, "192.168", id) 402 } 403 404 fn default_net_string(&self) -> String { 405 format!( 406 "tap=,mac={},ip={},mask=255.255.255.0", 407 self.network.guest_mac, self.network.host_ip 408 ) 409 } 410 411 fn default_net_string_w_iommu(&self) -> String { 412 format!( 413 "tap=,mac={},ip={},mask=255.255.255.0,iommu=on", 414 self.network.guest_mac, self.network.host_ip 415 ) 416 } 417 418 fn ssh_command(&self, command: &str) -> Result<String, SshCommandError> { 419 ssh_command_ip( 420 command, 421 &self.network.guest_ip, 422 DEFAULT_SSH_RETRIES, 423 DEFAULT_SSH_TIMEOUT, 424 ) 425 } 426 427 fn ssh_command_l1(&self, command: &str) -> Result<String, SshCommandError> { 428 ssh_command_ip( 429 command, 430 &self.network.guest_ip, 431 DEFAULT_SSH_RETRIES, 432 DEFAULT_SSH_TIMEOUT, 433 ) 434 } 435 436 fn ssh_command_l2_1(&self, command: &str) -> Result<String, SshCommandError> { 437 ssh_command_ip( 438 command, 439 &self.network.l2_guest_ip1, 440 DEFAULT_SSH_RETRIES, 441 DEFAULT_SSH_TIMEOUT, 442 ) 443 } 444 445 fn ssh_command_l2_2(&self, command: &str) -> Result<String, SshCommandError> { 446 ssh_command_ip( 447 command, 448 &self.network.l2_guest_ip2, 449 DEFAULT_SSH_RETRIES, 450 DEFAULT_SSH_TIMEOUT, 451 ) 452 } 453 454 fn ssh_command_l2_3(&self, command: &str) -> Result<String, SshCommandError> { 455 ssh_command_ip( 456 command, 457 &self.network.l2_guest_ip3, 458 DEFAULT_SSH_RETRIES, 459 DEFAULT_SSH_TIMEOUT, 460 ) 461 } 462 463 fn api_create_body(&self, cpu_count: u8) -> String { 464 #[cfg(target_arch = "x86_64")] 465 format! {"{{\"cpus\":{{\"boot_vcpus\":{},\"max_vcpus\":{}}},\"kernel\":{{\"path\":\"{}\"}},\"cmdline\":{{\"args\": \"\"}},\"net\":[{{\"ip\":\"{}\", \"mask\":\"255.255.255.0\", \"mac\":\"{}\"}}], \"disks\":[{{\"path\":\"{}\"}}, {{\"path\":\"{}\"}}]}}", 466 cpu_count, 467 cpu_count, 468 self.fw_path.as_str(), 469 self.network.host_ip, 470 self.network.guest_mac, 471 self.disk_config.disk(DiskType::OperatingSystem).unwrap().as_str(), 472 self.disk_config.disk(DiskType::CloudInit).unwrap().as_str(), 473 } 474 #[cfg(target_arch = "aarch64")] 475 format! {"{{\"cpus\":{{\"boot_vcpus\":{},\"max_vcpus\":{}}},\"kernel\":{{\"path\":\"{}\"}},\"cmdline\":{{\"args\": \"{}\"}},\"net\":[{{\"ip\":\"{}\", \"mask\":\"255.255.255.0\", \"mac\":\"{}\"}}], \"disks\":[{{\"path\":\"{}\"}}, {{\"path\":\"{}\"}}]}}", 476 cpu_count, 477 cpu_count, 478 direct_kernel_boot_path().to_str().unwrap(), 479 DIRECT_KERNEL_BOOT_CMDLINE, 480 self.network.host_ip, 481 self.network.guest_mac, 482 self.disk_config.disk(DiskType::OperatingSystem).unwrap().as_str(), 483 self.disk_config.disk(DiskType::CloudInit).unwrap().as_str(), 484 } 485 } 486 487 fn get_cpu_count(&self) -> Result<u32, Error> { 488 self.ssh_command("grep -c processor /proc/cpuinfo")? 489 .trim() 490 .parse() 491 .map_err(Error::Parsing) 492 } 493 494 fn get_initial_apicid(&self) -> Result<u32, Error> { 495 self.ssh_command("grep \"initial apicid\" /proc/cpuinfo | grep -o \"[0-9]*\"")? 496 .trim() 497 .parse() 498 .map_err(Error::Parsing) 499 } 500 501 fn get_total_memory(&self) -> Result<u32, Error> { 502 self.ssh_command("grep MemTotal /proc/meminfo | grep -o \"[0-9]*\"")? 503 .trim() 504 .parse() 505 .map_err(Error::Parsing) 506 } 507 508 fn get_total_memory_l2(&self) -> Result<u32, Error> { 509 self.ssh_command_l2_1("grep MemTotal /proc/meminfo | grep -o \"[0-9]*\"")? 510 .trim() 511 .parse() 512 .map_err(Error::Parsing) 513 } 514 515 fn get_numa_node_memory(&self, node_id: usize) -> Result<u32, Error> { 516 self.ssh_command( 517 format!( 518 "grep MemTotal /sys/devices/system/node/node{}/meminfo \ 519 | cut -d \":\" -f 2 | grep -o \"[0-9]*\"", 520 node_id 521 ) 522 .as_str(), 523 )? 524 .trim() 525 .parse() 526 .map_err(Error::Parsing) 527 } 528 529 fn wait_vm_boot(&self, custom_timeout: Option<i32>) -> Result<(), Error> { 530 self.network 531 .wait_vm_boot(custom_timeout) 532 .map_err(Error::WaitForBoot) 533 } 534 535 fn check_numa_node_cpus(&self, node_id: usize, cpus: Vec<usize>) -> Result<(), Error> { 536 for cpu in cpus.iter() { 537 let cmd = format!( 538 "[ -d \"/sys/devices/system/node/node{}/cpu{}\" ]", 539 node_id, cpu 540 ); 541 self.ssh_command(cmd.as_str())?; 542 } 543 544 Ok(()) 545 } 546 547 fn check_numa_node_distances( 548 &self, 549 node_id: usize, 550 distances: &str, 551 ) -> Result<bool, Error> { 552 let cmd = format!("cat /sys/devices/system/node/node{}/distance", node_id); 553 if self.ssh_command(cmd.as_str())?.trim() == distances { 554 Ok(true) 555 } else { 556 Ok(false) 557 } 558 } 559 560 fn check_sgx_support(&self) -> Result<(), Error> { 561 self.ssh_command( 562 "cpuid -l 0x7 -s 0 | tr -s [:space:] | grep -q 'SGX: \ 563 Software Guard Extensions supported = true'", 564 )?; 565 self.ssh_command( 566 "cpuid -l 0x7 -s 0 | tr -s [:space:] | grep -q 'SGX_LC: \ 567 SGX launch config supported = true'", 568 )?; 569 self.ssh_command( 570 "cpuid -l 0x12 -s 0 | tr -s [:space:] | grep -q 'SGX1 \ 571 supported = true'", 572 )?; 573 574 Ok(()) 575 } 576 577 fn get_entropy(&self) -> Result<u32, Error> { 578 self.ssh_command("cat /proc/sys/kernel/random/entropy_avail")? 579 .trim() 580 .parse() 581 .map_err(Error::Parsing) 582 } 583 584 fn get_pci_bridge_class(&self) -> Result<String, Error> { 585 Ok(self 586 .ssh_command("cat /sys/bus/pci/devices/0000:00:00.0/class")? 587 .trim() 588 .to_string()) 589 } 590 591 fn get_pci_device_ids(&self) -> Result<String, Error> { 592 Ok(self 593 .ssh_command("cat /sys/bus/pci/devices/*/device")? 594 .trim() 595 .to_string()) 596 } 597 598 fn get_pci_vendor_ids(&self) -> Result<String, Error> { 599 Ok(self 600 .ssh_command("cat /sys/bus/pci/devices/*/vendor")? 601 .trim() 602 .to_string()) 603 } 604 605 fn does_device_vendor_pair_match( 606 &self, 607 device_id: &str, 608 vendor_id: &str, 609 ) -> Result<bool, Error> { 610 // We are checking if console device's device id and vendor id pair matches 611 let devices = self.get_pci_device_ids()?; 612 let devices: Vec<&str> = devices.split('\n').collect(); 613 let vendors = self.get_pci_vendor_ids()?; 614 let vendors: Vec<&str> = vendors.split('\n').collect(); 615 616 for (index, d_id) in devices.iter().enumerate() { 617 if *d_id == device_id { 618 if let Some(v_id) = vendors.get(index) { 619 if *v_id == vendor_id { 620 return Ok(true); 621 } 622 } 623 } 624 } 625 626 Ok(false) 627 } 628 629 fn valid_virtio_fs_cache_size( 630 &self, 631 dax: bool, 632 cache_size: Option<u64>, 633 ) -> Result<bool, Error> { 634 // SHM region is called different things depending on kernel 635 let shm_region = self 636 .ssh_command("sudo grep 'virtio[0-9]\\|virtio-pci-shm' /proc/iomem || true")? 637 .trim() 638 .to_string(); 639 640 if shm_region.is_empty() { 641 return Ok(!dax); 642 } 643 644 // From this point, the region is not empty, hence it is an error 645 // if DAX is off. 646 if !dax { 647 return Ok(false); 648 } 649 650 let cache = if let Some(cache) = cache_size { 651 cache 652 } else { 653 // 8Gib by default 654 0x0002_0000_0000 655 }; 656 657 let args: Vec<&str> = shm_region.split(':').collect(); 658 if args.is_empty() { 659 return Ok(false); 660 } 661 662 let args: Vec<&str> = args[0].trim().split('-').collect(); 663 if args.len() != 2 { 664 return Ok(false); 665 } 666 667 let start_addr = u64::from_str_radix(args[0], 16).map_err(Error::Parsing)?; 668 let end_addr = u64::from_str_radix(args[1], 16).map_err(Error::Parsing)?; 669 670 Ok(cache == (end_addr - start_addr + 1)) 671 } 672 673 fn check_vsock(&self, socket: &str) { 674 // Listen from guest on vsock CID=3 PORT=16 675 // SOCKET-LISTEN:<domain>:<protocol>:<local-address> 676 let guest_ip = self.network.guest_ip.clone(); 677 let listen_socat = thread::spawn(move || { 678 ssh_command_ip("sudo socat - SOCKET-LISTEN:40:0:x00x00x10x00x00x00x03x00x00x00x00x00x00x00 > vsock_log", &guest_ip, DEFAULT_SSH_RETRIES, DEFAULT_SSH_TIMEOUT).unwrap(); 679 }); 680 681 // Make sure socat is listening, which might take a few second on slow systems 682 thread::sleep(std::time::Duration::new(10, 0)); 683 684 // Write something to vsock from the host 685 assert!(exec_host_command_status(&format!( 686 "echo -e \"CONNECT 16\\nHelloWorld!\" | socat - UNIX-CONNECT:{}", 687 socket 688 )) 689 .success()); 690 691 // Wait for the thread to terminate. 692 listen_socat.join().unwrap(); 693 694 assert_eq!( 695 self.ssh_command("cat vsock_log").unwrap().trim(), 696 "HelloWorld!" 697 ); 698 } 699 700 fn check_nvidia_gpu(&self) { 701 // Run CUDA sample to validate it can find the device 702 let device_query_result = self 703 .ssh_command( 704 "sudo /root/NVIDIA_CUDA-11.3_Samples/bin/x86_64/linux/release/deviceQuery", 705 ) 706 .unwrap(); 707 assert!(device_query_result.contains("Detected 1 CUDA Capable device")); 708 assert!(device_query_result.contains("Device 0: \"NVIDIA Tesla T4\"")); 709 assert!(device_query_result.contains("Result = PASS")); 710 711 // Run NVIDIA DCGM Diagnostics to validate the device is functional 712 self.ssh_command("sudo nv-hostengine").unwrap(); 713 714 assert!(self 715 .ssh_command("sudo dcgmi discovery -l") 716 .unwrap() 717 .contains("Name: NVIDIA Tesla T4")); 718 assert_eq!( 719 self.ssh_command("sudo dcgmi diag -r 'diagnostic' | grep Pass | wc -l") 720 .unwrap() 721 .trim(), 722 "10" 723 ); 724 } 725 726 fn reboot_linux(&self, current_reboot_count: u32, custom_timeout: Option<i32>) { 727 let list_boots_cmd = "sudo journalctl --list-boots | wc -l"; 728 let boot_count = self 729 .ssh_command(list_boots_cmd) 730 .unwrap() 731 .trim() 732 .parse::<u32>() 733 .unwrap_or_default(); 734 735 assert_eq!(boot_count, current_reboot_count + 1); 736 self.ssh_command("sudo reboot").unwrap(); 737 738 self.wait_vm_boot(custom_timeout).unwrap(); 739 let boot_count = self 740 .ssh_command(list_boots_cmd) 741 .unwrap() 742 .trim() 743 .parse::<u32>() 744 .unwrap_or_default(); 745 assert_eq!(boot_count, current_reboot_count + 2); 746 } 747 748 fn enable_memory_hotplug(&self) { 749 self.ssh_command( 750 "echo online | sudo tee /sys/devices/system/memory/auto_online_blocks", 751 ) 752 .unwrap(); 753 } 754 } 755 756 struct GuestCommand<'a> { 757 command: Command, 758 guest: &'a Guest, 759 capture_output: bool, 760 } 761 762 impl<'a> GuestCommand<'a> { 763 fn new(guest: &'a Guest) -> Self { 764 Self::new_with_binary_name(guest, "cloud-hypervisor") 765 } 766 767 fn new_with_binary_name(guest: &'a Guest, binary_name: &str) -> Self { 768 Self { 769 command: Command::new(clh_command(binary_name)), 770 guest, 771 capture_output: false, 772 } 773 } 774 775 fn capture_output(&mut self) -> &mut Self { 776 self.capture_output = true; 777 self 778 } 779 780 fn spawn(&mut self) -> io::Result<Child> { 781 println!( 782 "\n\n==== Start cloud-hypervisor command-line ====\n\n\ 783 {:?}\n\ 784 \n==== End cloud-hypervisor command-line ====\n\n", 785 self.command 786 ); 787 788 if self.capture_output { 789 let child = self 790 .command 791 .arg("-v") 792 .stderr(Stdio::piped()) 793 .stdout(Stdio::piped()) 794 .spawn() 795 .unwrap(); 796 797 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 798 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 799 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 800 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 801 802 if pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE { 803 Ok(child) 804 } else { 805 Err(std::io::Error::new( 806 std::io::ErrorKind::Other, 807 "resizing pipe w/ 'fnctl' failed!", 808 )) 809 } 810 } else { 811 self.command.arg("-v").spawn() 812 } 813 } 814 815 fn args<I, S>(&mut self, args: I) -> &mut Self 816 where 817 I: IntoIterator<Item = S>, 818 S: AsRef<OsStr>, 819 { 820 self.command.args(args); 821 self 822 } 823 824 fn default_disks(&mut self) -> &mut Self { 825 if self.guest.disk_config.disk(DiskType::CloudInit).is_some() { 826 self.args(&[ 827 "--disk", 828 format!( 829 "path={}", 830 self.guest 831 .disk_config 832 .disk(DiskType::OperatingSystem) 833 .unwrap() 834 ) 835 .as_str(), 836 format!( 837 "path={}", 838 self.guest.disk_config.disk(DiskType::CloudInit).unwrap() 839 ) 840 .as_str(), 841 ]) 842 } else { 843 self.args(&[ 844 "--disk", 845 format!( 846 "path={}", 847 self.guest 848 .disk_config 849 .disk(DiskType::OperatingSystem) 850 .unwrap() 851 ) 852 .as_str(), 853 ]) 854 } 855 } 856 857 fn default_net(&mut self) -> &mut Self { 858 self.args(&["--net", self.guest.default_net_string().as_str()]) 859 } 860 } 861 862 fn test_cpu_topology(threads_per_core: u8, cores_per_package: u8, packages: u8) { 863 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 864 let guest = Guest::new(Box::new(focal)); 865 let total_vcpus = threads_per_core * cores_per_package * packages; 866 let mut child = GuestCommand::new(&guest) 867 .args(&[ 868 "--cpus", 869 &format!( 870 "boot={},topology={}:{}:1:{}", 871 total_vcpus, threads_per_core, cores_per_package, packages 872 ), 873 ]) 874 .args(&["--memory", "size=512M"]) 875 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 876 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 877 .default_disks() 878 .default_net() 879 .capture_output() 880 .spawn() 881 .unwrap(); 882 883 let r = std::panic::catch_unwind(|| { 884 guest.wait_vm_boot(None).unwrap(); 885 assert_eq!( 886 guest.get_cpu_count().unwrap_or_default(), 887 u32::from(total_vcpus) 888 ); 889 assert_eq!( 890 guest 891 .ssh_command("lscpu | grep \"per core\" | cut -f 2 -d \":\" | sed \"s# *##\"") 892 .unwrap() 893 .trim() 894 .parse::<u8>() 895 .unwrap_or(0), 896 threads_per_core 897 ); 898 899 assert_eq!( 900 guest 901 .ssh_command("lscpu | grep \"per socket\" | cut -f 2 -d \":\" | sed \"s# *##\"") 902 .unwrap() 903 .trim() 904 .parse::<u8>() 905 .unwrap_or(0), 906 cores_per_package 907 ); 908 909 assert_eq!( 910 guest 911 .ssh_command("lscpu | grep \"Socket\" | cut -f 2 -d \":\" | sed \"s# *##\"") 912 .unwrap() 913 .trim() 914 .parse::<u8>() 915 .unwrap_or(0), 916 packages 917 ); 918 }); 919 920 let _ = child.kill(); 921 let output = child.wait_with_output().unwrap(); 922 923 handle_child_output(r, &output); 924 } 925 926 type PrepareNetDaemon = 927 dyn Fn(&TempDir, &str, Option<&str>, usize, bool) -> (std::process::Command, String); 928 929 fn test_vhost_user_net( 930 tap: Option<&str>, 931 num_queues: usize, 932 prepare_daemon: &PrepareNetDaemon, 933 generate_host_mac: bool, 934 client_mode_daemon: bool, 935 ) { 936 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 937 let guest = Guest::new(Box::new(focal)); 938 let api_socket = temp_api_path(&guest.tmp_dir); 939 940 let kernel_path = direct_kernel_boot_path(); 941 942 let host_mac = if generate_host_mac { 943 Some(MacAddr::local_random()) 944 } else { 945 None 946 }; 947 948 let (mut daemon_command, vunet_socket_path) = prepare_daemon( 949 &guest.tmp_dir, 950 &guest.network.host_ip, 951 tap, 952 num_queues, 953 client_mode_daemon, 954 ); 955 956 let net_params = format!( 957 "vhost_user=true,mac={},socket={},num_queues={},queue_size=1024{},vhost_mode={}", 958 guest.network.guest_mac, 959 vunet_socket_path, 960 num_queues, 961 if let Some(host_mac) = host_mac { 962 format!(",host_mac={}", host_mac) 963 } else { 964 "".to_owned() 965 }, 966 if client_mode_daemon { 967 "server" 968 } else { 969 "client" 970 }, 971 ); 972 973 let mut ch_command = GuestCommand::new(&guest); 974 ch_command 975 .args(&["--cpus", format!("boot={}", num_queues / 2).as_str()]) 976 .args(&["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 977 .args(&["--kernel", kernel_path.to_str().unwrap()]) 978 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 979 .default_disks() 980 .args(&["--net", net_params.as_str()]) 981 .args(&["--api-socket", &api_socket]) 982 .capture_output(); 983 984 let mut daemon_child: std::process::Child; 985 let mut child: std::process::Child; 986 987 if client_mode_daemon { 988 child = ch_command.spawn().unwrap(); 989 // Make sure the VMM is waiting for the backend to connect 990 thread::sleep(std::time::Duration::new(10, 0)); 991 daemon_child = daemon_command.spawn().unwrap(); 992 } else { 993 daemon_child = daemon_command.spawn().unwrap(); 994 // Make sure the backend is waiting for the VMM to connect 995 thread::sleep(std::time::Duration::new(10, 0)); 996 child = ch_command.spawn().unwrap(); 997 } 998 999 let r = std::panic::catch_unwind(|| { 1000 guest.wait_vm_boot(None).unwrap(); 1001 1002 if let Some(tap_name) = tap { 1003 let tap_count = 1004 exec_host_command_output(&format!("ip link | grep -c {}", tap_name)); 1005 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 1006 } 1007 1008 if let Some(host_mac) = tap { 1009 let mac_count = 1010 exec_host_command_output(&format!("ip link | grep -c {}", host_mac)); 1011 assert_eq!(String::from_utf8_lossy(&mac_count.stdout).trim(), "1"); 1012 } 1013 1014 // 1 network interface + default localhost ==> 2 interfaces 1015 // It's important to note that this test is fully exercising the 1016 // vhost-user-net implementation and the associated backend since 1017 // it does not define any --net network interface. That means all 1018 // the ssh communication in that test happens through the network 1019 // interface backed by vhost-user-net. 1020 assert_eq!( 1021 guest 1022 .ssh_command("ip -o link | wc -l") 1023 .unwrap() 1024 .trim() 1025 .parse::<u32>() 1026 .unwrap_or_default(), 1027 2 1028 ); 1029 1030 // The following pci devices will appear on guest with PCI-MSI 1031 // interrupt vectors assigned. 1032 // 1 virtio-console with 3 vectors: config, Rx, Tx 1033 // 1 virtio-blk with 2 vectors: config, Request 1034 // 1 virtio-blk with 2 vectors: config, Request 1035 // 1 virtio-rng with 2 vectors: config, Request 1036 // Since virtio-net has 2 queue pairs, its vectors is as follows: 1037 // 1 virtio-net with 5 vectors: config, Rx (2), Tx (2) 1038 // Based on the above, the total vectors should 14. 1039 #[cfg(target_arch = "x86_64")] 1040 let grep_cmd = "grep -c PCI-MSI /proc/interrupts"; 1041 #[cfg(target_arch = "aarch64")] 1042 let grep_cmd = "grep -c ITS-MSI /proc/interrupts"; 1043 assert_eq!( 1044 guest 1045 .ssh_command(grep_cmd) 1046 .unwrap() 1047 .trim() 1048 .parse::<u32>() 1049 .unwrap_or_default(), 1050 10 + (num_queues as u32) 1051 ); 1052 1053 // ACPI feature is needed. 1054 #[cfg(all(target_arch = "x86_64", feature = "acpi"))] 1055 { 1056 guest.enable_memory_hotplug(); 1057 1058 // Add RAM to the VM 1059 let desired_ram = 1024 << 20; 1060 resize_command(&api_socket, None, Some(desired_ram), None); 1061 1062 thread::sleep(std::time::Duration::new(10, 0)); 1063 1064 // Here by simply checking the size (through ssh), we validate 1065 // the connection is still working, which means vhost-user-net 1066 // keeps working after the resize. 1067 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 1068 } 1069 }); 1070 1071 let _ = child.kill(); 1072 let output = child.wait_with_output().unwrap(); 1073 1074 thread::sleep(std::time::Duration::new(5, 0)); 1075 let _ = daemon_child.kill(); 1076 let _ = daemon_child.wait(); 1077 1078 handle_child_output(r, &output); 1079 } 1080 1081 type PrepareBlkDaemon = 1082 dyn Fn(&TempDir, &str, usize, bool, bool) -> (std::process::Child, String); 1083 1084 fn test_vhost_user_blk( 1085 num_queues: usize, 1086 readonly: bool, 1087 direct: bool, 1088 prepare_vhost_user_blk_daemon: Option<&PrepareBlkDaemon>, 1089 ) { 1090 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1091 let guest = Guest::new(Box::new(focal)); 1092 let api_socket = temp_api_path(&guest.tmp_dir); 1093 1094 let kernel_path = direct_kernel_boot_path(); 1095 1096 let (blk_params, daemon_child) = { 1097 let prepare_daemon = prepare_vhost_user_blk_daemon.unwrap(); 1098 // Start the daemon 1099 let (daemon_child, vubd_socket_path) = 1100 prepare_daemon(&guest.tmp_dir, "blk.img", num_queues, readonly, direct); 1101 1102 ( 1103 format!( 1104 "vhost_user=true,socket={},num_queues={},queue_size=128", 1105 vubd_socket_path, num_queues, 1106 ), 1107 Some(daemon_child), 1108 ) 1109 }; 1110 1111 let mut child = GuestCommand::new(&guest) 1112 .args(&["--cpus", format!("boot={}", num_queues).as_str()]) 1113 .args(&["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 1114 .args(&["--kernel", kernel_path.to_str().unwrap()]) 1115 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1116 .args(&[ 1117 "--disk", 1118 format!( 1119 "path={}", 1120 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 1121 ) 1122 .as_str(), 1123 format!( 1124 "path={}", 1125 guest.disk_config.disk(DiskType::CloudInit).unwrap() 1126 ) 1127 .as_str(), 1128 blk_params.as_str(), 1129 ]) 1130 .default_net() 1131 .args(&["--api-socket", &api_socket]) 1132 .capture_output() 1133 .spawn() 1134 .unwrap(); 1135 1136 let r = std::panic::catch_unwind(|| { 1137 guest.wait_vm_boot(None).unwrap(); 1138 1139 // Check both if /dev/vdc exists and if the block size is 16M. 1140 assert_eq!( 1141 guest 1142 .ssh_command("lsblk | grep vdc | grep -c 16M") 1143 .unwrap() 1144 .trim() 1145 .parse::<u32>() 1146 .unwrap_or_default(), 1147 1 1148 ); 1149 1150 // Check if this block is RO or RW. 1151 assert_eq!( 1152 guest 1153 .ssh_command("lsblk | grep vdc | awk '{print $5}'") 1154 .unwrap() 1155 .trim() 1156 .parse::<u32>() 1157 .unwrap_or_default(), 1158 readonly as u32 1159 ); 1160 1161 // Check if the number of queues in /sys/block/vdc/mq matches the 1162 // expected num_queues. 1163 assert_eq!( 1164 guest 1165 .ssh_command("ls -ll /sys/block/vdc/mq | grep ^d | wc -l") 1166 .unwrap() 1167 .trim() 1168 .parse::<u32>() 1169 .unwrap_or_default(), 1170 num_queues as u32 1171 ); 1172 1173 // Mount the device 1174 let mount_ro_rw_flag = if readonly { "ro,noload" } else { "rw" }; 1175 guest.ssh_command("mkdir mount_image").unwrap(); 1176 guest 1177 .ssh_command( 1178 format!( 1179 "sudo mount -o {} -t ext4 /dev/vdc mount_image/", 1180 mount_ro_rw_flag 1181 ) 1182 .as_str(), 1183 ) 1184 .unwrap(); 1185 1186 // Check the content of the block device. The file "foo" should 1187 // contain "bar". 1188 assert_eq!( 1189 guest.ssh_command("cat mount_image/foo").unwrap().trim(), 1190 "bar" 1191 ); 1192 1193 // ACPI feature is needed. 1194 #[cfg(all(target_arch = "x86_64", feature = "acpi"))] 1195 { 1196 guest.enable_memory_hotplug(); 1197 1198 // Add RAM to the VM 1199 let desired_ram = 1024 << 20; 1200 resize_command(&api_socket, None, Some(desired_ram), None); 1201 1202 thread::sleep(std::time::Duration::new(10, 0)); 1203 1204 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 1205 1206 // Check again the content of the block device after the resize 1207 // has been performed. 1208 assert_eq!( 1209 guest.ssh_command("cat mount_image/foo").unwrap().trim(), 1210 "bar" 1211 ); 1212 } 1213 1214 // Unmount the device 1215 guest.ssh_command("sudo umount /dev/vdc").unwrap(); 1216 guest.ssh_command("rm -r mount_image").unwrap(); 1217 }); 1218 1219 let _ = child.kill(); 1220 let output = child.wait_with_output().unwrap(); 1221 1222 if let Some(mut daemon_child) = daemon_child { 1223 thread::sleep(std::time::Duration::new(5, 0)); 1224 let _ = daemon_child.kill(); 1225 let _ = daemon_child.wait(); 1226 } 1227 1228 handle_child_output(r, &output); 1229 } 1230 1231 fn test_boot_from_vhost_user_blk( 1232 num_queues: usize, 1233 readonly: bool, 1234 direct: bool, 1235 prepare_vhost_user_blk_daemon: Option<&PrepareBlkDaemon>, 1236 ) { 1237 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1238 let guest = Guest::new(Box::new(focal)); 1239 1240 let kernel_path = direct_kernel_boot_path(); 1241 1242 let disk_path = guest.disk_config.disk(DiskType::OperatingSystem).unwrap(); 1243 1244 let (blk_boot_params, daemon_child) = { 1245 let prepare_daemon = prepare_vhost_user_blk_daemon.unwrap(); 1246 // Start the daemon 1247 let (daemon_child, vubd_socket_path) = prepare_daemon( 1248 &guest.tmp_dir, 1249 disk_path.as_str(), 1250 num_queues, 1251 readonly, 1252 direct, 1253 ); 1254 1255 ( 1256 format!( 1257 "vhost_user=true,socket={},num_queues={},queue_size=128", 1258 vubd_socket_path, num_queues, 1259 ), 1260 Some(daemon_child), 1261 ) 1262 }; 1263 1264 let mut child = GuestCommand::new(&guest) 1265 .args(&["--cpus", format!("boot={}", num_queues).as_str()]) 1266 .args(&["--memory", "size=512M,shared=on"]) 1267 .args(&["--kernel", kernel_path.to_str().unwrap()]) 1268 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1269 .args(&[ 1270 "--disk", 1271 blk_boot_params.as_str(), 1272 format!( 1273 "path={}", 1274 guest.disk_config.disk(DiskType::CloudInit).unwrap() 1275 ) 1276 .as_str(), 1277 ]) 1278 .default_net() 1279 .capture_output() 1280 .spawn() 1281 .unwrap(); 1282 1283 let r = std::panic::catch_unwind(|| { 1284 guest.wait_vm_boot(None).unwrap(); 1285 1286 // Just check the VM booted correctly. 1287 assert_eq!(guest.get_cpu_count().unwrap_or_default(), num_queues as u32); 1288 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 1289 }); 1290 let _ = child.kill(); 1291 let output = child.wait_with_output().unwrap(); 1292 1293 if let Some(mut daemon_child) = daemon_child { 1294 thread::sleep(std::time::Duration::new(5, 0)); 1295 let _ = daemon_child.kill(); 1296 let _ = daemon_child.wait(); 1297 } 1298 1299 handle_child_output(r, &output); 1300 } 1301 1302 fn test_virtio_fs( 1303 dax: bool, 1304 cache_size: Option<u64>, 1305 virtiofsd_cache: &str, 1306 prepare_daemon: &dyn Fn(&TempDir, &str, &str) -> (std::process::Child, String), 1307 hotplug: bool, 1308 ) { 1309 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1310 let guest = Guest::new(Box::new(focal)); 1311 let api_socket = temp_api_path(&guest.tmp_dir); 1312 1313 let mut workload_path = dirs::home_dir().unwrap(); 1314 workload_path.push("workloads"); 1315 1316 let mut shared_dir = workload_path; 1317 shared_dir.push("shared_dir"); 1318 1319 let kernel_path = direct_kernel_boot_path(); 1320 1321 let (dax_vmm_param, dax_mount_param) = if dax { ("on", "-o dax") } else { ("off", "") }; 1322 let cache_size_vmm_param = if let Some(cache) = cache_size { 1323 format!(",cache_size={}", cache) 1324 } else { 1325 "".to_string() 1326 }; 1327 1328 let (mut daemon_child, virtiofsd_socket_path) = prepare_daemon( 1329 &guest.tmp_dir, 1330 shared_dir.to_str().unwrap(), 1331 virtiofsd_cache, 1332 ); 1333 1334 let mut guest_command = GuestCommand::new(&guest); 1335 guest_command 1336 .args(&["--cpus", "boot=1"]) 1337 .args(&["--memory", "size=512M,hotplug_size=2048M,shared=on"]) 1338 .args(&["--kernel", kernel_path.to_str().unwrap()]) 1339 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1340 .default_disks() 1341 .default_net() 1342 .args(&["--api-socket", &api_socket]); 1343 1344 let fs_params = format!( 1345 "id=myfs0,tag=myfs,socket={},num_queues=1,queue_size=1024,dax={}{}", 1346 virtiofsd_socket_path, dax_vmm_param, cache_size_vmm_param 1347 ); 1348 1349 if !hotplug { 1350 guest_command.args(&["--fs", fs_params.as_str()]); 1351 } 1352 1353 let mut child = guest_command.capture_output().spawn().unwrap(); 1354 1355 let r = std::panic::catch_unwind(|| { 1356 guest.wait_vm_boot(None).unwrap(); 1357 1358 if hotplug { 1359 // Add fs to the VM 1360 let (cmd_success, cmd_output) = 1361 remote_command_w_output(&api_socket, "add-fs", Some(&fs_params)); 1362 assert!(cmd_success); 1363 assert!(String::from_utf8_lossy(&cmd_output) 1364 .contains("{\"id\":\"myfs0\",\"bdf\":\"0000:00:06.0\"}")); 1365 1366 thread::sleep(std::time::Duration::new(10, 0)); 1367 } 1368 1369 // Mount shared directory through virtio_fs filesystem 1370 let mount_cmd = format!( 1371 "mkdir -p mount_dir && \ 1372 sudo mount -t virtiofs {} myfs mount_dir/", 1373 dax_mount_param 1374 ); 1375 guest.ssh_command(&mount_cmd).unwrap(); 1376 1377 assert!(guest 1378 .valid_virtio_fs_cache_size(dax, cache_size) 1379 .unwrap_or_default()); 1380 1381 // Check file1 exists and its content is "foo" 1382 assert_eq!( 1383 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1384 "foo" 1385 ); 1386 // Check file2 does not exist 1387 guest 1388 .ssh_command("[ ! -f 'mount_dir/file2' ] || true") 1389 .unwrap(); 1390 1391 // Check file3 exists and its content is "bar" 1392 assert_eq!( 1393 guest.ssh_command("cat mount_dir/file3").unwrap().trim(), 1394 "bar" 1395 ); 1396 1397 // ACPI feature is needed. 1398 #[cfg(all(target_arch = "x86_64", feature = "acpi"))] 1399 { 1400 guest.enable_memory_hotplug(); 1401 1402 // Add RAM to the VM 1403 let desired_ram = 1024 << 20; 1404 resize_command(&api_socket, None, Some(desired_ram), None); 1405 1406 thread::sleep(std::time::Duration::new(30, 0)); 1407 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 1408 1409 // After the resize, check again that file1 exists and its 1410 // content is "foo". 1411 assert_eq!( 1412 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1413 "foo" 1414 ); 1415 } 1416 1417 if hotplug { 1418 // Remove from VM 1419 guest.ssh_command("sudo umount mount_dir").unwrap(); 1420 assert!(remote_command(&api_socket, "remove-device", Some("myfs0"))); 1421 } 1422 }); 1423 1424 let (r, hotplug_daemon_child) = if r.is_ok() && hotplug { 1425 thread::sleep(std::time::Duration::new(10, 0)); 1426 let (daemon_child, virtiofsd_socket_path) = prepare_daemon( 1427 &guest.tmp_dir, 1428 shared_dir.to_str().unwrap(), 1429 virtiofsd_cache, 1430 ); 1431 1432 let r = std::panic::catch_unwind(|| { 1433 thread::sleep(std::time::Duration::new(10, 0)); 1434 let fs_params = format!( 1435 "id=myfs0,tag=myfs,socket={},num_queues=1,queue_size=1024,dax={}{}", 1436 virtiofsd_socket_path, dax_vmm_param, cache_size_vmm_param 1437 ); 1438 1439 // Add back and check it works 1440 let (cmd_success, cmd_output) = 1441 remote_command_w_output(&api_socket, "add-fs", Some(&fs_params)); 1442 assert!(cmd_success); 1443 assert!(String::from_utf8_lossy(&cmd_output) 1444 .contains("{\"id\":\"myfs0\",\"bdf\":\"0000:00:06.0\"}")); 1445 thread::sleep(std::time::Duration::new(10, 0)); 1446 // Mount shared directory through virtio_fs filesystem 1447 let mount_cmd = format!( 1448 "mkdir -p mount_dir && \ 1449 sudo mount -t virtiofs {} myfs mount_dir/", 1450 dax_mount_param 1451 ); 1452 guest.ssh_command(&mount_cmd).unwrap(); 1453 // Check file1 exists and its content is "foo" 1454 assert_eq!( 1455 guest.ssh_command("cat mount_dir/file1").unwrap().trim(), 1456 "foo" 1457 ); 1458 }); 1459 1460 (r, Some(daemon_child)) 1461 } else { 1462 (r, None) 1463 }; 1464 1465 let _ = child.kill(); 1466 let output = child.wait_with_output().unwrap(); 1467 1468 let _ = daemon_child.kill(); 1469 let _ = daemon_child.wait(); 1470 1471 if let Some(mut daemon_child) = hotplug_daemon_child { 1472 let _ = daemon_child.kill(); 1473 let _ = daemon_child.wait(); 1474 } 1475 1476 handle_child_output(r, &output); 1477 } 1478 1479 fn test_virtio_pmem(discard_writes: bool, specify_size: bool) { 1480 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1481 let guest = Guest::new(Box::new(focal)); 1482 1483 let kernel_path = direct_kernel_boot_path(); 1484 1485 let pmem_temp_file = TempFile::new().unwrap(); 1486 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 1487 1488 std::process::Command::new("mkfs.ext4") 1489 .arg(pmem_temp_file.as_path()) 1490 .output() 1491 .expect("Expect creating disk image to succeed"); 1492 1493 let mut child = GuestCommand::new(&guest) 1494 .args(&["--cpus", "boot=1"]) 1495 .args(&["--memory", "size=512M"]) 1496 .args(&["--kernel", kernel_path.to_str().unwrap()]) 1497 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1498 .default_disks() 1499 .default_net() 1500 .args(&[ 1501 "--pmem", 1502 format!( 1503 "file={}{}{}", 1504 pmem_temp_file.as_path().to_str().unwrap(), 1505 if specify_size { ",size=128M" } else { "" }, 1506 if discard_writes { 1507 ",discard_writes=on" 1508 } else { 1509 "" 1510 } 1511 ) 1512 .as_str(), 1513 ]) 1514 .capture_output() 1515 .spawn() 1516 .unwrap(); 1517 1518 let r = std::panic::catch_unwind(|| { 1519 guest.wait_vm_boot(None).unwrap(); 1520 1521 // Check for the presence of /dev/pmem0 1522 assert_eq!( 1523 guest.ssh_command("ls /dev/pmem0").unwrap().trim(), 1524 "/dev/pmem0" 1525 ); 1526 1527 // Check changes persist after reboot 1528 assert_eq!(guest.ssh_command("sudo mount /dev/pmem0 /mnt").unwrap(), ""); 1529 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), "lost+found\n"); 1530 guest 1531 .ssh_command("echo test123 | sudo tee /mnt/test") 1532 .unwrap(); 1533 assert_eq!(guest.ssh_command("sudo umount /mnt").unwrap(), ""); 1534 assert_eq!(guest.ssh_command("ls /mnt").unwrap(), ""); 1535 1536 guest.reboot_linux(0, None); 1537 assert_eq!(guest.ssh_command("sudo mount /dev/pmem0 /mnt").unwrap(), ""); 1538 assert_eq!( 1539 guest 1540 .ssh_command("sudo cat /mnt/test || true") 1541 .unwrap() 1542 .trim(), 1543 if discard_writes { "" } else { "test123" } 1544 ); 1545 }); 1546 1547 let _ = child.kill(); 1548 let output = child.wait_with_output().unwrap(); 1549 1550 handle_child_output(r, &output); 1551 } 1552 1553 fn get_fd_count(pid: u32) -> usize { 1554 fs::read_dir(format!("/proc/{}/fd", pid)).unwrap().count() 1555 } 1556 1557 fn _test_virtio_vsock(hotplug: bool) { 1558 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1559 let guest = Guest::new(Box::new(focal)); 1560 1561 let kernel_path = direct_kernel_boot_path(); 1562 1563 let socket = temp_vsock_path(&guest.tmp_dir); 1564 let api_socket = temp_api_path(&guest.tmp_dir); 1565 1566 let mut cmd = GuestCommand::new(&guest); 1567 cmd.args(&["--api-socket", &api_socket]); 1568 cmd.args(&["--cpus", "boot=1"]); 1569 cmd.args(&["--memory", "size=512M"]); 1570 cmd.args(&["--kernel", kernel_path.to_str().unwrap()]); 1571 cmd.args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]); 1572 cmd.default_disks(); 1573 cmd.default_net(); 1574 1575 if !hotplug { 1576 cmd.args(&["--vsock", format!("cid=3,socket={}", socket).as_str()]); 1577 } 1578 1579 let mut child = cmd.capture_output().spawn().unwrap(); 1580 1581 let r = std::panic::catch_unwind(|| { 1582 guest.wait_vm_boot(None).unwrap(); 1583 1584 if hotplug { 1585 let (cmd_success, cmd_output) = remote_command_w_output( 1586 &api_socket, 1587 "add-vsock", 1588 Some(format!("cid=3,socket={},id=test0", socket).as_str()), 1589 ); 1590 assert!(cmd_success); 1591 assert!(String::from_utf8_lossy(&cmd_output) 1592 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 1593 thread::sleep(std::time::Duration::new(10, 0)); 1594 // Check adding a second one fails 1595 assert!(!remote_command( 1596 &api_socket, 1597 "add-vsock", 1598 Some("cid=1234,socket=/tmp/fail") 1599 )); 1600 } 1601 1602 // Validate vsock works as expected. 1603 guest.check_vsock(socket.as_str()); 1604 1605 // AArch64 currently does not support reboot, and therefore we 1606 // skip the reboot test here. 1607 #[cfg(target_arch = "x86_64")] 1608 { 1609 guest.reboot_linux(0, None); 1610 1611 // Validate vsock still works after a reboot. 1612 guest.check_vsock(socket.as_str()); 1613 } 1614 if hotplug { 1615 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 1616 } 1617 }); 1618 1619 let _ = child.kill(); 1620 let output = child.wait_with_output().unwrap(); 1621 1622 handle_child_output(r, &output); 1623 } 1624 1625 fn get_ksm_pages_shared() -> u32 { 1626 fs::read_to_string("/sys/kernel/mm/ksm/pages_shared") 1627 .unwrap() 1628 .trim() 1629 .parse::<u32>() 1630 .unwrap() 1631 } 1632 1633 fn test_memory_mergeable(mergeable: bool) { 1634 let memory_param = if mergeable { 1635 "mergeable=on" 1636 } else { 1637 "mergeable=off" 1638 }; 1639 1640 // We are assuming the rest of the system in our CI is not using mergeable memeory 1641 let ksm_ps_init = get_ksm_pages_shared(); 1642 assert!(ksm_ps_init == 0); 1643 1644 let focal1 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1645 let guest1 = Guest::new(Box::new(focal1)); 1646 let mut child1 = GuestCommand::new(&guest1) 1647 .args(&["--cpus", "boot=1"]) 1648 .args(&["--memory", format!("size=512M,{}", memory_param).as_str()]) 1649 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 1650 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1651 .default_disks() 1652 .args(&["--net", guest1.default_net_string().as_str()]) 1653 .args(&["--serial", "tty", "--console", "off"]) 1654 .capture_output() 1655 .spawn() 1656 .unwrap(); 1657 1658 let r = std::panic::catch_unwind(|| { 1659 guest1.wait_vm_boot(None).unwrap(); 1660 }); 1661 if r.is_err() { 1662 let _ = child1.kill(); 1663 let output = child1.wait_with_output().unwrap(); 1664 handle_child_output(r, &output); 1665 panic!("Test should already be failed/panicked"); // To explicitly mark this block never return 1666 } 1667 1668 let ksm_ps_guest1 = get_ksm_pages_shared(); 1669 1670 let focal2 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1671 let guest2 = Guest::new(Box::new(focal2)); 1672 let mut child2 = GuestCommand::new(&guest2) 1673 .args(&["--cpus", "boot=1"]) 1674 .args(&["--memory", format!("size=512M,{}", memory_param).as_str()]) 1675 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 1676 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1677 .default_disks() 1678 .args(&["--net", guest2.default_net_string().as_str()]) 1679 .args(&["--serial", "tty", "--console", "off"]) 1680 .capture_output() 1681 .spawn() 1682 .unwrap(); 1683 1684 let r = std::panic::catch_unwind(|| { 1685 guest2.wait_vm_boot(None).unwrap(); 1686 let ksm_ps_guest2 = get_ksm_pages_shared(); 1687 1688 if mergeable { 1689 println!( 1690 "ksm pages_shared after vm1 booted '{}', ksm pages_shared after vm2 booted '{}'", 1691 ksm_ps_guest1, ksm_ps_guest2 1692 ); 1693 // We are expecting the number of shared pages to increase as the number of VM increases 1694 assert!(ksm_ps_guest1 < ksm_ps_guest2); 1695 } else { 1696 assert!(ksm_ps_guest1 == 0); 1697 assert!(ksm_ps_guest2 == 0); 1698 } 1699 }); 1700 1701 let _ = child1.kill(); 1702 let _ = child2.kill(); 1703 1704 let output = child1.wait_with_output().unwrap(); 1705 child2.wait().unwrap(); 1706 1707 handle_child_output(r, &output); 1708 } 1709 1710 fn _get_vmm_overhead(pid: u32, guest_memory_size: u32) -> HashMap<String, u32> { 1711 let smaps = fs::File::open(format!("/proc/{}/smaps", pid)).unwrap(); 1712 let reader = io::BufReader::new(smaps); 1713 1714 let mut skip_map: bool = false; 1715 let mut region_name: String = "".to_string(); 1716 let mut region_maps = HashMap::new(); 1717 for line in reader.lines() { 1718 let l = line.unwrap(); 1719 1720 if l.contains('-') { 1721 let values: Vec<&str> = l.split_whitespace().collect(); 1722 region_name = values.last().unwrap().trim().to_string(); 1723 if region_name == "0" { 1724 region_name = "anonymous".to_string() 1725 } 1726 } 1727 1728 // Each section begins with something that looks like: 1729 // Size: 2184 kB 1730 if l.starts_with("Size:") { 1731 let values: Vec<&str> = l.split_whitespace().collect(); 1732 let map_size = values[1].parse::<u32>().unwrap(); 1733 // We skip the assigned guest RAM map, its RSS is only 1734 // dependent on the guest actual memory usage. 1735 // Everything else can be added to the VMM overhead. 1736 skip_map = map_size >= guest_memory_size; 1737 continue; 1738 } 1739 1740 // If this is a map we're taking into account, then we only 1741 // count the RSS. The sum of all counted RSS is the VMM overhead. 1742 if !skip_map && l.starts_with("Rss:") { 1743 let values: Vec<&str> = l.split_whitespace().collect(); 1744 let value = values[1].trim().parse::<u32>().unwrap(); 1745 *region_maps.entry(region_name.clone()).or_insert(0) += value; 1746 } 1747 } 1748 1749 region_maps 1750 } 1751 1752 fn get_vmm_overhead(pid: u32, guest_memory_size: u32) -> u32 { 1753 let mut total = 0; 1754 1755 for (region_name, value) in &_get_vmm_overhead(pid, guest_memory_size) { 1756 eprintln!("{}: {}", region_name, value); 1757 total += value; 1758 } 1759 1760 total 1761 } 1762 1763 // 10MB is our maximum accepted overhead. 1764 const MAXIMUM_VMM_OVERHEAD_KB: u32 = 10 * 1024; 1765 1766 #[derive(PartialEq, PartialOrd)] 1767 struct Counters { 1768 rx_bytes: u64, 1769 rx_frames: u64, 1770 tx_bytes: u64, 1771 tx_frames: u64, 1772 read_bytes: u64, 1773 write_bytes: u64, 1774 read_ops: u64, 1775 write_ops: u64, 1776 } 1777 1778 fn get_counters(api_socket: &str) -> Counters { 1779 // Get counters 1780 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "counters", None); 1781 assert!(cmd_success); 1782 1783 let counters: HashMap<&str, HashMap<&str, u64>> = 1784 serde_json::from_slice(&cmd_output).unwrap_or_default(); 1785 1786 let rx_bytes = *counters.get("_net2").unwrap().get("rx_bytes").unwrap(); 1787 let rx_frames = *counters.get("_net2").unwrap().get("rx_frames").unwrap(); 1788 let tx_bytes = *counters.get("_net2").unwrap().get("tx_bytes").unwrap(); 1789 let tx_frames = *counters.get("_net2").unwrap().get("tx_frames").unwrap(); 1790 1791 let read_bytes = *counters.get("_disk0").unwrap().get("read_bytes").unwrap(); 1792 let write_bytes = *counters.get("_disk0").unwrap().get("write_bytes").unwrap(); 1793 let read_ops = *counters.get("_disk0").unwrap().get("read_ops").unwrap(); 1794 let write_ops = *counters.get("_disk0").unwrap().get("write_ops").unwrap(); 1795 1796 Counters { 1797 rx_bytes, 1798 rx_frames, 1799 tx_bytes, 1800 tx_frames, 1801 read_bytes, 1802 write_bytes, 1803 read_ops, 1804 write_ops, 1805 } 1806 } 1807 1808 fn pty_read(mut pty: std::fs::File) -> Receiver<String> { 1809 let (tx, rx) = mpsc::channel::<String>(); 1810 thread::spawn(move || loop { 1811 thread::sleep(std::time::Duration::new(1, 0)); 1812 let mut buf = [0; 512]; 1813 match pty.read(&mut buf) { 1814 Ok(_) => { 1815 let output = std::str::from_utf8(&buf).unwrap().to_string(); 1816 match tx.send(output) { 1817 Ok(_) => (), 1818 Err(_) => break, 1819 } 1820 } 1821 Err(_) => break, 1822 } 1823 }); 1824 rx 1825 } 1826 1827 fn get_pty_path(api_socket: &str, pty_type: &str) -> PathBuf { 1828 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "info", None); 1829 assert!(cmd_success); 1830 let info: serde_json::Value = serde_json::from_slice(&cmd_output).unwrap_or_default(); 1831 assert_eq!("Pty", info["config"][pty_type]["mode"]); 1832 PathBuf::from( 1833 info["config"][pty_type]["file"] 1834 .as_str() 1835 .expect("Missing pty path"), 1836 ) 1837 } 1838 1839 // VFIO test network setup. 1840 // We reserve a different IP class for it: 172.18.0.0/24. 1841 fn setup_vfio_network_interfaces() { 1842 // 'vfio-br0' 1843 assert!(exec_host_command_status("sudo ip link add name vfio-br0 type bridge").success()); 1844 assert!(exec_host_command_status("sudo ip link set vfio-br0 up").success()); 1845 assert!(exec_host_command_status("sudo ip addr add 172.18.0.1/24 dev vfio-br0").success()); 1846 // 'vfio-tap0' 1847 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap0 mode tap").success()); 1848 assert!(exec_host_command_status("sudo ip link set vfio-tap0 master vfio-br0").success()); 1849 assert!(exec_host_command_status("sudo ip link set vfio-tap0 up").success()); 1850 // 'vfio-tap1' 1851 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap1 mode tap").success()); 1852 assert!(exec_host_command_status("sudo ip link set vfio-tap1 master vfio-br0").success()); 1853 assert!(exec_host_command_status("sudo ip link set vfio-tap1 up").success()); 1854 // 'vfio-tap2' 1855 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap2 mode tap").success()); 1856 assert!(exec_host_command_status("sudo ip link set vfio-tap2 master vfio-br0").success()); 1857 assert!(exec_host_command_status("sudo ip link set vfio-tap2 up").success()); 1858 // 'vfio-tap3' 1859 assert!(exec_host_command_status("sudo ip tuntap add vfio-tap3 mode tap").success()); 1860 assert!(exec_host_command_status("sudo ip link set vfio-tap3 master vfio-br0").success()); 1861 assert!(exec_host_command_status("sudo ip link set vfio-tap3 up").success()); 1862 } 1863 1864 // Tear VFIO test network down 1865 fn cleanup_vfio_network_interfaces() { 1866 assert!(exec_host_command_status("sudo ip link del vfio-br0").success()); 1867 assert!(exec_host_command_status("sudo ip link del vfio-tap0").success()); 1868 assert!(exec_host_command_status("sudo ip link del vfio-tap1").success()); 1869 assert!(exec_host_command_status("sudo ip link del vfio-tap2").success()); 1870 assert!(exec_host_command_status("sudo ip link del vfio-tap3").success()); 1871 } 1872 1873 fn balloon_size(api_socket: &str) -> u64 { 1874 let (cmd_success, cmd_output) = remote_command_w_output(api_socket, "info", None); 1875 assert!(cmd_success); 1876 1877 let info: serde_json::Value = serde_json::from_slice(&cmd_output).unwrap_or_default(); 1878 let total_mem = &info["config"]["memory"]["size"] 1879 .to_string() 1880 .parse::<u64>() 1881 .unwrap(); 1882 let actual_mem = &info["memory_actual_size"] 1883 .to_string() 1884 .parse::<u64>() 1885 .unwrap(); 1886 total_mem - actual_mem 1887 } 1888 1889 mod parallel { 1890 use crate::tests::*; 1891 1892 #[test] 1893 #[cfg(target_arch = "x86_64")] 1894 fn test_simple_launch() { 1895 let bionic = UbuntuDiskConfig::new(BIONIC_IMAGE_NAME.to_string()); 1896 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1897 1898 vec![Box::new(bionic), Box::new(focal)] 1899 .drain(..) 1900 .for_each(|disk_config| { 1901 let guest = Guest::new(disk_config); 1902 1903 let mut child = GuestCommand::new(&guest) 1904 .args(&["--cpus", "boot=1"]) 1905 .args(&["--memory", "size=512M"]) 1906 .args(&["--kernel", guest.fw_path.as_str()]) 1907 .default_disks() 1908 .default_net() 1909 .args(&["--serial", "tty", "--console", "off"]) 1910 .capture_output() 1911 .spawn() 1912 .unwrap(); 1913 1914 let r = std::panic::catch_unwind(|| { 1915 guest.wait_vm_boot(Some(120)).unwrap(); 1916 1917 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 1918 assert_eq!(guest.get_initial_apicid().unwrap_or(1), 0); 1919 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 1920 assert!(guest.get_entropy().unwrap_or_default() >= 900); 1921 assert_eq!(guest.get_pci_bridge_class().unwrap_or_default(), "0x060000"); 1922 }); 1923 1924 let _ = child.kill(); 1925 let output = child.wait_with_output().unwrap(); 1926 1927 handle_child_output(r, &output); 1928 }); 1929 } 1930 1931 #[test] 1932 #[cfg(all(target_arch = "aarch64", feature = "acpi"))] 1933 fn test_edk2_acpi_launch() { 1934 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 1935 1936 vec![Box::new(focal)].drain(..).for_each(|disk_config| { 1937 let guest = Guest::new(disk_config); 1938 1939 let mut child = GuestCommand::new(&guest) 1940 .args(&["--cpus", "boot=1"]) 1941 .args(&["--memory", "size=512M"]) 1942 .args(&["--kernel", edk2_path().to_str().unwrap()]) 1943 .default_disks() 1944 .default_net() 1945 .args(&["--serial", "tty", "--console", "off"]) 1946 .capture_output() 1947 .spawn() 1948 .unwrap(); 1949 1950 let r = std::panic::catch_unwind(|| { 1951 guest.wait_vm_boot(Some(120)).unwrap(); 1952 1953 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 1954 assert!(guest.get_total_memory().unwrap_or_default() > 400_000); 1955 assert!(guest.get_entropy().unwrap_or_default() >= 900); 1956 assert_eq!(guest.get_pci_bridge_class().unwrap_or_default(), "0x060000"); 1957 }); 1958 1959 let _ = child.kill(); 1960 let output = child.wait_with_output().unwrap(); 1961 1962 handle_child_output(r, &output); 1963 }); 1964 } 1965 1966 #[test] 1967 fn test_multi_cpu() { 1968 let bionic = UbuntuDiskConfig::new(BIONIC_IMAGE_NAME.to_string()); 1969 let guest = Guest::new(Box::new(bionic)); 1970 let mut cmd = GuestCommand::new(&guest); 1971 cmd.args(&["--cpus", "boot=2,max=4"]) 1972 .args(&["--memory", "size=512M"]) 1973 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 1974 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 1975 .capture_output() 1976 .default_disks() 1977 .default_net(); 1978 1979 let mut child = cmd.spawn().unwrap(); 1980 1981 let r = std::panic::catch_unwind(|| { 1982 guest.wait_vm_boot(Some(120)).unwrap(); 1983 1984 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 1985 1986 #[cfg(target_arch = "x86_64")] 1987 assert_eq!( 1988 guest 1989 .ssh_command( 1990 r#"dmesg | grep "smpboot: Allowing" | sed "s/\[\ *[0-9.]*\] //""# 1991 ) 1992 .unwrap() 1993 .trim(), 1994 "smpboot: Allowing 4 CPUs, 2 hotplug CPUs" 1995 ); 1996 #[cfg(target_arch = "aarch64")] 1997 assert_eq!( 1998 guest 1999 .ssh_command( 2000 r#"dmesg | grep "smp: Brought up" | sed "s/\[\ *[0-9.]*\] //""# 2001 ) 2002 .unwrap() 2003 .trim(), 2004 "smp: Brought up 1 node, 2 CPUs" 2005 ); 2006 }); 2007 2008 let _ = child.kill(); 2009 let output = child.wait_with_output().unwrap(); 2010 2011 handle_child_output(r, &output); 2012 } 2013 2014 #[test] 2015 #[cfg(target_arch = "x86_64")] 2016 fn test_cpu_topology_421() { 2017 test_cpu_topology(4, 2, 1); 2018 } 2019 2020 #[test] 2021 #[cfg(target_arch = "x86_64")] 2022 fn test_cpu_topology_142() { 2023 test_cpu_topology(1, 4, 2); 2024 } 2025 2026 #[test] 2027 #[cfg(target_arch = "x86_64")] 2028 fn test_cpu_topology_262() { 2029 test_cpu_topology(2, 6, 2); 2030 } 2031 2032 #[test] 2033 #[cfg(target_arch = "x86_64")] 2034 fn test_cpu_physical_bits() { 2035 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2036 let guest = Guest::new(Box::new(focal)); 2037 let max_phys_bits: u8 = 36; 2038 let mut child = GuestCommand::new(&guest) 2039 .args(&["--cpus", &format!("max_phys_bits={}", max_phys_bits)]) 2040 .args(&["--memory", "size=512M"]) 2041 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2042 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2043 .default_disks() 2044 .default_net() 2045 .capture_output() 2046 .spawn() 2047 .unwrap(); 2048 2049 let r = std::panic::catch_unwind(|| { 2050 guest.wait_vm_boot(None).unwrap(); 2051 2052 assert!( 2053 guest 2054 .ssh_command("lscpu | grep \"Address sizes:\" | cut -f 2 -d \":\" | sed \"s# *##\" | cut -f 1 -d \" \"") 2055 .unwrap() 2056 .trim() 2057 .parse::<u8>() 2058 .unwrap_or(max_phys_bits + 1) <= max_phys_bits, 2059 ); 2060 }); 2061 2062 let _ = child.kill(); 2063 let output = child.wait_with_output().unwrap(); 2064 2065 handle_child_output(r, &output); 2066 } 2067 2068 #[test] 2069 fn test_large_vm() { 2070 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2071 let guest = Guest::new(Box::new(focal)); 2072 let mut cmd = GuestCommand::new(&guest); 2073 cmd.args(&["--cpus", "boot=48"]) 2074 .args(&["--memory", "size=5120M"]) 2075 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2076 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2077 .args(&["--serial", "tty"]) 2078 .args(&["--console", "off"]) 2079 .capture_output() 2080 .default_disks() 2081 .default_net(); 2082 2083 let mut child = cmd.spawn().unwrap(); 2084 2085 guest.wait_vm_boot(None).unwrap(); 2086 2087 let r = std::panic::catch_unwind(|| { 2088 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 48); 2089 assert_eq!( 2090 guest 2091 .ssh_command( 2092 "lscpu | grep \"On-line\" | cut -f 2 -d \":\" | sed \"s# *##\"" 2093 ) 2094 .unwrap() 2095 .trim(), 2096 "0-47" 2097 ); 2098 2099 assert!(guest.get_total_memory().unwrap_or_default() > 5_000_000); 2100 }); 2101 2102 let _ = child.kill(); 2103 let output = child.wait_with_output().unwrap(); 2104 2105 handle_child_output(r, &output); 2106 } 2107 2108 #[test] 2109 fn test_huge_memory() { 2110 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2111 let guest = Guest::new(Box::new(focal)); 2112 let mut cmd = GuestCommand::new(&guest); 2113 cmd.args(&["--cpus", "boot=1"]) 2114 .args(&["--memory", "size=128G"]) 2115 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2116 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2117 .capture_output() 2118 .default_disks() 2119 .default_net(); 2120 2121 let mut child = cmd.spawn().unwrap(); 2122 2123 guest.wait_vm_boot(Some(120)).unwrap(); 2124 2125 let r = std::panic::catch_unwind(|| { 2126 assert!(guest.get_total_memory().unwrap_or_default() > 128_000_000); 2127 }); 2128 2129 let _ = child.kill(); 2130 let output = child.wait_with_output().unwrap(); 2131 2132 handle_child_output(r, &output); 2133 } 2134 2135 #[test] 2136 fn test_power_button() { 2137 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2138 let guest = Guest::new(Box::new(focal)); 2139 let mut cmd = GuestCommand::new(&guest); 2140 let api_socket = temp_api_path(&guest.tmp_dir); 2141 2142 cmd.args(&["--cpus", "boot=1"]) 2143 .args(&["--memory", "size=512M"]) 2144 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2145 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2146 .capture_output() 2147 .default_disks() 2148 .default_net() 2149 .args(&["--api-socket", &api_socket]); 2150 2151 let child = cmd.spawn().unwrap(); 2152 2153 let r = std::panic::catch_unwind(|| { 2154 guest.wait_vm_boot(None).unwrap(); 2155 assert!(remote_command(&api_socket, "power-button", None)); 2156 }); 2157 2158 let output = child.wait_with_output().unwrap(); 2159 assert!(output.status.success()); 2160 handle_child_output(r, &output); 2161 } 2162 2163 #[test] 2164 #[cfg(target_arch = "x86_64")] 2165 fn test_user_defined_memory_regions() { 2166 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2167 let guest = Guest::new(Box::new(focal)); 2168 let api_socket = temp_api_path(&guest.tmp_dir); 2169 2170 let kernel_path = direct_kernel_boot_path(); 2171 2172 let mut child = GuestCommand::new(&guest) 2173 .args(&["--cpus", "boot=1"]) 2174 .args(&["--memory", "size=0,hotplug_method=virtio-mem"]) 2175 .args(&[ 2176 "--memory-zone", 2177 "id=mem0,size=1G,hotplug_size=2G", 2178 "id=mem1,size=1G,file=/dev/shm", 2179 "id=mem2,size=1G,host_numa_node=0,hotplug_size=2G", 2180 ]) 2181 .args(&["--kernel", kernel_path.to_str().unwrap()]) 2182 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2183 .args(&["--api-socket", &api_socket]) 2184 .capture_output() 2185 .default_disks() 2186 .default_net() 2187 .spawn() 2188 .unwrap(); 2189 2190 let r = std::panic::catch_unwind(|| { 2191 guest.wait_vm_boot(None).unwrap(); 2192 2193 assert!(guest.get_total_memory().unwrap_or_default() > 2_880_000); 2194 2195 guest.enable_memory_hotplug(); 2196 2197 resize_zone_command(&api_socket, "mem0", "3G"); 2198 thread::sleep(std::time::Duration::new(5, 0)); 2199 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2200 resize_zone_command(&api_socket, "mem2", "3G"); 2201 thread::sleep(std::time::Duration::new(5, 0)); 2202 assert!(guest.get_total_memory().unwrap_or_default() > 6_720_000); 2203 resize_zone_command(&api_socket, "mem0", "2G"); 2204 thread::sleep(std::time::Duration::new(5, 0)); 2205 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 2206 resize_zone_command(&api_socket, "mem2", "2G"); 2207 thread::sleep(std::time::Duration::new(5, 0)); 2208 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2209 2210 guest.reboot_linux(0, None); 2211 2212 // Check the amount of RAM after reboot 2213 assert!(guest.get_total_memory().unwrap_or_default() > 4_800_000); 2214 assert!(guest.get_total_memory().unwrap_or_default() < 5_760_000); 2215 2216 // Check if we can still resize down to the initial 'boot'size 2217 resize_zone_command(&api_socket, "mem0", "1G"); 2218 thread::sleep(std::time::Duration::new(5, 0)); 2219 assert!(guest.get_total_memory().unwrap_or_default() < 4_800_000); 2220 resize_zone_command(&api_socket, "mem2", "1G"); 2221 thread::sleep(std::time::Duration::new(5, 0)); 2222 assert!(guest.get_total_memory().unwrap_or_default() < 3_840_000); 2223 }); 2224 2225 let _ = child.kill(); 2226 let output = child.wait_with_output().unwrap(); 2227 2228 handle_child_output(r, &output); 2229 } 2230 2231 #[test] 2232 #[cfg(feature = "acpi")] 2233 fn test_guest_numa_nodes() { 2234 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2235 let guest = Guest::new(Box::new(focal)); 2236 let api_socket = temp_api_path(&guest.tmp_dir); 2237 2238 #[cfg(target_arch = "x86_64")] 2239 let kernel_path = direct_kernel_boot_path(); 2240 #[cfg(target_arch = "aarch64")] 2241 let kernel_path = edk2_path(); 2242 2243 let mut child = GuestCommand::new(&guest) 2244 .args(&["--cpus", "boot=6,max=12"]) 2245 .args(&["--memory", "size=0,hotplug_method=virtio-mem"]) 2246 .args(&[ 2247 "--memory-zone", 2248 "id=mem0,size=1G,hotplug_size=3G", 2249 "id=mem1,size=2G,hotplug_size=3G", 2250 "id=mem2,size=3G,hotplug_size=3G", 2251 ]) 2252 .args(&[ 2253 "--numa", 2254 "guest_numa_id=0,cpus=0-2:9,distances=1@15:2@20,memory_zones=mem0", 2255 "guest_numa_id=1,cpus=3-4:6-8,distances=0@20:2@25,memory_zones=mem1", 2256 "guest_numa_id=2,cpus=5:10-11,distances=0@25:1@30,memory_zones=mem2", 2257 ]) 2258 .args(&["--kernel", kernel_path.to_str().unwrap()]) 2259 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2260 .args(&["--api-socket", &api_socket]) 2261 .capture_output() 2262 .default_disks() 2263 .default_net() 2264 .spawn() 2265 .unwrap(); 2266 2267 let r = std::panic::catch_unwind(|| { 2268 guest.wait_vm_boot(None).unwrap(); 2269 2270 // Check each NUMA node has been assigned the right amount of 2271 // memory. 2272 assert!(guest.get_numa_node_memory(0).unwrap_or_default() > 960_000); 2273 assert!(guest.get_numa_node_memory(1).unwrap_or_default() > 1_920_000); 2274 assert!(guest.get_numa_node_memory(2).unwrap_or_default() > 2_880_000); 2275 2276 // Check each NUMA node has been assigned the right CPUs set. 2277 guest.check_numa_node_cpus(0, vec![0, 1, 2]).unwrap(); 2278 guest.check_numa_node_cpus(1, vec![3, 4]).unwrap(); 2279 guest.check_numa_node_cpus(2, vec![5]).unwrap(); 2280 2281 // Check each NUMA node has been assigned the right distances. 2282 assert!(guest.check_numa_node_distances(0, "10 15 20").unwrap()); 2283 assert!(guest.check_numa_node_distances(1, "20 10 25").unwrap()); 2284 assert!(guest.check_numa_node_distances(2, "25 30 10").unwrap()); 2285 2286 // AArch64 currently does not support hotplug, and therefore we only 2287 // test hotplug-related function on x86_64 here. 2288 #[cfg(target_arch = "x86_64")] 2289 { 2290 guest.enable_memory_hotplug(); 2291 2292 // Resize every memory zone and check each associated NUMA node 2293 // has been assigned the right amount of memory. 2294 resize_zone_command(&api_socket, "mem0", "4G"); 2295 thread::sleep(std::time::Duration::new(5, 0)); 2296 assert!(guest.get_numa_node_memory(0).unwrap_or_default() > 3_840_000); 2297 resize_zone_command(&api_socket, "mem1", "4G"); 2298 thread::sleep(std::time::Duration::new(5, 0)); 2299 assert!(guest.get_numa_node_memory(1).unwrap_or_default() > 3_840_000); 2300 resize_zone_command(&api_socket, "mem2", "4G"); 2301 thread::sleep(std::time::Duration::new(5, 0)); 2302 assert!(guest.get_numa_node_memory(2).unwrap_or_default() > 3_840_000); 2303 2304 // Resize to the maximum amount of CPUs and check each NUMA 2305 // node has been assigned the right CPUs set. 2306 resize_command(&api_socket, Some(12), None, None); 2307 guest.check_numa_node_cpus(0, vec![0, 1, 2, 9]).unwrap(); 2308 guest.check_numa_node_cpus(1, vec![3, 4, 6, 7, 8]).unwrap(); 2309 guest.check_numa_node_cpus(2, vec![5, 10, 11]).unwrap(); 2310 } 2311 }); 2312 2313 let _ = child.kill(); 2314 let output = child.wait_with_output().unwrap(); 2315 2316 handle_child_output(r, &output); 2317 } 2318 2319 #[test] 2320 fn test_pci_msi() { 2321 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2322 let guest = Guest::new(Box::new(focal)); 2323 let mut cmd = GuestCommand::new(&guest); 2324 cmd.args(&["--cpus", "boot=1"]) 2325 .args(&["--memory", "size=512M"]) 2326 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2327 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2328 .capture_output() 2329 .default_disks() 2330 .default_net(); 2331 2332 let mut child = cmd.spawn().unwrap(); 2333 2334 guest.wait_vm_boot(None).unwrap(); 2335 2336 #[cfg(target_arch = "x86_64")] 2337 let grep_cmd = "grep -c PCI-MSI /proc/interrupts"; 2338 #[cfg(target_arch = "aarch64")] 2339 let grep_cmd = "grep -c ITS-MSI /proc/interrupts"; 2340 2341 let r = std::panic::catch_unwind(|| { 2342 assert_eq!( 2343 guest 2344 .ssh_command(grep_cmd) 2345 .unwrap() 2346 .trim() 2347 .parse::<u32>() 2348 .unwrap_or_default(), 2349 12 2350 ); 2351 }); 2352 2353 let _ = child.kill(); 2354 let output = child.wait_with_output().unwrap(); 2355 2356 handle_child_output(r, &output); 2357 } 2358 2359 #[test] 2360 fn test_direct_kernel_boot() { 2361 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2362 let guest = Guest::new(Box::new(focal)); 2363 2364 let kernel_path = direct_kernel_boot_path(); 2365 2366 let mut child = GuestCommand::new(&guest) 2367 .args(&["--cpus", "boot=1"]) 2368 .args(&["--memory", "size=512M"]) 2369 .args(&["--kernel", kernel_path.to_str().unwrap()]) 2370 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2371 .default_disks() 2372 .default_net() 2373 .capture_output() 2374 .spawn() 2375 .unwrap(); 2376 2377 let r = std::panic::catch_unwind(|| { 2378 guest.wait_vm_boot(None).unwrap(); 2379 2380 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 2381 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 2382 assert!(guest.get_entropy().unwrap_or_default() >= 900); 2383 2384 let grep_cmd = if cfg!(target_arch = "x86_64") { 2385 "grep -c PCI-MSI /proc/interrupts" 2386 } else { 2387 "grep -c ITS-MSI /proc/interrupts" 2388 }; 2389 assert_eq!( 2390 guest 2391 .ssh_command(grep_cmd) 2392 .unwrap() 2393 .trim() 2394 .parse::<u32>() 2395 .unwrap_or_default(), 2396 12 2397 ); 2398 }); 2399 2400 let _ = child.kill(); 2401 let output = child.wait_with_output().unwrap(); 2402 2403 handle_child_output(r, &output); 2404 } 2405 2406 fn _test_virtio_block(image_name: &str, disable_io_uring: bool) { 2407 let focal = UbuntuDiskConfig::new(image_name.to_string()); 2408 let guest = Guest::new(Box::new(focal)); 2409 2410 let mut workload_path = dirs::home_dir().unwrap(); 2411 workload_path.push("workloads"); 2412 2413 let mut blk_file_path = workload_path; 2414 blk_file_path.push("blk.img"); 2415 2416 let kernel_path = direct_kernel_boot_path(); 2417 2418 let mut cloud_child = GuestCommand::new(&guest) 2419 .args(&["--cpus", "boot=4"]) 2420 .args(&["--memory", "size=512M,shared=on"]) 2421 .args(&["--kernel", kernel_path.to_str().unwrap()]) 2422 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2423 .args(&[ 2424 "--disk", 2425 format!( 2426 "path={}", 2427 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 2428 ) 2429 .as_str(), 2430 format!( 2431 "path={}", 2432 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2433 ) 2434 .as_str(), 2435 format!( 2436 "path={},readonly=on,direct=on,num_queues=4,_disable_io_uring={}", 2437 blk_file_path.to_str().unwrap(), 2438 disable_io_uring 2439 ) 2440 .as_str(), 2441 ]) 2442 .default_net() 2443 .capture_output() 2444 .spawn() 2445 .unwrap(); 2446 2447 let r = std::panic::catch_unwind(|| { 2448 guest.wait_vm_boot(None).unwrap(); 2449 2450 // Check both if /dev/vdc exists and if the block size is 16M. 2451 assert_eq!( 2452 guest 2453 .ssh_command("lsblk | grep vdc | grep -c 16M") 2454 .unwrap() 2455 .trim() 2456 .parse::<u32>() 2457 .unwrap_or_default(), 2458 1 2459 ); 2460 2461 // Check both if /dev/vdc exists and if this block is RO. 2462 assert_eq!( 2463 guest 2464 .ssh_command("lsblk | grep vdc | awk '{print $5}'") 2465 .unwrap() 2466 .trim() 2467 .parse::<u32>() 2468 .unwrap_or_default(), 2469 1 2470 ); 2471 2472 // Check if the number of queues is 4. 2473 assert_eq!( 2474 guest 2475 .ssh_command("ls -ll /sys/block/vdc/mq | grep ^d | wc -l") 2476 .unwrap() 2477 .trim() 2478 .parse::<u32>() 2479 .unwrap_or_default(), 2480 4 2481 ); 2482 }); 2483 2484 let _ = cloud_child.kill(); 2485 let output = cloud_child.wait_with_output().unwrap(); 2486 2487 handle_child_output(r, &output); 2488 } 2489 2490 #[test] 2491 fn test_virtio_block() { 2492 _test_virtio_block(FOCAL_IMAGE_NAME, false) 2493 } 2494 2495 #[test] 2496 fn test_virtio_block_disable_io_uring() { 2497 _test_virtio_block(FOCAL_IMAGE_NAME, true) 2498 } 2499 2500 #[test] 2501 fn test_virtio_block_qcow2() { 2502 _test_virtio_block(FOCAL_IMAGE_NAME_QCOW2, false) 2503 } 2504 2505 #[test] 2506 fn test_virtio_block_vhd() { 2507 let mut workload_path = dirs::home_dir().unwrap(); 2508 workload_path.push("workloads"); 2509 2510 let mut raw_file_path = workload_path.clone(); 2511 let mut vhd_file_path = workload_path; 2512 raw_file_path.push(FOCAL_IMAGE_NAME); 2513 vhd_file_path.push(FOCAL_IMAGE_NAME_VHD); 2514 2515 // Generate VHD file from RAW file 2516 std::process::Command::new("qemu-img") 2517 .arg("convert") 2518 .arg("-p") 2519 .args(&["-f", "raw"]) 2520 .args(&["-O", "vpc"]) 2521 .args(&["-o", "subformat=fixed"]) 2522 .arg(raw_file_path.to_str().unwrap()) 2523 .arg(vhd_file_path.to_str().unwrap()) 2524 .output() 2525 .expect("Expect generating VHD image from RAW image"); 2526 2527 _test_virtio_block(FOCAL_IMAGE_NAME_VHD, false) 2528 } 2529 2530 #[test] 2531 fn test_vhost_user_net_default() { 2532 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, false, false) 2533 } 2534 2535 #[test] 2536 fn test_vhost_user_net_named_tap() { 2537 test_vhost_user_net( 2538 Some("mytap0"), 2539 2, 2540 &prepare_vhost_user_net_daemon, 2541 false, 2542 false, 2543 ) 2544 } 2545 2546 #[test] 2547 fn test_vhost_user_net_existing_tap() { 2548 test_vhost_user_net( 2549 Some("vunet-tap0"), 2550 2, 2551 &prepare_vhost_user_net_daemon, 2552 false, 2553 false, 2554 ) 2555 } 2556 2557 #[test] 2558 fn test_vhost_user_net_multiple_queues() { 2559 test_vhost_user_net(None, 4, &prepare_vhost_user_net_daemon, false, false) 2560 } 2561 2562 #[test] 2563 fn test_vhost_user_net_tap_multiple_queues() { 2564 test_vhost_user_net( 2565 Some("vunet-tap1"), 2566 4, 2567 &prepare_vhost_user_net_daemon, 2568 false, 2569 false, 2570 ) 2571 } 2572 2573 #[test] 2574 fn test_vhost_user_net_host_mac() { 2575 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, true, false) 2576 } 2577 2578 #[test] 2579 fn test_vhost_user_net_client_mode() { 2580 test_vhost_user_net(None, 2, &prepare_vhost_user_net_daemon, false, true) 2581 } 2582 2583 #[test] 2584 fn test_vhost_user_blk_default() { 2585 test_vhost_user_blk(2, false, false, Some(&prepare_vubd)) 2586 } 2587 2588 #[test] 2589 fn test_vhost_user_blk_readonly() { 2590 test_vhost_user_blk(1, true, false, Some(&prepare_vubd)) 2591 } 2592 2593 #[test] 2594 fn test_vhost_user_blk_direct() { 2595 test_vhost_user_blk(1, false, true, Some(&prepare_vubd)) 2596 } 2597 2598 #[test] 2599 fn test_boot_from_vhost_user_blk_default() { 2600 test_boot_from_vhost_user_blk(1, false, false, Some(&prepare_vubd)) 2601 } 2602 2603 #[test] 2604 #[cfg(target_arch = "x86_64")] 2605 fn test_split_irqchip() { 2606 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2607 let guest = Guest::new(Box::new(focal)); 2608 2609 let mut child = GuestCommand::new(&guest) 2610 .args(&["--cpus", "boot=1"]) 2611 .args(&["--memory", "size=512M"]) 2612 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2613 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2614 .default_disks() 2615 .default_net() 2616 .capture_output() 2617 .spawn() 2618 .unwrap(); 2619 2620 let r = std::panic::catch_unwind(|| { 2621 guest.wait_vm_boot(None).unwrap(); 2622 2623 assert_eq!( 2624 guest 2625 .ssh_command("grep -c IO-APIC.*timer /proc/interrupts || true") 2626 .unwrap() 2627 .trim() 2628 .parse::<u32>() 2629 .unwrap_or(1), 2630 0 2631 ); 2632 assert_eq!( 2633 guest 2634 .ssh_command("grep -c IO-APIC.*cascade /proc/interrupts || true") 2635 .unwrap() 2636 .trim() 2637 .parse::<u32>() 2638 .unwrap_or(1), 2639 0 2640 ); 2641 }); 2642 2643 let _ = child.kill(); 2644 let output = child.wait_with_output().unwrap(); 2645 2646 handle_child_output(r, &output); 2647 } 2648 2649 #[test] 2650 fn test_virtio_fs_dax_on_default_cache_size() { 2651 test_virtio_fs(true, None, "none", &prepare_virtiofsd, false) 2652 } 2653 2654 #[test] 2655 fn test_virtio_fs_dax_on_cache_size_1_gib() { 2656 test_virtio_fs(true, Some(0x4000_0000), "none", &prepare_virtiofsd, false) 2657 } 2658 2659 #[test] 2660 fn test_virtio_fs_dax_off() { 2661 test_virtio_fs(false, None, "none", &prepare_virtiofsd, false) 2662 } 2663 2664 #[test] 2665 fn test_virtio_fs_dax_on_default_cache_size_w_virtiofsd_rs_daemon() { 2666 test_virtio_fs(true, None, "none", &prepare_virtofsd_rs_daemon, false) 2667 } 2668 2669 #[test] 2670 fn test_virtio_fs_dax_on_cache_size_1_gib_w_virtiofsd_rs_daemon() { 2671 test_virtio_fs( 2672 true, 2673 Some(0x4000_0000), 2674 "none", 2675 &prepare_virtofsd_rs_daemon, 2676 false, 2677 ) 2678 } 2679 2680 #[test] 2681 fn test_virtio_fs_dax_off_w_virtiofsd_rs_daemon() { 2682 test_virtio_fs(false, None, "none", &prepare_virtofsd_rs_daemon, false) 2683 } 2684 2685 #[test] 2686 #[cfg(target_arch = "x86_64")] 2687 fn test_virtio_fs_hotplug_dax_on() { 2688 test_virtio_fs(true, None, "none", &prepare_virtiofsd, true) 2689 } 2690 2691 #[test] 2692 #[cfg(target_arch = "x86_64")] 2693 fn test_virtio_fs_hotplug_dax_off() { 2694 test_virtio_fs(false, None, "none", &prepare_virtiofsd, true) 2695 } 2696 2697 #[test] 2698 #[cfg(target_arch = "x86_64")] 2699 fn test_virtio_fs_hotplug_dax_on_w_virtiofsd_rs_daemon() { 2700 test_virtio_fs(true, None, "none", &prepare_virtofsd_rs_daemon, true) 2701 } 2702 2703 #[test] 2704 #[cfg(target_arch = "x86_64")] 2705 fn test_virtio_fs_hotplug_dax_off_w_virtiofsd_rs_daemon() { 2706 test_virtio_fs(false, None, "none", &prepare_virtofsd_rs_daemon, true) 2707 } 2708 2709 #[test] 2710 #[cfg(target_arch = "x86_64")] 2711 fn test_virtio_pmem_persist_writes() { 2712 test_virtio_pmem(false, false) 2713 } 2714 2715 #[test] 2716 #[cfg(target_arch = "x86_64")] 2717 fn test_virtio_pmem_discard_writes() { 2718 test_virtio_pmem(true, false) 2719 } 2720 2721 #[test] 2722 #[cfg(target_arch = "x86_64")] 2723 fn test_virtio_pmem_with_size() { 2724 test_virtio_pmem(true, true) 2725 } 2726 2727 #[test] 2728 fn test_boot_from_virtio_pmem() { 2729 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2730 let guest = Guest::new(Box::new(focal)); 2731 2732 let kernel_path = direct_kernel_boot_path(); 2733 2734 let mut child = GuestCommand::new(&guest) 2735 .args(&["--cpus", "boot=1"]) 2736 .args(&["--memory", "size=512M"]) 2737 .args(&["--kernel", kernel_path.to_str().unwrap()]) 2738 .args(&[ 2739 "--disk", 2740 format!( 2741 "path={}", 2742 guest.disk_config.disk(DiskType::CloudInit).unwrap() 2743 ) 2744 .as_str(), 2745 ]) 2746 .default_net() 2747 .args(&[ 2748 "--pmem", 2749 format!( 2750 "file={},size={}", 2751 guest.disk_config.disk(DiskType::OperatingSystem).unwrap(), 2752 fs::metadata(&guest.disk_config.disk(DiskType::OperatingSystem).unwrap()) 2753 .unwrap() 2754 .len() 2755 ) 2756 .as_str(), 2757 ]) 2758 .args(&[ 2759 "--cmdline", 2760 DIRECT_KERNEL_BOOT_CMDLINE 2761 .replace("vda1", "pmem0p1") 2762 .as_str(), 2763 ]) 2764 .capture_output() 2765 .spawn() 2766 .unwrap(); 2767 2768 let r = std::panic::catch_unwind(|| { 2769 guest.wait_vm_boot(None).unwrap(); 2770 2771 // Simple checks to validate the VM booted properly 2772 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 2773 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 2774 }); 2775 2776 let _ = child.kill(); 2777 let output = child.wait_with_output().unwrap(); 2778 2779 handle_child_output(r, &output); 2780 } 2781 2782 #[test] 2783 fn test_multiple_network_interfaces() { 2784 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2785 let guest = Guest::new(Box::new(focal)); 2786 2787 let kernel_path = direct_kernel_boot_path(); 2788 2789 let mut child = GuestCommand::new(&guest) 2790 .args(&["--cpus", "boot=1"]) 2791 .args(&["--memory", "size=512M"]) 2792 .args(&["--kernel", kernel_path.to_str().unwrap()]) 2793 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2794 .default_disks() 2795 .args(&[ 2796 "--net", 2797 guest.default_net_string().as_str(), 2798 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 2799 "tap=mytap1,mac=fe:1f:9e:e1:60:f2,ip=192.168.4.1,mask=255.255.255.0", 2800 ]) 2801 .capture_output() 2802 .spawn() 2803 .unwrap(); 2804 2805 let r = std::panic::catch_unwind(|| { 2806 guest.wait_vm_boot(None).unwrap(); 2807 2808 let tap_count = exec_host_command_output("ip link | grep -c mytap1"); 2809 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 2810 2811 // 3 network interfaces + default localhost ==> 4 interfaces 2812 assert_eq!( 2813 guest 2814 .ssh_command("ip -o link | wc -l") 2815 .unwrap() 2816 .trim() 2817 .parse::<u32>() 2818 .unwrap_or_default(), 2819 4 2820 ); 2821 }); 2822 2823 let _ = child.kill(); 2824 let output = child.wait_with_output().unwrap(); 2825 2826 handle_child_output(r, &output); 2827 } 2828 2829 #[test] 2830 fn test_serial_off() { 2831 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2832 let guest = Guest::new(Box::new(focal)); 2833 let mut child = GuestCommand::new(&guest) 2834 .args(&["--cpus", "boot=1"]) 2835 .args(&["--memory", "size=512M"]) 2836 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2837 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 2838 .default_disks() 2839 .default_net() 2840 .args(&["--serial", "off"]) 2841 .capture_output() 2842 .spawn() 2843 .unwrap(); 2844 2845 let r = std::panic::catch_unwind(|| { 2846 guest.wait_vm_boot(None).unwrap(); 2847 2848 // Test that there is no ttyS0 2849 assert_eq!( 2850 guest 2851 .ssh_command(GREP_SERIAL_IRQ_CMD) 2852 .unwrap() 2853 .trim() 2854 .parse::<u32>() 2855 .unwrap_or(1), 2856 0 2857 ); 2858 }); 2859 2860 let _ = child.kill(); 2861 let output = child.wait_with_output().unwrap(); 2862 2863 handle_child_output(r, &output); 2864 } 2865 2866 #[test] 2867 fn test_serial_null() { 2868 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2869 let guest = Guest::new(Box::new(focal)); 2870 let mut cmd = GuestCommand::new(&guest); 2871 #[cfg(target_arch = "x86_64")] 2872 let console_str: &str = "console=ttyS0"; 2873 #[cfg(target_arch = "aarch64")] 2874 let console_str: &str = "console=ttyAMA0"; 2875 2876 cmd.args(&["--cpus", "boot=1"]) 2877 .args(&["--memory", "size=512M"]) 2878 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2879 .args(&[ 2880 "--cmdline", 2881 DIRECT_KERNEL_BOOT_CMDLINE 2882 .replace("console=hvc0 ", console_str) 2883 .as_str(), 2884 ]) 2885 .default_disks() 2886 .default_net() 2887 .args(&["--serial", "null"]) 2888 .args(&["--console", "off"]) 2889 .capture_output(); 2890 2891 let mut child = cmd.spawn().unwrap(); 2892 2893 let r = std::panic::catch_unwind(|| { 2894 guest.wait_vm_boot(None).unwrap(); 2895 2896 // Test that there is a ttyS0 2897 assert_eq!( 2898 guest 2899 .ssh_command(GREP_SERIAL_IRQ_CMD) 2900 .unwrap() 2901 .trim() 2902 .parse::<u32>() 2903 .unwrap_or_default(), 2904 1 2905 ); 2906 }); 2907 2908 let _ = child.kill(); 2909 let output = child.wait_with_output().unwrap(); 2910 handle_child_output(r, &output); 2911 2912 let r = std::panic::catch_unwind(|| { 2913 assert!(!String::from_utf8_lossy(&output.stdout).contains(CONSOLE_TEST_STRING)); 2914 }); 2915 2916 handle_child_output(r, &output); 2917 } 2918 2919 #[test] 2920 fn test_serial_tty() { 2921 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2922 let guest = Guest::new(Box::new(focal)); 2923 2924 let kernel_path = direct_kernel_boot_path(); 2925 2926 #[cfg(target_arch = "x86_64")] 2927 let console_str: &str = "console=ttyS0"; 2928 #[cfg(target_arch = "aarch64")] 2929 let console_str: &str = "console=ttyAMA0"; 2930 2931 let mut child = GuestCommand::new(&guest) 2932 .args(&["--cpus", "boot=1"]) 2933 .args(&["--memory", "size=512M"]) 2934 .args(&["--kernel", kernel_path.to_str().unwrap()]) 2935 .args(&[ 2936 "--cmdline", 2937 DIRECT_KERNEL_BOOT_CMDLINE 2938 .replace("console=hvc0 ", console_str) 2939 .as_str(), 2940 ]) 2941 .default_disks() 2942 .default_net() 2943 .args(&["--serial", "tty"]) 2944 .args(&["--console", "off"]) 2945 .capture_output() 2946 .spawn() 2947 .unwrap(); 2948 2949 let r = std::panic::catch_unwind(|| { 2950 guest.wait_vm_boot(None).unwrap(); 2951 2952 // Test that there is a ttyS0 2953 assert_eq!( 2954 guest 2955 .ssh_command(GREP_SERIAL_IRQ_CMD) 2956 .unwrap() 2957 .trim() 2958 .parse::<u32>() 2959 .unwrap_or_default(), 2960 1 2961 ); 2962 }); 2963 2964 // This sleep is needed to wait for the login prompt 2965 thread::sleep(std::time::Duration::new(2, 0)); 2966 2967 let _ = child.kill(); 2968 let output = child.wait_with_output().unwrap(); 2969 handle_child_output(r, &output); 2970 2971 let r = std::panic::catch_unwind(|| { 2972 assert!(String::from_utf8_lossy(&output.stdout).contains(CONSOLE_TEST_STRING)); 2973 }); 2974 2975 handle_child_output(r, &output); 2976 } 2977 2978 #[test] 2979 fn test_serial_file() { 2980 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 2981 let guest = Guest::new(Box::new(focal)); 2982 2983 let serial_path = guest.tmp_dir.as_path().join("/tmp/serial-output"); 2984 #[cfg(target_arch = "x86_64")] 2985 let console_str: &str = "console=ttyS0"; 2986 #[cfg(target_arch = "aarch64")] 2987 let console_str: &str = "console=ttyAMA0"; 2988 2989 let mut child = GuestCommand::new(&guest) 2990 .args(&["--cpus", "boot=1"]) 2991 .args(&["--memory", "size=512M"]) 2992 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 2993 .args(&[ 2994 "--cmdline", 2995 DIRECT_KERNEL_BOOT_CMDLINE 2996 .replace("console=hvc0 ", console_str) 2997 .as_str(), 2998 ]) 2999 .default_disks() 3000 .default_net() 3001 .args(&[ 3002 "--serial", 3003 format!("file={}", serial_path.to_str().unwrap()).as_str(), 3004 ]) 3005 .capture_output() 3006 .spawn() 3007 .unwrap(); 3008 3009 let r = std::panic::catch_unwind(|| { 3010 guest.wait_vm_boot(None).unwrap(); 3011 3012 // Test that there is a ttyS0 3013 assert_eq!( 3014 guest 3015 .ssh_command(GREP_SERIAL_IRQ_CMD) 3016 .unwrap() 3017 .trim() 3018 .parse::<u32>() 3019 .unwrap_or_default(), 3020 1 3021 ); 3022 3023 guest.ssh_command("sudo shutdown -h now").unwrap(); 3024 }); 3025 3026 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 3027 let _ = child.kill(); 3028 let output = child.wait_with_output().unwrap(); 3029 handle_child_output(r, &output); 3030 3031 let r = std::panic::catch_unwind(|| { 3032 // Check that the cloud-hypervisor binary actually terminated 3033 assert!(output.status.success()); 3034 3035 // Do this check after shutdown of the VM as an easy way to ensure 3036 // all writes are flushed to disk 3037 let mut f = std::fs::File::open(serial_path).unwrap(); 3038 let mut buf = String::new(); 3039 f.read_to_string(&mut buf).unwrap(); 3040 assert!(buf.contains(CONSOLE_TEST_STRING)); 3041 }); 3042 3043 handle_child_output(r, &output); 3044 } 3045 3046 #[test] 3047 fn test_pty_interaction() { 3048 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3049 let guest = Guest::new(Box::new(focal)); 3050 let api_socket = temp_api_path(&guest.tmp_dir); 3051 let serial_option = if cfg!(target_arch = "x86_64") { 3052 " console=ttyS0" 3053 } else { 3054 " console=ttyAMA0" 3055 }; 3056 let cmdline = DIRECT_KERNEL_BOOT_CMDLINE.to_owned() + serial_option; 3057 3058 let mut child = GuestCommand::new(&guest) 3059 .args(&["--cpus", "boot=1"]) 3060 .args(&["--memory", "size=512M"]) 3061 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3062 .args(&["--cmdline", &cmdline]) 3063 .default_disks() 3064 .default_net() 3065 .args(&["--serial", "null"]) 3066 .args(&["--console", "pty"]) 3067 .args(&["--api-socket", &api_socket]) 3068 .spawn() 3069 .unwrap(); 3070 3071 let r = std::panic::catch_unwind(|| { 3072 guest.wait_vm_boot(None).unwrap(); 3073 // Get pty fd for console 3074 let console_path = get_pty_path(&api_socket, "console"); 3075 // TODO: Get serial pty test working 3076 let mut cf = std::fs::OpenOptions::new() 3077 .write(true) 3078 .read(true) 3079 .open(console_path) 3080 .unwrap(); 3081 3082 // Some dumb sleeps but we don't want to write 3083 // before the console is up and we don't want 3084 // to try and write the next line before the 3085 // login process is ready. 3086 thread::sleep(std::time::Duration::new(5, 0)); 3087 assert_eq!(cf.write(b"cloud\n").unwrap(), 6); 3088 thread::sleep(std::time::Duration::new(2, 0)); 3089 assert_eq!(cf.write(b"cloud123\n").unwrap(), 9); 3090 thread::sleep(std::time::Duration::new(2, 0)); 3091 assert_eq!(cf.write(b"echo test_pty_console\n").unwrap(), 22); 3092 thread::sleep(std::time::Duration::new(2, 0)); 3093 3094 // read pty and ensure they have a login shell 3095 // some fairly hacky workarounds to avoid looping 3096 // forever in case the channel is blocked getting output 3097 let ptyc = pty_read(cf); 3098 let mut empty = 0; 3099 let mut prev = String::new(); 3100 loop { 3101 thread::sleep(std::time::Duration::new(2, 0)); 3102 match ptyc.try_recv() { 3103 Ok(line) => { 3104 empty = 0; 3105 prev = prev + &line; 3106 if prev.contains("test_pty_console") { 3107 break; 3108 } 3109 } 3110 Err(mpsc::TryRecvError::Empty) => { 3111 empty += 1; 3112 if empty > 5 { 3113 panic!("No login on pty"); 3114 } 3115 } 3116 _ => panic!("No login on pty"), 3117 } 3118 } 3119 3120 guest.ssh_command("sudo shutdown -h now").unwrap(); 3121 }); 3122 3123 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 3124 let _ = child.kill(); 3125 let output = child.wait_with_output().unwrap(); 3126 handle_child_output(r, &output); 3127 3128 let r = std::panic::catch_unwind(|| { 3129 // Check that the cloud-hypervisor binary actually terminated 3130 assert!(output.status.success()) 3131 }); 3132 handle_child_output(r, &output); 3133 } 3134 3135 #[test] 3136 fn test_virtio_console() { 3137 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3138 let guest = Guest::new(Box::new(focal)); 3139 3140 let kernel_path = direct_kernel_boot_path(); 3141 3142 let mut child = GuestCommand::new(&guest) 3143 .args(&["--cpus", "boot=1"]) 3144 .args(&["--memory", "size=512M"]) 3145 .args(&["--kernel", kernel_path.to_str().unwrap()]) 3146 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3147 .default_disks() 3148 .default_net() 3149 .args(&["--console", "tty"]) 3150 .args(&["--serial", "null"]) 3151 .capture_output() 3152 .spawn() 3153 .unwrap(); 3154 3155 let text = String::from("On a branch floating down river a cricket, singing."); 3156 let cmd = format!("echo {} | sudo tee /dev/hvc0", text); 3157 3158 let r = std::panic::catch_unwind(|| { 3159 guest.wait_vm_boot(None).unwrap(); 3160 3161 #[cfg(feature = "acpi")] 3162 assert!(guest 3163 .does_device_vendor_pair_match("0x1043", "0x1af4") 3164 .unwrap_or_default()); 3165 3166 guest.ssh_command(&cmd).unwrap(); 3167 }); 3168 3169 let _ = child.kill(); 3170 let output = child.wait_with_output().unwrap(); 3171 handle_child_output(r, &output); 3172 3173 let r = std::panic::catch_unwind(|| { 3174 assert!(String::from_utf8_lossy(&output.stdout).contains(&text)); 3175 }); 3176 3177 handle_child_output(r, &output); 3178 } 3179 3180 #[test] 3181 fn test_console_file() { 3182 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3183 let guest = Guest::new(Box::new(focal)); 3184 3185 let console_path = guest.tmp_dir.as_path().join("/tmp/console-output"); 3186 let mut child = GuestCommand::new(&guest) 3187 .args(&["--cpus", "boot=1"]) 3188 .args(&["--memory", "size=512M"]) 3189 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3190 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3191 .default_disks() 3192 .default_net() 3193 .args(&[ 3194 "--console", 3195 format!("file={}", console_path.to_str().unwrap()).as_str(), 3196 ]) 3197 .capture_output() 3198 .spawn() 3199 .unwrap(); 3200 3201 guest.wait_vm_boot(None).unwrap(); 3202 3203 guest.ssh_command("sudo shutdown -h now").unwrap(); 3204 3205 let _ = child.wait_timeout(std::time::Duration::from_secs(20)); 3206 let _ = child.kill(); 3207 let output = child.wait_with_output().unwrap(); 3208 3209 let r = std::panic::catch_unwind(|| { 3210 // Check that the cloud-hypervisor binary actually terminated 3211 assert!(output.status.success()); 3212 3213 // Do this check after shutdown of the VM as an easy way to ensure 3214 // all writes are flushed to disk 3215 let mut f = std::fs::File::open(console_path).unwrap(); 3216 let mut buf = String::new(); 3217 f.read_to_string(&mut buf).unwrap(); 3218 3219 if !buf.contains(CONSOLE_TEST_STRING) { 3220 eprintln!( 3221 "\n\n==== Console file output ====\n\n{}\n\n==== End console file output ====", 3222 buf 3223 ); 3224 } 3225 assert!(buf.contains(CONSOLE_TEST_STRING)); 3226 }); 3227 3228 handle_child_output(r, &output); 3229 } 3230 3231 #[test] 3232 #[cfg(target_arch = "x86_64")] 3233 // The VFIO integration test starts cloud-hypervisor guest with 3 TAP 3234 // backed networking interfaces, bound through a simple bridge on the host. 3235 // So if the nested cloud-hypervisor succeeds in getting a directly 3236 // assigned interface from its cloud-hypervisor host, we should be able to 3237 // ssh into it, and verify that it's running with the right kernel command 3238 // line (We tag the command line from cloud-hypervisor for that purpose). 3239 // The third device is added to validate that hotplug works correctly since 3240 // it is being added to the L2 VM through hotplugging mechanism. 3241 // Also, we pass-through a vitio-blk device to the L2 VM to test the 32-bit 3242 // vfio device support 3243 fn test_vfio() { 3244 setup_vfio_network_interfaces(); 3245 3246 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3247 let guest = Guest::new_from_ip_range(Box::new(focal), "172.18", 0); 3248 3249 let mut workload_path = dirs::home_dir().unwrap(); 3250 workload_path.push("workloads"); 3251 3252 let kernel_path = direct_kernel_boot_path(); 3253 3254 let mut vfio_path = workload_path.clone(); 3255 vfio_path.push("vfio"); 3256 3257 let mut cloud_init_vfio_base_path = vfio_path.clone(); 3258 cloud_init_vfio_base_path.push("cloudinit.img"); 3259 3260 // We copy our cloudinit into the vfio mount point, for the nested 3261 // cloud-hypervisor guest to use. 3262 rate_limited_copy( 3263 &guest.disk_config.disk(DiskType::CloudInit).unwrap(), 3264 &cloud_init_vfio_base_path, 3265 ) 3266 .expect("copying of cloud-init disk failed"); 3267 3268 let mut vfio_disk_path = workload_path.clone(); 3269 vfio_disk_path.push("vfio.img"); 3270 3271 // Create the vfio disk image 3272 let output = Command::new("mkfs.ext4") 3273 .arg("-d") 3274 .arg(vfio_path.to_str().unwrap()) 3275 .arg(vfio_disk_path.to_str().unwrap()) 3276 .arg("2g") 3277 .output() 3278 .unwrap(); 3279 if !output.status.success() { 3280 eprintln!("{}", String::from_utf8_lossy(&output.stderr)); 3281 panic!("mkfs.ext4 command generated an error"); 3282 } 3283 3284 let mut blk_file_path = workload_path; 3285 blk_file_path.push("blk.img"); 3286 3287 let vfio_tap0 = "vfio-tap0"; 3288 let vfio_tap1 = "vfio-tap1"; 3289 let vfio_tap2 = "vfio-tap2"; 3290 let vfio_tap3 = "vfio-tap3"; 3291 3292 let mut child = GuestCommand::new(&guest) 3293 .args(&["--cpus", "boot=4"]) 3294 .args(&["--memory", "size=2G,hugepages=on,shared=on"]) 3295 .args(&["--kernel", kernel_path.to_str().unwrap()]) 3296 .args(&[ 3297 "--disk", 3298 format!( 3299 "path={}", 3300 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 3301 ) 3302 .as_str(), 3303 format!( 3304 "path={}", 3305 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3306 ) 3307 .as_str(), 3308 format!("path={}", vfio_disk_path.to_str().unwrap()).as_str(), 3309 format!("path={},iommu=on", blk_file_path.to_str().unwrap()).as_str(), 3310 ]) 3311 .args(&[ 3312 "--cmdline", 3313 format!( 3314 "{} kvm-intel.nested=1 vfio_iommu_type1.allow_unsafe_interrupts", 3315 DIRECT_KERNEL_BOOT_CMDLINE 3316 ) 3317 .as_str(), 3318 ]) 3319 .args(&[ 3320 "--net", 3321 format!("tap={},mac={}", vfio_tap0, guest.network.guest_mac).as_str(), 3322 format!( 3323 "tap={},mac={},iommu=on", 3324 vfio_tap1, guest.network.l2_guest_mac1 3325 ) 3326 .as_str(), 3327 format!( 3328 "tap={},mac={},iommu=on", 3329 vfio_tap2, guest.network.l2_guest_mac2 3330 ) 3331 .as_str(), 3332 format!( 3333 "tap={},mac={},iommu=on", 3334 vfio_tap3, guest.network.l2_guest_mac3 3335 ) 3336 .as_str(), 3337 ]) 3338 .capture_output() 3339 .spawn() 3340 .unwrap(); 3341 3342 thread::sleep(std::time::Duration::new(30, 0)); 3343 3344 let r = std::panic::catch_unwind(|| { 3345 guest.ssh_command_l1("sudo systemctl start vfio").unwrap(); 3346 thread::sleep(std::time::Duration::new(120, 0)); 3347 3348 // We booted our cloud hypervisor L2 guest with a "VFIOTAG" tag 3349 // added to its kernel command line. 3350 // Let's ssh into it and verify that it's there. If it is it means 3351 // we're in the right guest (The L2 one) because the QEMU L1 guest 3352 // does not have this command line tag. 3353 assert_eq!( 3354 guest 3355 .ssh_command_l2_1("grep -c VFIOTAG /proc/cmdline") 3356 .unwrap() 3357 .trim() 3358 .parse::<u32>() 3359 .unwrap_or_default(), 3360 1 3361 ); 3362 3363 // Let's also verify from the second virtio-net device passed to 3364 // the L2 VM. 3365 assert_eq!( 3366 guest 3367 .ssh_command_l2_2("grep -c VFIOTAG /proc/cmdline") 3368 .unwrap() 3369 .trim() 3370 .parse::<u32>() 3371 .unwrap_or_default(), 3372 1 3373 ); 3374 3375 // Check the amount of PCI devices appearing in L2 VM. 3376 assert_eq!( 3377 guest 3378 .ssh_command_l2_1("ls /sys/bus/pci/devices | wc -l") 3379 .unwrap() 3380 .trim() 3381 .parse::<u32>() 3382 .unwrap_or_default(), 3383 8, 3384 ); 3385 3386 // Check both if /dev/vdc exists and if the block size is 16M in L2 VM 3387 assert_eq!( 3388 guest 3389 .ssh_command_l2_1("lsblk | grep vdc | grep -c 16M") 3390 .unwrap() 3391 .trim() 3392 .parse::<u32>() 3393 .unwrap_or_default(), 3394 1 3395 ); 3396 3397 // Hotplug an extra virtio-net device through L2 VM. 3398 guest.ssh_command_l1( 3399 "echo 0000:00:09.0 | sudo tee /sys/bus/pci/devices/0000:00:09.0/driver/unbind", 3400 ).unwrap(); 3401 guest 3402 .ssh_command_l1( 3403 "echo 0000:00:09.0 | sudo tee /sys/bus/pci/drivers/vfio-pci/bind", 3404 ) 3405 .unwrap(); 3406 let vfio_hotplug_output = guest 3407 .ssh_command_l1( 3408 "sudo /mnt/ch-remote \ 3409 --api-socket=/tmp/ch_api.sock \ 3410 add-device path=/sys/bus/pci/devices/0000:00:09.0,id=vfio123", 3411 ) 3412 .unwrap(); 3413 assert!( 3414 vfio_hotplug_output.contains("{\"id\":\"vfio123\",\"bdf\":\"0000:00:08.0\"}") 3415 ); 3416 3417 thread::sleep(std::time::Duration::new(10, 0)); 3418 3419 // Let's also verify from the third virtio-net device passed to 3420 // the L2 VM. This third device has been hotplugged through the L2 3421 // VM, so this is our way to validate hotplug works for VFIO PCI. 3422 assert_eq!( 3423 guest 3424 .ssh_command_l2_3("grep -c VFIOTAG /proc/cmdline") 3425 .unwrap() 3426 .trim() 3427 .parse::<u32>() 3428 .unwrap_or_default(), 3429 1 3430 ); 3431 3432 // Check the amount of PCI devices appearing in L2 VM. 3433 // There should be one more device than before, raising the count 3434 // up to 9 PCI devices. 3435 assert_eq!( 3436 guest 3437 .ssh_command_l2_1("ls /sys/bus/pci/devices | wc -l") 3438 .unwrap() 3439 .trim() 3440 .parse::<u32>() 3441 .unwrap_or_default(), 3442 9, 3443 ); 3444 3445 // Let's now verify that we can correctly remove the virtio-net 3446 // device through the "remove-device" command responsible for 3447 // unplugging VFIO devices. 3448 guest 3449 .ssh_command_l1( 3450 "sudo /mnt/ch-remote \ 3451 --api-socket=/tmp/ch_api.sock \ 3452 remove-device vfio123", 3453 ) 3454 .unwrap(); 3455 thread::sleep(std::time::Duration::new(10, 0)); 3456 3457 // Check the amount of PCI devices appearing in L2 VM is back down 3458 // to 8 devices. 3459 assert_eq!( 3460 guest 3461 .ssh_command_l2_1("ls /sys/bus/pci/devices | wc -l") 3462 .unwrap() 3463 .trim() 3464 .parse::<u32>() 3465 .unwrap_or_default(), 3466 8, 3467 ); 3468 3469 // Perform memory hotplug in L2 and validate the memory is showing 3470 // up as expected. In order to check, we will use the virtio-net 3471 // device already passed through L2 as a VFIO device, this will 3472 // verify that VFIO devices are functional with memory hotplug. 3473 assert!(guest.get_total_memory_l2().unwrap_or_default() > 480_000); 3474 guest.ssh_command_l2_1( 3475 "sudo bash -c 'echo online > /sys/devices/system/memory/auto_online_blocks'", 3476 ).unwrap(); 3477 guest 3478 .ssh_command_l1( 3479 "sudo /mnt/ch-remote \ 3480 --api-socket=/tmp/ch_api.sock \ 3481 resize --memory=1073741824", 3482 ) 3483 .unwrap(); 3484 assert!(guest.get_total_memory_l2().unwrap_or_default() > 960_000); 3485 }); 3486 3487 let _ = child.kill(); 3488 let output = child.wait_with_output().unwrap(); 3489 3490 cleanup_vfio_network_interfaces(); 3491 3492 handle_child_output(r, &output); 3493 } 3494 3495 #[test] 3496 fn test_direct_kernel_boot_noacpi() { 3497 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3498 let guest = Guest::new(Box::new(focal)); 3499 3500 let kernel_path = direct_kernel_boot_path(); 3501 3502 let mut child = GuestCommand::new(&guest) 3503 .args(&["--cpus", "boot=1"]) 3504 .args(&["--memory", "size=512M"]) 3505 .args(&["--kernel", kernel_path.to_str().unwrap()]) 3506 .args(&[ 3507 "--cmdline", 3508 format!("{} acpi=off", DIRECT_KERNEL_BOOT_CMDLINE).as_str(), 3509 ]) 3510 .default_disks() 3511 .default_net() 3512 .capture_output() 3513 .spawn() 3514 .unwrap(); 3515 3516 let r = std::panic::catch_unwind(|| { 3517 guest.wait_vm_boot(None).unwrap(); 3518 3519 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 1); 3520 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 3521 assert!(guest.get_entropy().unwrap_or_default() >= 900); 3522 }); 3523 3524 let _ = child.kill(); 3525 let output = child.wait_with_output().unwrap(); 3526 3527 handle_child_output(r, &output); 3528 } 3529 3530 #[test] 3531 fn test_reboot() { 3532 let bionic = UbuntuDiskConfig::new(BIONIC_IMAGE_NAME.to_string()); 3533 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3534 3535 vec![Box::new(bionic), Box::new(focal)] 3536 .drain(..) 3537 .for_each(|disk_config| { 3538 let guest = Guest::new(disk_config); 3539 3540 let mut cmd = GuestCommand::new(&guest); 3541 cmd.args(&["--cpus", "boot=1"]) 3542 .args(&["--memory", "size=512M"]) 3543 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3544 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3545 .default_disks() 3546 .default_net() 3547 .capture_output(); 3548 3549 let mut child = cmd.spawn().unwrap(); 3550 3551 let r = std::panic::catch_unwind(|| { 3552 guest.wait_vm_boot(Some(120)).unwrap(); 3553 3554 let fd_count_1 = get_fd_count(child.id()); 3555 guest.reboot_linux(0, Some(120)); 3556 let fd_count_2 = get_fd_count(child.id()); 3557 assert_eq!(fd_count_1, fd_count_2); 3558 3559 guest.ssh_command("sudo shutdown -h now").unwrap(); 3560 }); 3561 3562 let _ = child.wait_timeout(std::time::Duration::from_secs(40)); 3563 let _ = child.kill(); 3564 let output = child.wait_with_output().unwrap(); 3565 handle_child_output(r, &output); 3566 3567 let r = std::panic::catch_unwind(|| { 3568 // Check that the cloud-hypervisor binary actually terminated 3569 assert!(output.status.success()); 3570 }); 3571 3572 handle_child_output(r, &output); 3573 }); 3574 } 3575 3576 #[test] 3577 fn test_virtio_vsock() { 3578 _test_virtio_vsock(false) 3579 } 3580 3581 #[test] 3582 #[cfg(target_arch = "x86_64")] 3583 fn test_virtio_vsock_hotplug() { 3584 _test_virtio_vsock(true); 3585 } 3586 3587 #[test] 3588 // Start cloud-hypervisor with no VM parameters, only the API server running. 3589 // From the API: Create a VM, boot it and check that it looks as expected. 3590 fn test_api_create_boot() { 3591 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3592 let guest = Guest::new(Box::new(focal)); 3593 3594 let api_socket = temp_api_path(&guest.tmp_dir); 3595 3596 let mut child = GuestCommand::new(&guest) 3597 .args(&["--api-socket", &api_socket]) 3598 .capture_output() 3599 .spawn() 3600 .unwrap(); 3601 3602 thread::sleep(std::time::Duration::new(1, 0)); 3603 3604 // Verify API server is running 3605 curl_command(&api_socket, "GET", "http://localhost/api/v1/vmm.ping", None); 3606 3607 // Create the VM first 3608 let cpu_count: u8 = 4; 3609 let http_body = guest.api_create_body(cpu_count); 3610 curl_command( 3611 &api_socket, 3612 "PUT", 3613 "http://localhost/api/v1/vm.create", 3614 Some(&http_body), 3615 ); 3616 3617 // Then boot it 3618 curl_command(&api_socket, "PUT", "http://localhost/api/v1/vm.boot", None); 3619 thread::sleep(std::time::Duration::new(20, 0)); 3620 3621 let r = std::panic::catch_unwind(|| { 3622 // Check that the VM booted as expected 3623 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 3624 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 3625 assert!(guest.get_entropy().unwrap_or_default() >= 900); 3626 }); 3627 3628 let _ = child.kill(); 3629 let output = child.wait_with_output().unwrap(); 3630 3631 handle_child_output(r, &output); 3632 } 3633 3634 #[test] 3635 // Start cloud-hypervisor with no VM parameters, only the API server running. 3636 // From the API: Create a VM, boot it and check that it looks as expected. 3637 // Then we pause the VM, check that it's no longer available. 3638 // Finally we resume the VM and check that it's available. 3639 fn test_api_pause_resume() { 3640 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3641 let guest = Guest::new(Box::new(focal)); 3642 3643 let api_socket = temp_api_path(&guest.tmp_dir); 3644 3645 let mut child = GuestCommand::new(&guest) 3646 .args(&["--api-socket", &api_socket]) 3647 .capture_output() 3648 .spawn() 3649 .unwrap(); 3650 3651 thread::sleep(std::time::Duration::new(1, 0)); 3652 3653 // Verify API server is running 3654 curl_command(&api_socket, "GET", "http://localhost/api/v1/vmm.ping", None); 3655 3656 // Create the VM first 3657 let cpu_count: u8 = 4; 3658 let http_body = guest.api_create_body(cpu_count); 3659 curl_command( 3660 &api_socket, 3661 "PUT", 3662 "http://localhost/api/v1/vm.create", 3663 Some(&http_body), 3664 ); 3665 3666 // Then boot it 3667 curl_command(&api_socket, "PUT", "http://localhost/api/v1/vm.boot", None); 3668 thread::sleep(std::time::Duration::new(20, 0)); 3669 3670 let r = std::panic::catch_unwind(|| { 3671 // Check that the VM booted as expected 3672 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 3673 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 3674 assert!(guest.get_entropy().unwrap_or_default() >= 900); 3675 3676 // We now pause the VM 3677 assert!(remote_command(&api_socket, "pause", None)); 3678 3679 // Check pausing again fails 3680 assert!(!remote_command(&api_socket, "pause", None)); 3681 3682 thread::sleep(std::time::Duration::new(2, 0)); 3683 3684 // SSH into the VM should fail 3685 assert!(ssh_command_ip( 3686 "grep -c processor /proc/cpuinfo", 3687 &guest.network.guest_ip, 3688 2, 3689 5 3690 ) 3691 .is_err()); 3692 3693 // Resume the VM 3694 assert!(remote_command(&api_socket, "resume", None)); 3695 3696 // Check resuming again fails 3697 assert!(!remote_command(&api_socket, "resume", None)); 3698 3699 thread::sleep(std::time::Duration::new(2, 0)); 3700 3701 // Now we should be able to SSH back in and get the right number of CPUs 3702 assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); 3703 }); 3704 3705 let _ = child.kill(); 3706 let output = child.wait_with_output().unwrap(); 3707 3708 handle_child_output(r, &output); 3709 } 3710 3711 #[test] 3712 #[cfg(target_arch = "x86_64")] 3713 // This test validates that it can find the virtio-iommu device at first. 3714 // It also verifies that both disks and the network card are attached to 3715 // the virtual IOMMU by looking at /sys/kernel/iommu_groups directory. 3716 // The last interesting part of this test is that it exercises the network 3717 // interface attached to the virtual IOMMU since this is the one used to 3718 // send all commands through SSH. 3719 fn test_virtio_iommu() { 3720 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3721 let guest = Guest::new(Box::new(focal)); 3722 3723 let kernel_path = direct_kernel_boot_path(); 3724 3725 let mut child = GuestCommand::new(&guest) 3726 .args(&["--cpus", "boot=1"]) 3727 .args(&["--memory", "size=512M"]) 3728 .args(&["--kernel", kernel_path.to_str().unwrap()]) 3729 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3730 .args(&[ 3731 "--disk", 3732 format!( 3733 "path={},iommu=on", 3734 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 3735 ) 3736 .as_str(), 3737 format!( 3738 "path={},iommu=on", 3739 guest.disk_config.disk(DiskType::CloudInit).unwrap() 3740 ) 3741 .as_str(), 3742 ]) 3743 .args(&["--net", guest.default_net_string_w_iommu().as_str()]) 3744 .capture_output() 3745 .spawn() 3746 .unwrap(); 3747 3748 let r = std::panic::catch_unwind(|| { 3749 guest.wait_vm_boot(None).unwrap(); 3750 3751 // Verify the virtio-iommu device is present. 3752 assert!(guest 3753 .does_device_vendor_pair_match("0x1057", "0x1af4") 3754 .unwrap_or_default()); 3755 3756 // Verify the first disk is located under IOMMU group 0. 3757 assert_eq!( 3758 guest 3759 .ssh_command("ls /sys/kernel/iommu_groups/0/devices") 3760 .unwrap() 3761 .trim(), 3762 "0000:00:02.0" 3763 ); 3764 3765 // Verify the second disk is located under IOMMU group 1. 3766 assert_eq!( 3767 guest 3768 .ssh_command("ls /sys/kernel/iommu_groups/1/devices") 3769 .unwrap() 3770 .trim(), 3771 "0000:00:03.0" 3772 ); 3773 3774 // Verify the network card is located under IOMMU group 2. 3775 assert_eq!( 3776 guest 3777 .ssh_command("ls /sys/kernel/iommu_groups/2/devices") 3778 .unwrap() 3779 .trim(), 3780 "0000:00:04.0" 3781 ); 3782 }); 3783 3784 let _ = child.kill(); 3785 let output = child.wait_with_output().unwrap(); 3786 3787 handle_child_output(r, &output); 3788 } 3789 3790 #[test] 3791 #[cfg(target_arch = "x86_64")] 3792 // We cannot force the software running in the guest to reprogram the BAR 3793 // with some different addresses, but we have a reliable way of testing it 3794 // with a standard Linux kernel. 3795 // By removing a device from the PCI tree, and then rescanning the tree, 3796 // Linux consistently chooses to reorganize the PCI device BARs to other 3797 // locations in the guest address space. 3798 // This test creates a dedicated PCI network device to be checked as being 3799 // properly probed first, then removing it, and adding it again by doing a 3800 // rescan. 3801 fn test_pci_bar_reprogramming() { 3802 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3803 let guest = Guest::new(Box::new(focal)); 3804 let mut child = GuestCommand::new(&guest) 3805 .args(&["--cpus", "boot=1"]) 3806 .args(&["--memory", "size=512M"]) 3807 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 3808 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3809 .default_disks() 3810 .args(&[ 3811 "--net", 3812 guest.default_net_string().as_str(), 3813 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 3814 ]) 3815 .capture_output() 3816 .spawn() 3817 .unwrap(); 3818 3819 let r = 3820 std::panic::catch_unwind(|| { 3821 guest.wait_vm_boot(None).unwrap(); 3822 3823 // 2 network interfaces + default localhost ==> 3 interfaces 3824 assert_eq!( 3825 guest 3826 .ssh_command("ip -o link | wc -l") 3827 .unwrap() 3828 .trim() 3829 .parse::<u32>() 3830 .unwrap_or_default(), 3831 3 3832 ); 3833 3834 let init_bar_addr = guest.ssh_command( 3835 "sudo awk '{print $1; exit}' /sys/bus/pci/devices/0000:00:05.0/resource", 3836 ).unwrap(); 3837 3838 // Remove the PCI device 3839 guest 3840 .ssh_command("echo 1 | sudo tee /sys/bus/pci/devices/0000:00:05.0/remove") 3841 .unwrap(); 3842 3843 // Only 1 network interface left + default localhost ==> 2 interfaces 3844 assert_eq!( 3845 guest 3846 .ssh_command("ip -o link | wc -l") 3847 .unwrap() 3848 .trim() 3849 .parse::<u32>() 3850 .unwrap_or_default(), 3851 2 3852 ); 3853 3854 // Remove the PCI device 3855 guest 3856 .ssh_command("echo 1 | sudo tee /sys/bus/pci/rescan") 3857 .unwrap(); 3858 3859 // Back to 2 network interface + default localhost ==> 3 interfaces 3860 assert_eq!( 3861 guest 3862 .ssh_command("ip -o link | wc -l") 3863 .unwrap() 3864 .trim() 3865 .parse::<u32>() 3866 .unwrap_or_default(), 3867 3 3868 ); 3869 3870 let new_bar_addr = guest.ssh_command( 3871 "sudo awk '{print $1; exit}' /sys/bus/pci/devices/0000:00:05.0/resource", 3872 ).unwrap(); 3873 3874 // Let's compare the BAR addresses for our virtio-net device. 3875 // They should be different as we expect the BAR reprogramming 3876 // to have happened. 3877 assert_ne!(init_bar_addr, new_bar_addr); 3878 }); 3879 3880 let _ = child.kill(); 3881 let output = child.wait_with_output().unwrap(); 3882 3883 handle_child_output(r, &output); 3884 } 3885 3886 #[test] 3887 fn test_memory_mergeable_off() { 3888 test_memory_mergeable(false) 3889 } 3890 3891 #[test] 3892 #[cfg(target_arch = "x86_64")] 3893 fn test_cpu_hotplug() { 3894 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3895 let guest = Guest::new(Box::new(focal)); 3896 let api_socket = temp_api_path(&guest.tmp_dir); 3897 3898 let kernel_path = direct_kernel_boot_path(); 3899 3900 let mut child = GuestCommand::new(&guest) 3901 .args(&["--cpus", "boot=2,max=4"]) 3902 .args(&["--memory", "size=512M"]) 3903 .args(&["--kernel", kernel_path.to_str().unwrap()]) 3904 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3905 .default_disks() 3906 .default_net() 3907 .args(&["--api-socket", &api_socket]) 3908 .capture_output() 3909 .spawn() 3910 .unwrap(); 3911 3912 let r = std::panic::catch_unwind(|| { 3913 guest.wait_vm_boot(None).unwrap(); 3914 3915 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 3916 3917 // Resize the VM 3918 let desired_vcpus = 4; 3919 resize_command(&api_socket, Some(desired_vcpus), None, None); 3920 3921 guest 3922 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 3923 .unwrap(); 3924 guest 3925 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 3926 .unwrap(); 3927 thread::sleep(std::time::Duration::new(10, 0)); 3928 assert_eq!( 3929 guest.get_cpu_count().unwrap_or_default(), 3930 u32::from(desired_vcpus) 3931 ); 3932 3933 guest.reboot_linux(0, None); 3934 3935 assert_eq!( 3936 guest.get_cpu_count().unwrap_or_default(), 3937 u32::from(desired_vcpus) 3938 ); 3939 3940 // Resize the VM 3941 let desired_vcpus = 2; 3942 resize_command(&api_socket, Some(desired_vcpus), None, None); 3943 3944 thread::sleep(std::time::Duration::new(10, 0)); 3945 assert_eq!( 3946 guest.get_cpu_count().unwrap_or_default(), 3947 u32::from(desired_vcpus) 3948 ); 3949 3950 // Resize the VM back up to 4 3951 let desired_vcpus = 4; 3952 resize_command(&api_socket, Some(desired_vcpus), None, None); 3953 3954 guest 3955 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 3956 .unwrap(); 3957 guest 3958 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 3959 .unwrap(); 3960 thread::sleep(std::time::Duration::new(10, 0)); 3961 assert_eq!( 3962 guest.get_cpu_count().unwrap_or_default(), 3963 u32::from(desired_vcpus) 3964 ); 3965 }); 3966 3967 let _ = child.kill(); 3968 let output = child.wait_with_output().unwrap(); 3969 3970 handle_child_output(r, &output); 3971 } 3972 3973 #[test] 3974 #[cfg(target_arch = "x86_64")] 3975 fn test_memory_hotplug() { 3976 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 3977 let guest = Guest::new(Box::new(focal)); 3978 let api_socket = temp_api_path(&guest.tmp_dir); 3979 3980 let kernel_path = direct_kernel_boot_path(); 3981 3982 let mut child = GuestCommand::new(&guest) 3983 .args(&["--cpus", "boot=2,max=4"]) 3984 .args(&["--memory", "size=512M,hotplug_size=8192M"]) 3985 .args(&["--kernel", kernel_path.to_str().unwrap()]) 3986 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 3987 .default_disks() 3988 .default_net() 3989 .args(&["--balloon", "size=0"]) 3990 .args(&["--api-socket", &api_socket]) 3991 .capture_output() 3992 .spawn() 3993 .unwrap(); 3994 3995 let r = std::panic::catch_unwind(|| { 3996 guest.wait_vm_boot(None).unwrap(); 3997 3998 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 3999 4000 guest.enable_memory_hotplug(); 4001 4002 // Add RAM to the VM 4003 let desired_ram = 1024 << 20; 4004 resize_command(&api_socket, None, Some(desired_ram), None); 4005 4006 thread::sleep(std::time::Duration::new(10, 0)); 4007 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4008 4009 // Use balloon to remove RAM from the VM 4010 let desired_balloon = 512 << 20; 4011 resize_command(&api_socket, None, None, Some(desired_balloon)); 4012 4013 thread::sleep(std::time::Duration::new(10, 0)); 4014 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4015 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 4016 4017 guest.reboot_linux(0, None); 4018 4019 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 4020 4021 // Use balloon add RAM to the VM 4022 let desired_balloon = 0; 4023 resize_command(&api_socket, None, None, Some(desired_balloon)); 4024 4025 thread::sleep(std::time::Duration::new(10, 0)); 4026 4027 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4028 4029 guest.enable_memory_hotplug(); 4030 4031 // Add RAM to the VM 4032 let desired_ram = 2048 << 20; 4033 resize_command(&api_socket, None, Some(desired_ram), None); 4034 4035 thread::sleep(std::time::Duration::new(10, 0)); 4036 assert!(guest.get_total_memory().unwrap_or_default() > 1_920_000); 4037 4038 // Remove RAM to the VM (only applies after reboot) 4039 let desired_ram = 1024 << 20; 4040 resize_command(&api_socket, None, Some(desired_ram), None); 4041 4042 guest.reboot_linux(1, None); 4043 4044 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4045 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 4046 }); 4047 4048 let _ = child.kill(); 4049 let output = child.wait_with_output().unwrap(); 4050 4051 handle_child_output(r, &output); 4052 } 4053 4054 #[test] 4055 #[cfg(target_arch = "x86_64")] 4056 fn test_virtio_mem() { 4057 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4058 let guest = Guest::new(Box::new(focal)); 4059 let api_socket = temp_api_path(&guest.tmp_dir); 4060 4061 let kernel_path = direct_kernel_boot_path(); 4062 4063 let mut child = GuestCommand::new(&guest) 4064 .args(&["--cpus", "boot=2,max=4"]) 4065 .args(&[ 4066 "--memory", 4067 "size=512M,hotplug_method=virtio-mem,hotplug_size=8192M", 4068 ]) 4069 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4070 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4071 .default_disks() 4072 .default_net() 4073 .args(&["--api-socket", &api_socket]) 4074 .capture_output() 4075 .spawn() 4076 .unwrap(); 4077 4078 let r = std::panic::catch_unwind(|| { 4079 guest.wait_vm_boot(None).unwrap(); 4080 4081 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4082 4083 guest.enable_memory_hotplug(); 4084 4085 // Add RAM to the VM 4086 let desired_ram = 1024 << 20; 4087 resize_command(&api_socket, None, Some(desired_ram), None); 4088 4089 thread::sleep(std::time::Duration::new(10, 0)); 4090 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4091 4092 // Add RAM to the VM 4093 let desired_ram = 2048 << 20; 4094 resize_command(&api_socket, None, Some(desired_ram), None); 4095 4096 thread::sleep(std::time::Duration::new(10, 0)); 4097 assert!(guest.get_total_memory().unwrap_or_default() > 1_920_000); 4098 4099 // Remove RAM from the VM 4100 let desired_ram = 1024 << 20; 4101 resize_command(&api_socket, None, Some(desired_ram), None); 4102 4103 thread::sleep(std::time::Duration::new(10, 0)); 4104 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4105 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 4106 4107 guest.reboot_linux(0, None); 4108 4109 // Check the amount of memory after reboot is 1GiB 4110 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4111 assert!(guest.get_total_memory().unwrap_or_default() < 1_920_000); 4112 4113 // Check we can still resize to 512MiB 4114 let desired_ram = 512 << 20; 4115 resize_command(&api_socket, None, Some(desired_ram), None); 4116 thread::sleep(std::time::Duration::new(10, 0)); 4117 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4118 assert!(guest.get_total_memory().unwrap_or_default() < 960_000); 4119 }); 4120 4121 let _ = child.kill(); 4122 let output = child.wait_with_output().unwrap(); 4123 4124 handle_child_output(r, &output); 4125 } 4126 4127 #[test] 4128 #[cfg(target_arch = "x86_64")] 4129 // Test both vCPU and memory resizing together 4130 fn test_resize() { 4131 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4132 let guest = Guest::new(Box::new(focal)); 4133 let api_socket = temp_api_path(&guest.tmp_dir); 4134 4135 let kernel_path = direct_kernel_boot_path(); 4136 4137 let mut child = GuestCommand::new(&guest) 4138 .args(&["--cpus", "boot=2,max=4"]) 4139 .args(&["--memory", "size=512M,hotplug_size=8192M"]) 4140 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4141 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4142 .default_disks() 4143 .default_net() 4144 .args(&["--api-socket", &api_socket]) 4145 .capture_output() 4146 .spawn() 4147 .unwrap(); 4148 4149 let r = std::panic::catch_unwind(|| { 4150 guest.wait_vm_boot(None).unwrap(); 4151 4152 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 2); 4153 assert!(guest.get_total_memory().unwrap_or_default() > 480_000); 4154 4155 guest.enable_memory_hotplug(); 4156 4157 // Resize the VM 4158 let desired_vcpus = 4; 4159 let desired_ram = 1024 << 20; 4160 resize_command(&api_socket, Some(desired_vcpus), Some(desired_ram), None); 4161 4162 guest 4163 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu2/online") 4164 .unwrap(); 4165 guest 4166 .ssh_command("echo 1 | sudo tee /sys/bus/cpu/devices/cpu3/online") 4167 .unwrap(); 4168 thread::sleep(std::time::Duration::new(10, 0)); 4169 assert_eq!( 4170 guest.get_cpu_count().unwrap_or_default(), 4171 u32::from(desired_vcpus) 4172 ); 4173 4174 assert!(guest.get_total_memory().unwrap_or_default() > 960_000); 4175 }); 4176 4177 let _ = child.kill(); 4178 let output = child.wait_with_output().unwrap(); 4179 4180 handle_child_output(r, &output); 4181 } 4182 4183 #[test] 4184 fn test_memory_overhead() { 4185 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4186 let guest = Guest::new(Box::new(focal)); 4187 4188 let kernel_path = direct_kernel_boot_path(); 4189 4190 let guest_memory_size_kb = 512 * 1024; 4191 4192 let mut child = GuestCommand::new(&guest) 4193 .args(&["--cpus", "boot=1"]) 4194 .args(&[ 4195 "--memory", 4196 format!("size={}K", guest_memory_size_kb).as_str(), 4197 ]) 4198 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4199 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4200 .default_disks() 4201 .capture_output() 4202 .spawn() 4203 .unwrap(); 4204 4205 thread::sleep(std::time::Duration::new(20, 0)); 4206 4207 let r = std::panic::catch_unwind(|| { 4208 // On AArch64 when acpi is enabled, there is a 4 MiB gap between the RAM 4209 // that the VMM gives and the guest can see. 4210 // This is a temporary solution, will be fixed in future. 4211 #[cfg(all(target_arch = "aarch64", feature = "acpi"))] 4212 let guest_memory_size_kb = guest_memory_size_kb - 4 * 1024; 4213 4214 let overhead = get_vmm_overhead(child.id(), guest_memory_size_kb); 4215 eprintln!( 4216 "Guest memory overhead: {} vs {}", 4217 overhead, MAXIMUM_VMM_OVERHEAD_KB 4218 ); 4219 assert!(overhead <= MAXIMUM_VMM_OVERHEAD_KB); 4220 }); 4221 4222 let _ = child.kill(); 4223 let output = child.wait_with_output().unwrap(); 4224 4225 handle_child_output(r, &output); 4226 } 4227 4228 #[test] 4229 #[cfg(target_arch = "x86_64")] 4230 fn test_disk_hotplug() { 4231 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4232 let guest = Guest::new(Box::new(focal)); 4233 4234 let kernel_path = direct_kernel_boot_path(); 4235 4236 let api_socket = temp_api_path(&guest.tmp_dir); 4237 4238 let mut child = GuestCommand::new(&guest) 4239 .args(&["--api-socket", &api_socket]) 4240 .args(&["--cpus", "boot=1"]) 4241 .args(&["--memory", "size=512M"]) 4242 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4243 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4244 .default_disks() 4245 .default_net() 4246 .capture_output() 4247 .spawn() 4248 .unwrap(); 4249 4250 let r = std::panic::catch_unwind(|| { 4251 guest.wait_vm_boot(None).unwrap(); 4252 4253 // Check /dev/vdc is not there 4254 assert_eq!( 4255 guest 4256 .ssh_command("lsblk | grep -c vdc.*16M || true") 4257 .unwrap() 4258 .trim() 4259 .parse::<u32>() 4260 .unwrap_or(1), 4261 0 4262 ); 4263 4264 // Now let's add the extra disk. 4265 let mut blk_file_path = dirs::home_dir().unwrap(); 4266 blk_file_path.push("workloads"); 4267 blk_file_path.push("blk.img"); 4268 let (cmd_success, cmd_output) = remote_command_w_output( 4269 &api_socket, 4270 "add-disk", 4271 Some(format!("path={},id=test0", blk_file_path.to_str().unwrap()).as_str()), 4272 ); 4273 assert!(cmd_success); 4274 assert!(String::from_utf8_lossy(&cmd_output) 4275 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 4276 4277 thread::sleep(std::time::Duration::new(10, 0)); 4278 4279 // Check that /dev/vdc exists and the block size is 16M. 4280 assert_eq!( 4281 guest 4282 .ssh_command("lsblk | grep vdc | grep -c 16M") 4283 .unwrap() 4284 .trim() 4285 .parse::<u32>() 4286 .unwrap_or_default(), 4287 1 4288 ); 4289 // And check the block device can be read. 4290 guest 4291 .ssh_command("sudo dd if=/dev/vdc of=/dev/null bs=1M iflag=direct count=16") 4292 .unwrap(); 4293 4294 // Let's remove it the extra disk. 4295 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 4296 thread::sleep(std::time::Duration::new(5, 0)); 4297 // And check /dev/vdc is not there 4298 assert_eq!( 4299 guest 4300 .ssh_command("lsblk | grep -c vdc.*16M || true") 4301 .unwrap() 4302 .trim() 4303 .parse::<u32>() 4304 .unwrap_or(1), 4305 0 4306 ); 4307 4308 // And add it back to validate unplug did work correctly. 4309 let (cmd_success, cmd_output) = remote_command_w_output( 4310 &api_socket, 4311 "add-disk", 4312 Some(format!("path={},id=test0", blk_file_path.to_str().unwrap()).as_str()), 4313 ); 4314 assert!(cmd_success); 4315 assert!(String::from_utf8_lossy(&cmd_output) 4316 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 4317 4318 thread::sleep(std::time::Duration::new(10, 0)); 4319 4320 // Check that /dev/vdc exists and the block size is 16M. 4321 assert_eq!( 4322 guest 4323 .ssh_command("lsblk | grep vdc | grep -c 16M") 4324 .unwrap() 4325 .trim() 4326 .parse::<u32>() 4327 .unwrap_or_default(), 4328 1 4329 ); 4330 // And check the block device can be read. 4331 guest 4332 .ssh_command("sudo dd if=/dev/vdc of=/dev/null bs=1M iflag=direct count=16") 4333 .unwrap(); 4334 4335 // Reboot the VM. 4336 guest.reboot_linux(0, None); 4337 4338 // Check still there after reboot 4339 assert_eq!( 4340 guest 4341 .ssh_command("lsblk | grep vdc | grep -c 16M") 4342 .unwrap() 4343 .trim() 4344 .parse::<u32>() 4345 .unwrap_or_default(), 4346 1 4347 ); 4348 4349 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 4350 4351 thread::sleep(std::time::Duration::new(20, 0)); 4352 4353 // Check device has gone away 4354 assert_eq!( 4355 guest 4356 .ssh_command("lsblk | grep -c vdc.*16M || true") 4357 .unwrap() 4358 .trim() 4359 .parse::<u32>() 4360 .unwrap_or(1), 4361 0 4362 ); 4363 4364 guest.reboot_linux(1, None); 4365 4366 // Check device still absent 4367 assert_eq!( 4368 guest 4369 .ssh_command("lsblk | grep -c vdc.*16M || true") 4370 .unwrap() 4371 .trim() 4372 .parse::<u32>() 4373 .unwrap_or(1), 4374 0 4375 ); 4376 }); 4377 4378 let _ = child.kill(); 4379 let output = child.wait_with_output().unwrap(); 4380 4381 handle_child_output(r, &output); 4382 } 4383 4384 #[test] 4385 #[cfg(target_arch = "x86_64")] 4386 fn test_virtio_balloon() { 4387 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4388 let guest = Guest::new(Box::new(focal)); 4389 4390 let kernel_path = direct_kernel_boot_path(); 4391 4392 let api_socket = temp_api_path(&guest.tmp_dir); 4393 4394 //Let's start a 4G guest with balloon occupied 2G memory 4395 let mut child = GuestCommand::new(&guest) 4396 .args(&["--api-socket", &api_socket]) 4397 .args(&["--cpus", "boot=1"]) 4398 .args(&["--memory", "size=4G"]) 4399 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4400 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4401 .args(&["--balloon", "size=2G,deflate_on_oom=on"]) 4402 .default_disks() 4403 .default_net() 4404 .capture_output() 4405 .spawn() 4406 .unwrap(); 4407 4408 let r = std::panic::catch_unwind(|| { 4409 guest.wait_vm_boot(None).unwrap(); 4410 4411 // Wait for balloon memory's initialization and check its size. 4412 // The virtio-balloon driver might take a few seconds to report the 4413 // balloon effective size back to the VMM. 4414 thread::sleep(std::time::Duration::new(20, 0)); 4415 4416 let orig_balloon = balloon_size(&api_socket); 4417 println!("The original balloon memory size is {} bytes", orig_balloon); 4418 assert!(orig_balloon == 2147483648); 4419 4420 // Two steps to verify if the 'deflate_on_oom' parameter works. 4421 // 1st: run a command in guest to eat up memory heavily, which 4422 // will consume much more memory than $(total_mem - balloon_size) 4423 // to trigger an oom. 4424 guest 4425 .ssh_command("sudo stress --vm 25 --vm-keep --vm-bytes 1G --timeout 20") 4426 .unwrap(); 4427 4428 // 2nd: check balloon_mem's value to verify balloon has been automatically deflated 4429 let deflated_balloon = balloon_size(&api_socket); 4430 println!( 4431 "After deflating, balloon memory size is {} bytes", 4432 deflated_balloon 4433 ); 4434 // Verify the balloon size deflated 4435 assert!(deflated_balloon < 2147483648); 4436 }); 4437 4438 let _ = child.kill(); 4439 let output = child.wait_with_output().unwrap(); 4440 4441 handle_child_output(r, &output); 4442 } 4443 4444 #[test] 4445 #[cfg(target_arch = "x86_64")] 4446 fn test_pmem_hotplug() { 4447 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4448 let guest = Guest::new(Box::new(focal)); 4449 4450 let kernel_path = direct_kernel_boot_path(); 4451 4452 let api_socket = temp_api_path(&guest.tmp_dir); 4453 4454 let mut child = GuestCommand::new(&guest) 4455 .args(&["--api-socket", &api_socket]) 4456 .args(&["--cpus", "boot=1"]) 4457 .args(&["--memory", "size=512M"]) 4458 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4459 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4460 .default_disks() 4461 .default_net() 4462 .capture_output() 4463 .spawn() 4464 .unwrap(); 4465 4466 let r = std::panic::catch_unwind(|| { 4467 guest.wait_vm_boot(None).unwrap(); 4468 4469 // Check /dev/pmem0 is not there 4470 assert_eq!( 4471 guest 4472 .ssh_command("lsblk | grep -c pmem0 || true") 4473 .unwrap() 4474 .trim() 4475 .parse::<u32>() 4476 .unwrap_or(1), 4477 0 4478 ); 4479 4480 let pmem_temp_file = TempFile::new().unwrap(); 4481 pmem_temp_file.as_file().set_len(128 << 20).unwrap(); 4482 let (cmd_success, cmd_output) = remote_command_w_output( 4483 &api_socket, 4484 "add-pmem", 4485 Some(&format!( 4486 "file={},id=test0", 4487 pmem_temp_file.as_path().to_str().unwrap() 4488 )), 4489 ); 4490 assert!(cmd_success); 4491 assert!(String::from_utf8_lossy(&cmd_output) 4492 .contains("{\"id\":\"test0\",\"bdf\":\"0000:00:06.0\"}")); 4493 4494 // Check that /dev/pmem0 exists and the block size is 128M 4495 assert_eq!( 4496 guest 4497 .ssh_command("lsblk | grep pmem0 | grep -c 128M") 4498 .unwrap() 4499 .trim() 4500 .parse::<u32>() 4501 .unwrap_or_default(), 4502 1 4503 ); 4504 4505 guest.reboot_linux(0, None); 4506 4507 // Check still there after reboot 4508 assert_eq!( 4509 guest 4510 .ssh_command("lsblk | grep pmem0 | grep -c 128M") 4511 .unwrap() 4512 .trim() 4513 .parse::<u32>() 4514 .unwrap_or_default(), 4515 1 4516 ); 4517 4518 assert!(remote_command(&api_socket, "remove-device", Some("test0"))); 4519 4520 thread::sleep(std::time::Duration::new(20, 0)); 4521 4522 // Check device has gone away 4523 assert_eq!( 4524 guest 4525 .ssh_command("lsblk | grep -c pmem0.*128M || true") 4526 .unwrap() 4527 .trim() 4528 .parse::<u32>() 4529 .unwrap_or(1), 4530 0 4531 ); 4532 4533 guest.reboot_linux(1, None); 4534 4535 // Check still absent after reboot 4536 assert_eq!( 4537 guest 4538 .ssh_command("lsblk | grep -c pmem0.*128M || true") 4539 .unwrap() 4540 .trim() 4541 .parse::<u32>() 4542 .unwrap_or(1), 4543 0 4544 ); 4545 }); 4546 4547 let _ = child.kill(); 4548 let output = child.wait_with_output().unwrap(); 4549 4550 handle_child_output(r, &output); 4551 } 4552 4553 #[test] 4554 #[cfg(target_arch = "x86_64")] 4555 fn test_net_hotplug() { 4556 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4557 let guest = Guest::new(Box::new(focal)); 4558 4559 let kernel_path = direct_kernel_boot_path(); 4560 4561 let api_socket = temp_api_path(&guest.tmp_dir); 4562 4563 // Boot without network 4564 let mut child = GuestCommand::new(&guest) 4565 .args(&["--api-socket", &api_socket]) 4566 .args(&["--cpus", "boot=1"]) 4567 .args(&["--memory", "size=512M"]) 4568 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4569 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4570 .default_disks() 4571 .capture_output() 4572 .spawn() 4573 .unwrap(); 4574 4575 thread::sleep(std::time::Duration::new(20, 0)); 4576 4577 let r = std::panic::catch_unwind(|| { 4578 // Add network 4579 let (cmd_success, cmd_output) = remote_command_w_output( 4580 &api_socket, 4581 "add-net", 4582 Some(guest.default_net_string().as_str()), 4583 ); 4584 assert!(cmd_success); 4585 assert!(String::from_utf8_lossy(&cmd_output) 4586 .contains("{\"id\":\"_net2\",\"bdf\":\"0000:00:05.0\"}")); 4587 4588 thread::sleep(std::time::Duration::new(5, 0)); 4589 4590 // 1 network interfaces + default localhost ==> 2 interfaces 4591 assert_eq!( 4592 guest 4593 .ssh_command("ip -o link | wc -l") 4594 .unwrap() 4595 .trim() 4596 .parse::<u32>() 4597 .unwrap_or_default(), 4598 2 4599 ); 4600 4601 // Remove network 4602 assert!(remote_command(&api_socket, "remove-device", Some("_net2"))); 4603 thread::sleep(std::time::Duration::new(5, 0)); 4604 4605 // Add network again 4606 let (cmd_success, cmd_output) = remote_command_w_output( 4607 &api_socket, 4608 "add-net", 4609 Some(guest.default_net_string().as_str()), 4610 ); 4611 assert!(cmd_success); 4612 assert!(String::from_utf8_lossy(&cmd_output) 4613 .contains("{\"id\":\"_net3\",\"bdf\":\"0000:00:05.0\"}")); 4614 4615 thread::sleep(std::time::Duration::new(5, 0)); 4616 4617 // 1 network interfaces + default localhost ==> 2 interfaces 4618 assert_eq!( 4619 guest 4620 .ssh_command("ip -o link | wc -l") 4621 .unwrap() 4622 .trim() 4623 .parse::<u32>() 4624 .unwrap_or_default(), 4625 2 4626 ); 4627 4628 guest.reboot_linux(0, None); 4629 4630 // Check still there after reboot 4631 // 1 network interfaces + default localhost ==> 2 interfaces 4632 assert_eq!( 4633 guest 4634 .ssh_command("ip -o link | wc -l") 4635 .unwrap() 4636 .trim() 4637 .parse::<u32>() 4638 .unwrap_or_default(), 4639 2 4640 ); 4641 }); 4642 4643 let _ = child.kill(); 4644 let output = child.wait_with_output().unwrap(); 4645 4646 handle_child_output(r, &output); 4647 } 4648 4649 #[test] 4650 fn test_initramfs() { 4651 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4652 let guest = Guest::new(Box::new(focal)); 4653 let mut workload_path = dirs::home_dir().unwrap(); 4654 workload_path.push("workloads"); 4655 4656 let mut kernels = vec![direct_kernel_boot_path()]; 4657 4658 #[cfg(target_arch = "x86_64")] 4659 { 4660 let mut pvh_kernel_path = workload_path.clone(); 4661 pvh_kernel_path.push("vmlinux"); 4662 kernels.push(pvh_kernel_path); 4663 } 4664 4665 let mut initramfs_path = workload_path; 4666 initramfs_path.push("alpine_initramfs.img"); 4667 4668 let test_string = String::from("axz34i9rylotd8n50wbv6kcj7f2qushme1pg"); 4669 let cmdline = format!("console=hvc0 quiet TEST_STRING={}", test_string); 4670 4671 kernels.iter().for_each(|k_path| { 4672 let mut child = GuestCommand::new(&guest) 4673 .args(&["--kernel", k_path.to_str().unwrap()]) 4674 .args(&["--initramfs", initramfs_path.to_str().unwrap()]) 4675 .args(&["--cmdline", &cmdline]) 4676 .capture_output() 4677 .spawn() 4678 .unwrap(); 4679 4680 thread::sleep(std::time::Duration::new(20, 0)); 4681 4682 let _ = child.kill(); 4683 let output = child.wait_with_output().unwrap(); 4684 4685 let r = std::panic::catch_unwind(|| { 4686 let s = String::from_utf8_lossy(&output.stdout); 4687 4688 assert_ne!(s.lines().position(|line| line == test_string), None); 4689 }); 4690 4691 handle_child_output(r, &output); 4692 }); 4693 } 4694 4695 // One thing to note about this test. The virtio-net device is heavily used 4696 // through each ssh command. There's no need to perform a dedicated test to 4697 // verify the migration went well for virtio-net. 4698 #[test] 4699 fn test_snapshot_restore() { 4700 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4701 let guest = Guest::new(Box::new(focal)); 4702 let kernel_path = direct_kernel_boot_path(); 4703 4704 let api_socket = temp_api_path(&guest.tmp_dir); 4705 4706 let net_id = "net123"; 4707 let net_params = format!( 4708 "id={},tap=,mac={},ip={},mask=255.255.255.0", 4709 net_id, guest.network.guest_mac, guest.network.host_ip 4710 ); 4711 4712 let cloudinit_params = format!( 4713 "path={},iommu=on", 4714 guest.disk_config.disk(DiskType::CloudInit).unwrap() 4715 ); 4716 4717 let socket = temp_vsock_path(&guest.tmp_dir); 4718 4719 let mut child = GuestCommand::new(&guest) 4720 .args(&["--api-socket", &api_socket]) 4721 .args(&["--cpus", "boot=4"]) 4722 .args(&["--memory", "size=4G"]) 4723 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4724 .args(&[ 4725 "--disk", 4726 format!( 4727 "path={}", 4728 guest.disk_config.disk(DiskType::OperatingSystem).unwrap() 4729 ) 4730 .as_str(), 4731 cloudinit_params.as_str(), 4732 ]) 4733 .args(&["--net", net_params.as_str()]) 4734 .args(&["--vsock", format!("cid=3,socket={}", socket).as_str()]) 4735 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4736 .capture_output() 4737 .spawn() 4738 .unwrap(); 4739 4740 let console_text = String::from("On a branch floating down river a cricket, singing."); 4741 let console_cmd = format!("echo {} | sudo tee /dev/hvc0", console_text); 4742 // Create the snapshot directory 4743 let snapshot_dir = temp_snapshot_dir_path(&guest.tmp_dir); 4744 4745 let r = std::panic::catch_unwind(|| { 4746 guest.wait_vm_boot(None).unwrap(); 4747 4748 // Check the number of vCPUs 4749 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 4); 4750 // Check the guest RAM 4751 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 4752 // Check block devices are readable 4753 guest 4754 .ssh_command("sudo dd if=/dev/vda of=/dev/null bs=1M iflag=direct count=1024") 4755 .unwrap(); 4756 guest 4757 .ssh_command("sudo dd if=/dev/vdb of=/dev/null bs=1M iflag=direct count=8") 4758 .unwrap(); 4759 // Check if the rng device is readable 4760 guest 4761 .ssh_command("sudo head -c 1000 /dev/hwrng > /dev/null") 4762 .unwrap(); 4763 // Check vsock 4764 guest.check_vsock(socket.as_str()); 4765 // Check if the console is usable 4766 4767 guest.ssh_command(&console_cmd).unwrap(); 4768 4769 // x86_64: We check that removing and adding back the virtio-net device 4770 // does not break the snapshot/restore support for virtio-pci. 4771 // This is an important thing to test as the hotplug will 4772 // trigger a PCI BAR reprogramming, which is a good way of 4773 // checking if the stored resources are correctly restored. 4774 // Unplug the virtio-net device 4775 // AArch64: Device hotplug is currently not supported, skipping here. 4776 #[cfg(target_arch = "x86_64")] 4777 { 4778 assert!(remote_command(&api_socket, "remove-device", Some(net_id),)); 4779 thread::sleep(std::time::Duration::new(10, 0)); 4780 4781 // Plug the virtio-net device again 4782 assert!(remote_command( 4783 &api_socket, 4784 "add-net", 4785 Some(net_params.as_str()), 4786 )); 4787 thread::sleep(std::time::Duration::new(10, 0)); 4788 } 4789 4790 // Pause the VM 4791 assert!(remote_command(&api_socket, "pause", None)); 4792 4793 // Take a snapshot from the VM 4794 assert!(remote_command( 4795 &api_socket, 4796 "snapshot", 4797 Some(format!("file://{}", snapshot_dir).as_str()), 4798 )); 4799 4800 // Wait to make sure the snapshot is completed 4801 thread::sleep(std::time::Duration::new(10, 0)); 4802 }); 4803 4804 // Shutdown the source VM and check console output 4805 let _ = child.kill(); 4806 let output = child.wait_with_output().unwrap(); 4807 handle_child_output(r, &output); 4808 4809 let r = std::panic::catch_unwind(|| { 4810 assert!(String::from_utf8_lossy(&output.stdout).contains(&console_text)); 4811 }); 4812 4813 handle_child_output(r, &output); 4814 4815 // Remove the vsock socket file. 4816 Command::new("rm") 4817 .arg("-f") 4818 .arg(socket.as_str()) 4819 .output() 4820 .unwrap(); 4821 4822 // Restore the VM from the snapshot 4823 let mut child = GuestCommand::new(&guest) 4824 .args(&["--api-socket", &api_socket]) 4825 .args(&[ 4826 "--restore", 4827 format!("source_url=file://{}", snapshot_dir).as_str(), 4828 ]) 4829 .capture_output() 4830 .spawn() 4831 .unwrap(); 4832 4833 // Wait for the VM to be restored 4834 thread::sleep(std::time::Duration::new(10, 0)); 4835 4836 let r = std::panic::catch_unwind(|| { 4837 // Resume the VM 4838 assert!(remote_command(&api_socket, "resume", None)); 4839 4840 // Perform same checks to validate VM has been properly restored 4841 assert_eq!(guest.get_cpu_count().unwrap_or_default(), 4); 4842 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 4843 guest 4844 .ssh_command("sudo dd if=/dev/vda of=/dev/null bs=1M iflag=direct count=1024") 4845 .unwrap(); 4846 guest 4847 .ssh_command("sudo dd if=/dev/vdb of=/dev/null bs=1M iflag=direct count=8") 4848 .unwrap(); 4849 guest 4850 .ssh_command("sudo head -c 1000 /dev/hwrng > /dev/null") 4851 .unwrap(); 4852 guest.check_vsock(socket.as_str()); 4853 guest.ssh_command(&console_cmd).unwrap(); 4854 // Shutdown the target VM and check console output 4855 }); 4856 let _ = child.kill(); 4857 let output = child.wait_with_output().unwrap(); 4858 handle_child_output(r, &output); 4859 4860 let r = std::panic::catch_unwind(|| { 4861 assert!(String::from_utf8_lossy(&output.stdout).contains(&console_text)); 4862 }); 4863 4864 handle_child_output(r, &output); 4865 } 4866 4867 #[test] 4868 fn test_counters() { 4869 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4870 let guest = Guest::new(Box::new(focal)); 4871 let api_socket = temp_api_path(&guest.tmp_dir); 4872 4873 let mut cmd = GuestCommand::new(&guest); 4874 cmd.args(&["--cpus", "boot=1"]) 4875 .args(&["--memory", "size=512M"]) 4876 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 4877 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4878 .default_disks() 4879 .args(&["--net", guest.default_net_string().as_str()]) 4880 .args(&["--api-socket", &api_socket]) 4881 .capture_output(); 4882 4883 let mut child = cmd.spawn().unwrap(); 4884 4885 let r = std::panic::catch_unwind(|| { 4886 guest.wait_vm_boot(None).unwrap(); 4887 4888 let orig_counters = get_counters(&api_socket); 4889 guest 4890 .ssh_command("dd if=/dev/zero of=test count=8 bs=1M") 4891 .unwrap(); 4892 4893 let new_counters = get_counters(&api_socket); 4894 4895 // Check that all the counters have increased 4896 assert!(new_counters > orig_counters); 4897 }); 4898 4899 let _ = child.kill(); 4900 let output = child.wait_with_output().unwrap(); 4901 4902 handle_child_output(r, &output); 4903 } 4904 4905 #[test] 4906 fn test_watchdog() { 4907 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 4908 let guest = Guest::new(Box::new(focal)); 4909 let api_socket = temp_api_path(&guest.tmp_dir); 4910 4911 let kernel_path = direct_kernel_boot_path(); 4912 4913 let mut cmd = GuestCommand::new(&guest); 4914 cmd.args(&["--cpus", "boot=1"]) 4915 .args(&["--memory", "size=512M"]) 4916 .args(&["--kernel", kernel_path.to_str().unwrap()]) 4917 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 4918 .default_disks() 4919 .args(&["--net", guest.default_net_string().as_str()]) 4920 .args(&["--watchdog"]) 4921 .args(&["--api-socket", &api_socket]) 4922 .capture_output(); 4923 4924 let mut child = cmd.spawn().unwrap(); 4925 4926 let r = std::panic::catch_unwind(|| { 4927 guest.wait_vm_boot(None).unwrap(); 4928 4929 // Check for PCI device 4930 assert!(guest 4931 .does_device_vendor_pair_match("0x1063", "0x1af4") 4932 .unwrap_or_default()); 4933 4934 // Enable systemd watchdog 4935 guest 4936 .ssh_command( 4937 "echo RuntimeWatchdogSec=15s | sudo tee -a /etc/systemd/system.conf", 4938 ) 4939 .unwrap(); 4940 4941 guest.ssh_command("sudo reboot").unwrap(); 4942 4943 guest.wait_vm_boot(None).unwrap(); 4944 4945 // Check that systemd has activated the watchdog 4946 assert_eq!( 4947 guest 4948 .ssh_command("sudo journalctl | grep -c -- \"Watchdog started\"") 4949 .unwrap() 4950 .trim() 4951 .parse::<u32>() 4952 .unwrap_or_default(), 4953 2 4954 ); 4955 4956 // Ensure that the current boot journal is written so reboot counts are valid 4957 guest.ssh_command("sudo journalctl --sync").unwrap(); 4958 4959 let boot_count = guest 4960 .ssh_command("sudo journalctl --list-boots | wc -l") 4961 .unwrap() 4962 .trim() 4963 .parse::<u32>() 4964 .unwrap_or_default(); 4965 assert_eq!(boot_count, 2); 4966 // Allow some normal time to elapse to check we don't get spurious reboots 4967 thread::sleep(std::time::Duration::new(40, 0)); 4968 4969 // Check no reboot 4970 let boot_count = guest 4971 .ssh_command("sudo journalctl --list-boots | wc -l") 4972 .unwrap() 4973 .trim() 4974 .parse::<u32>() 4975 .unwrap_or_default(); 4976 assert_eq!(boot_count, 2); 4977 4978 // Ensure that the current boot journal is written so reboot counts are valid 4979 guest.ssh_command("sudo journalctl --sync").unwrap(); 4980 4981 // Trigger a panic (sync first). We need to do this inside a screen with a delay so the SSH command returns. 4982 guest.ssh_command("screen -dmS reboot sh -c \"sleep 5; echo s | tee /proc/sysrq-trigger; echo c | sudo tee /proc/sysrq-trigger\"").unwrap(); 4983 4984 // Allow some time for the watchdog to trigger (max 30s) and reboot to happen 4985 guest.wait_vm_boot(Some(50)).unwrap(); 4986 4987 // Check that watchdog triggered reboot 4988 let boot_count = guest 4989 .ssh_command("sudo journalctl --list-boots | wc -l") 4990 .unwrap() 4991 .trim() 4992 .parse::<u32>() 4993 .unwrap_or_default(); 4994 assert_eq!(boot_count, 3); 4995 4996 #[cfg(target_arch = "x86_64")] 4997 { 4998 // Now pause the VM and remain offline for 30s 4999 assert!(remote_command(&api_socket, "pause", None)); 5000 thread::sleep(std::time::Duration::new(30, 0)); 5001 assert!(remote_command(&api_socket, "resume", None)); 5002 5003 // Check no reboot 5004 let boot_count = guest 5005 .ssh_command("sudo journalctl --list-boots | wc -l") 5006 .unwrap() 5007 .trim() 5008 .parse::<u32>() 5009 .unwrap_or_default(); 5010 assert_eq!(boot_count, 3); 5011 } 5012 }); 5013 5014 let _ = child.kill(); 5015 let output = child.wait_with_output().unwrap(); 5016 5017 handle_child_output(r, &output); 5018 } 5019 5020 #[test] 5021 fn test_tap_from_fd() { 5022 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5023 let guest = Guest::new(Box::new(focal)); 5024 let kernel_path = direct_kernel_boot_path(); 5025 5026 // Create a TAP interface with multi-queue enabled 5027 let num_queue_pairs: usize = 2; 5028 5029 use std::str::FromStr; 5030 let taps = net_util::open_tap( 5031 Some("chtap0"), 5032 Some(std::net::Ipv4Addr::from_str(&guest.network.host_ip).unwrap()), 5033 None, 5034 &mut None, 5035 num_queue_pairs, 5036 Some(libc::O_RDWR | libc::O_NONBLOCK), 5037 ) 5038 .unwrap(); 5039 5040 let mut child = GuestCommand::new(&guest) 5041 .args(&["--cpus", &format!("boot={}", num_queue_pairs)]) 5042 .args(&["--memory", "size=512M"]) 5043 .args(&["--kernel", kernel_path.to_str().unwrap()]) 5044 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5045 .default_disks() 5046 .args(&[ 5047 "--net", 5048 &format!( 5049 "fd={}:{},mac={},num_queues={}", 5050 taps[0].as_raw_fd(), 5051 taps[1].as_raw_fd(), 5052 guest.network.guest_mac, 5053 num_queue_pairs * 2 5054 ), 5055 ]) 5056 .capture_output() 5057 .spawn() 5058 .unwrap(); 5059 5060 let r = std::panic::catch_unwind(|| { 5061 guest.wait_vm_boot(None).unwrap(); 5062 5063 assert_eq!( 5064 guest 5065 .ssh_command("ip -o link | wc -l") 5066 .unwrap() 5067 .trim() 5068 .parse::<u32>() 5069 .unwrap_or_default(), 5070 2 5071 ); 5072 5073 guest.reboot_linux(0, None); 5074 5075 assert_eq!( 5076 guest 5077 .ssh_command("ip -o link | wc -l") 5078 .unwrap() 5079 .trim() 5080 .parse::<u32>() 5081 .unwrap_or_default(), 5082 2 5083 ); 5084 }); 5085 5086 let _ = child.kill(); 5087 let output = child.wait_with_output().unwrap(); 5088 5089 handle_child_output(r, &output); 5090 } 5091 5092 #[test] 5093 // By design, a guest VM won't be able to connect to the host 5094 // machine when using a macvtap network interface (while it can 5095 // communicate externally). As a workaround, this integration 5096 // test creates two macvtap interfaces in 'bridge' mode on the 5097 // same physical net interface, one for the guest and one for 5098 // the host. With additional setup on the IP address and the 5099 // routing table, it enables the communications between the 5100 // guest VM and the host machine. 5101 // Details: https://wiki.libvirt.org/page/TroubleshootMacvtapHostFail 5102 fn test_macvtap() { 5103 let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5104 let guest = Guest::new(Box::new(focal)); 5105 let kernel_path = direct_kernel_boot_path(); 5106 let phy_net = "eth0"; 5107 5108 // Create a macvtap interface for the guest VM to use 5109 let guest_macvtap_name = "guestmacvtap0"; 5110 std::process::Command::new("bash") 5111 .args(&[ 5112 "-c", 5113 &format!( 5114 "sudo ip link add link {} name {} type macvtap mod bridge", 5115 phy_net, guest_macvtap_name 5116 ), 5117 ]) 5118 .status() 5119 .expect("Expected 'ip link add' to work"); 5120 std::process::Command::new("bash") 5121 .args(&[ 5122 "-c", 5123 &format!( 5124 "sudo ip link set {} address {} up", 5125 guest_macvtap_name, guest.network.guest_mac 5126 ), 5127 ]) 5128 .status() 5129 .expect("Expected 'ip link set' to work"); 5130 std::process::Command::new("bash") 5131 .args(&["-c", &format!("sudo ip link show {}", guest_macvtap_name)]) 5132 .status() 5133 .expect("Expected 'ip link show' to work"); 5134 5135 let tap_index = 5136 fs::read_to_string(&format!("/sys/class/net/{}/ifindex", guest_macvtap_name)) 5137 .unwrap(); 5138 let tap_device = format!("/dev/tap{}", tap_index.trim()); 5139 5140 std::process::Command::new("bash") 5141 .args(&["-c", &format!("sudo chown $UID.$UID {}", tap_device)]) 5142 .status() 5143 .expect("Expected 'chown' to work"); 5144 5145 let cstr_tap_device = std::ffi::CString::new(tap_device).unwrap(); 5146 let tap_fd = unsafe { libc::open(cstr_tap_device.as_ptr(), libc::O_RDWR) }; 5147 assert!(tap_fd > 0); 5148 5149 // Create a macvtap on the same physical net interface for 5150 // the host machine to use 5151 let host_macvtap_name = "hostmacvtap0"; 5152 std::process::Command::new("bash") 5153 .args(&[ 5154 "-c", 5155 &format!( 5156 "sudo ip link add link {} name {} type macvtap mod bridge", 5157 phy_net, host_macvtap_name 5158 ), 5159 ]) 5160 .status() 5161 .expect("Expected 'ip link add' to work"); 5162 // Use default mask "255.255.255.0" 5163 std::process::Command::new("bash") 5164 .args(&[ 5165 "-c", 5166 &format!( 5167 "sudo ip address add {}/24 dev {}", 5168 guest.network.host_ip, host_macvtap_name 5169 ), 5170 ]) 5171 .status() 5172 .expect("Expected 'ip address add' to work"); 5173 std::process::Command::new("bash") 5174 .args(&[ 5175 "-c", 5176 &format!("sudo ip link set dev {} up", host_macvtap_name), 5177 ]) 5178 .status() 5179 .expect("Expected 'ip link set dev up' to work"); 5180 5181 let mut child = GuestCommand::new(&guest) 5182 .args(&["--cpus", "boot=1"]) 5183 .args(&["--memory", "size=512M"]) 5184 .args(&["--kernel", kernel_path.to_str().unwrap()]) 5185 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5186 .default_disks() 5187 .args(&[ 5188 "--net", 5189 &format!("fd={},mac={}", tap_fd, guest.network.guest_mac), 5190 ]) 5191 .capture_output() 5192 .spawn() 5193 .unwrap(); 5194 5195 let r = std::panic::catch_unwind(|| { 5196 guest.wait_vm_boot(None).unwrap(); 5197 5198 assert_eq!( 5199 guest 5200 .ssh_command("ip -o link | wc -l") 5201 .unwrap() 5202 .trim() 5203 .parse::<u32>() 5204 .unwrap_or_default(), 5205 2 5206 ); 5207 }); 5208 5209 let _ = child.kill(); 5210 5211 std::process::Command::new("bash") 5212 .args(&["-c", &format!("sudo ip link del {}", guest_macvtap_name)]) 5213 .status() 5214 .expect("Expected 'ip link del' to work"); 5215 std::process::Command::new("bash") 5216 .args(&["-c", &format!("sudo ip link del {}", host_macvtap_name)]) 5217 .status() 5218 .expect("Expected 'ip link del' to work"); 5219 5220 let output = child.wait_with_output().unwrap(); 5221 5222 handle_child_output(r, &output); 5223 } 5224 5225 #[test] 5226 fn test_ovs_dpdk() { 5227 // Create OVS-DPDK bridge and ports 5228 assert!(exec_host_command_status( 5229 "ovs-vsctl add-br ovsbr0 -- set bridge ovsbr0 datapath_type=netdev", 5230 ) 5231 .success()); 5232 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user1 -- set Interface vhost-user1 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient1").success()); 5233 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user2 -- set Interface vhost-user2 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient2").success()); 5234 assert!(exec_host_command_status("ip link set up dev ovsbr0").success()); 5235 assert!(exec_host_command_status("service openvswitch-switch restart").success()); 5236 5237 let focal1 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5238 let guest1 = Guest::new(Box::new(focal1)); 5239 let mut child1 = GuestCommand::new(&guest1) 5240 .args(&["--cpus", "boot=2"]) 5241 .args(&["--memory", "size=0,shared=on"]) 5242 .args(&["--memory-zone", "id=mem0,size=1G,shared=on,host_numa_node=0"]) 5243 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 5244 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5245 .default_disks() 5246 .args(&["--net", guest1.default_net_string().as_str(), "vhost_user=true,socket=/tmp/dpdkvhostclient1,num_queues=2,queue_size=256,vhost_mode=server"]) 5247 .capture_output() 5248 .spawn() 5249 .unwrap(); 5250 5251 #[cfg(target_arch = "x86_64")] 5252 let guest_net_iface = "ens5"; 5253 #[cfg(target_arch = "aarch64")] 5254 let guest_net_iface = "enp0s5"; 5255 5256 let r = std::panic::catch_unwind(|| { 5257 guest1.wait_vm_boot(None).unwrap(); 5258 5259 guest1 5260 .ssh_command(&format!( 5261 "sudo ip addr add 172.100.0.1/24 dev {}", 5262 guest_net_iface 5263 )) 5264 .unwrap(); 5265 guest1 5266 .ssh_command(&format!("sudo ip link set up dev {}", guest_net_iface)) 5267 .unwrap(); 5268 5269 let guest_ip = guest1.network.guest_ip.clone(); 5270 thread::spawn(move || { 5271 ssh_command_ip( 5272 "nc -l 12345", 5273 &guest_ip, 5274 DEFAULT_SSH_RETRIES, 5275 DEFAULT_SSH_TIMEOUT, 5276 ) 5277 .unwrap(); 5278 }); 5279 }); 5280 if r.is_err() { 5281 let _ = child1.kill(); 5282 let output = child1.wait_with_output().unwrap(); 5283 handle_child_output(r, &output); 5284 panic!("Test should already be failed/panicked"); // To explicitly mark this block never return 5285 } 5286 5287 let focal2 = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); 5288 let guest2 = Guest::new(Box::new(focal2)); 5289 let mut child2 = GuestCommand::new(&guest2) 5290 .args(&["--cpus", "boot=2"]) 5291 .args(&["--memory", "size=0,shared=on"]) 5292 .args(&["--memory-zone", "id=mem0,size=1G,shared=on,host_numa_node=0"]) 5293 .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()]) 5294 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 5295 .default_disks() 5296 .args(&["--net", guest2.default_net_string().as_str(), "vhost_user=true,socket=/tmp/dpdkvhostclient2,num_queues=2,queue_size=256,vhost_mode=server"]) 5297 .capture_output() 5298 .spawn() 5299 .unwrap(); 5300 5301 let r = std::panic::catch_unwind(|| { 5302 guest2.wait_vm_boot(None).unwrap(); 5303 5304 guest2 5305 .ssh_command(&format!( 5306 "sudo ip addr add 172.100.0.2/24 dev {}", 5307 guest_net_iface 5308 )) 5309 .unwrap(); 5310 guest2 5311 .ssh_command(&format!("sudo ip link set up dev {}", guest_net_iface)) 5312 .unwrap(); 5313 5314 // Check the connection works properly between the two VMs 5315 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 5316 5317 // Remove one of the two ports from the OVS bridge 5318 assert!(exec_host_command_status("ovs-vsctl del-port vhost-user1").success()); 5319 5320 // Spawn a new netcat listener in the first VM 5321 let guest_ip = guest1.network.guest_ip.clone(); 5322 thread::spawn(move || { 5323 ssh_command_ip( 5324 "nc -l 12345", 5325 &guest_ip, 5326 DEFAULT_SSH_RETRIES, 5327 DEFAULT_SSH_TIMEOUT, 5328 ) 5329 .unwrap(); 5330 }); 5331 5332 // Check the connection fails this time 5333 assert!(guest2.ssh_command("nc -vz 172.100.0.1 12345").is_err()); 5334 5335 // Add the OVS port back 5336 assert!(exec_host_command_status("ovs-vsctl add-port ovsbr0 vhost-user1 -- set Interface vhost-user1 type=dpdkvhostuserclient options:vhost-server-path=/tmp/dpdkvhostclient1").success()); 5337 5338 // And finally check the connection is functional again 5339 guest2.ssh_command("nc -vz 172.100.0.1 12345").unwrap(); 5340 }); 5341 5342 let _ = child1.kill(); 5343 let _ = child2.kill(); 5344 5345 let output = child1.wait_with_output().unwrap(); 5346 child2.wait().unwrap(); 5347 5348 handle_child_output(r, &output); 5349 } 5350 } 5351 5352 mod sequential { 5353 use crate::tests::*; 5354 5355 #[test] 5356 fn test_memory_mergeable_on() { 5357 test_memory_mergeable(true) 5358 } 5359 } 5360 5361 #[cfg(target_arch = "x86_64")] 5362 mod windows { 5363 use crate::tests::*; 5364 5365 lazy_static! { 5366 static ref NEXT_DISK_ID: Mutex<u8> = Mutex::new(1); 5367 } 5368 5369 struct WindowsGuest { 5370 guest: Guest, 5371 auth: PasswordAuth, 5372 } 5373 5374 trait FsType { 5375 const FS_FAT: u8; 5376 const FS_NTFS: u8; 5377 } 5378 impl FsType for WindowsGuest { 5379 const FS_FAT: u8 = 0; 5380 const FS_NTFS: u8 = 1; 5381 } 5382 5383 impl WindowsGuest { 5384 fn new() -> Self { 5385 let disk = WindowsDiskConfig::new(WINDOWS_IMAGE_NAME.to_string()); 5386 let guest = Guest::new(Box::new(disk)); 5387 let auth = PasswordAuth { 5388 username: String::from("administrator"), 5389 password: String::from("Admin123"), 5390 }; 5391 5392 WindowsGuest { guest, auth } 5393 } 5394 5395 fn guest(&self) -> &Guest { 5396 &self.guest 5397 } 5398 5399 fn ssh_cmd(&self, cmd: &str) -> String { 5400 ssh_command_ip_with_auth( 5401 cmd, 5402 &self.auth, 5403 &self.guest.network.guest_ip, 5404 DEFAULT_SSH_RETRIES, 5405 DEFAULT_SSH_TIMEOUT, 5406 ) 5407 .unwrap() 5408 } 5409 5410 fn cpu_count(&self) -> u8 { 5411 self.ssh_cmd("powershell -Command \"(Get-CimInstance win32_computersystem).NumberOfLogicalProcessors\"") 5412 .trim() 5413 .parse::<u8>() 5414 .unwrap_or(0) 5415 } 5416 5417 fn ram_size(&self) -> usize { 5418 self.ssh_cmd("powershell -Command \"(Get-CimInstance win32_computersystem).TotalPhysicalMemory\"") 5419 .trim() 5420 .parse::<usize>() 5421 .unwrap_or(0) 5422 } 5423 5424 fn netdev_count(&self) -> u8 { 5425 self.ssh_cmd("powershell -Command \"netsh int ipv4 show interfaces | Select-String ethernet | Measure-Object -Line | Format-Table -HideTableHeaders\"") 5426 .trim() 5427 .parse::<u8>() 5428 .unwrap_or(0) 5429 } 5430 5431 fn disk_count(&self) -> u8 { 5432 self.ssh_cmd("powershell -Command \"Get-Disk | Measure-Object -Line | Format-Table -HideTableHeaders\"") 5433 .trim() 5434 .parse::<u8>() 5435 .unwrap_or(0) 5436 } 5437 5438 fn reboot(&self) { 5439 let _ = self.ssh_cmd("shutdown /r /t 0"); 5440 } 5441 5442 fn shutdown(&self) { 5443 let _ = self.ssh_cmd("shutdown /s /t 0"); 5444 } 5445 5446 fn run_dnsmasq(&self) -> std::process::Child { 5447 let listen_address = format!("--listen-address={}", self.guest.network.host_ip); 5448 let dhcp_host = format!( 5449 "--dhcp-host={},{}", 5450 self.guest.network.guest_mac, self.guest.network.guest_ip 5451 ); 5452 let dhcp_range = format!( 5453 "--dhcp-range=eth,{},{}", 5454 self.guest.network.guest_ip, self.guest.network.guest_ip 5455 ); 5456 5457 Command::new("dnsmasq") 5458 .arg("--no-daemon") 5459 .arg("--log-queries") 5460 .arg(listen_address.as_str()) 5461 .arg("--except-interface=lo") 5462 .arg("--bind-dynamic") // Allow listening to host_ip while the interface is not ready yet. 5463 .arg("--conf-file=/dev/null") 5464 .arg(dhcp_host.as_str()) 5465 .arg(dhcp_range.as_str()) 5466 .spawn() 5467 .unwrap() 5468 } 5469 5470 // TODO Cleanup image file explicitly after test, if there's some space issues. 5471 fn disk_new(&self, fs: u8, sz: usize) -> String { 5472 let mut guard = NEXT_DISK_ID.lock().unwrap(); 5473 let id = *guard; 5474 *guard = id + 1; 5475 5476 let img = PathBuf::from(format!("/tmp/test-hotplug-{}.raw", id)); 5477 let _ = fs::remove_file(&img); 5478 5479 // Create an image file 5480 let out = Command::new("qemu-img") 5481 .args(&[ 5482 "create", 5483 "-f", 5484 "raw", 5485 img.to_str().unwrap(), 5486 format!("{}m", sz).as_str(), 5487 ]) 5488 .output() 5489 .expect("qemu-img command failed") 5490 .stdout; 5491 println!("{:?}", out); 5492 5493 // Associate image to a loop device 5494 let out = Command::new("losetup") 5495 .args(&["--show", "-f", img.to_str().unwrap()]) 5496 .output() 5497 .expect("failed to create loop device") 5498 .stdout; 5499 let _tmp = String::from_utf8_lossy(&out); 5500 let loop_dev = _tmp.trim(); 5501 println!("{:?}", out); 5502 5503 // Create a partition table 5504 // echo 'type=7' | sudo sfdisk "${LOOP}" 5505 let mut child = Command::new("sfdisk") 5506 .args(&[loop_dev]) 5507 .stdin(Stdio::piped()) 5508 .spawn() 5509 .unwrap(); 5510 let stdin = child.stdin.as_mut().expect("failed to open stdin"); 5511 let _ = stdin 5512 .write_all("type=7".as_bytes()) 5513 .expect("failed to write stdin"); 5514 let out = child.wait_with_output().expect("sfdisk failed").stdout; 5515 println!("{:?}", out); 5516 5517 // Disengage the loop device 5518 let out = Command::new("losetup") 5519 .args(&["-d", loop_dev]) 5520 .output() 5521 .expect("loop device not found") 5522 .stdout; 5523 println!("{:?}", out); 5524 5525 // Re-associate loop device pointing to the partition only 5526 let out = Command::new("losetup") 5527 .args(&[ 5528 "--show", 5529 "--offset", 5530 (512 * 2048).to_string().as_str(), 5531 "-f", 5532 img.to_str().unwrap(), 5533 ]) 5534 .output() 5535 .expect("failed to create loop device") 5536 .stdout; 5537 let _tmp = String::from_utf8_lossy(&out); 5538 let loop_dev = _tmp.trim(); 5539 println!("{:?}", out); 5540 5541 // Create filesystem. 5542 let fs_cmd = match fs { 5543 WindowsGuest::FS_FAT => "mkfs.msdos", 5544 WindowsGuest::FS_NTFS => "mkfs.ntfs", 5545 _ => panic!("Unknown filesystem type '{}'", fs), 5546 }; 5547 let out = Command::new(fs_cmd) 5548 .args(&[&loop_dev]) 5549 .output() 5550 .unwrap_or_else(|_| panic!("{} failed", fs_cmd)) 5551 .stdout; 5552 println!("{:?}", out); 5553 5554 // Disengage the loop device 5555 let out = Command::new("losetup") 5556 .args(&["-d", loop_dev]) 5557 .output() 5558 .unwrap_or_else(|_| panic!("loop device '{}' not found", loop_dev)) 5559 .stdout; 5560 println!("{:?}", out); 5561 5562 img.to_str().unwrap().to_string() 5563 } 5564 5565 fn disks_set_rw(&self) { 5566 let _ = self.ssh_cmd("powershell -Command \"Get-Disk | Where-Object IsOffline -eq $True | Set-Disk -IsReadOnly $False\""); 5567 } 5568 5569 fn disks_online(&self) { 5570 let _ = self.ssh_cmd("powershell -Command \"Get-Disk | Where-Object IsOffline -eq $True | Set-Disk -IsOffline $False\""); 5571 } 5572 5573 fn disk_file_put(&self, fname: &str, data: &str) { 5574 let _ = self.ssh_cmd(&format!( 5575 "powershell -Command \"'{}' | Set-Content -Path {}\"", 5576 data, fname 5577 )); 5578 } 5579 5580 fn disk_file_read(&self, fname: &str) -> String { 5581 self.ssh_cmd(&format!( 5582 "powershell -Command \"Get-Content -Path {}\"", 5583 fname 5584 )) 5585 } 5586 5587 fn wait_for_boot(&self) -> bool { 5588 let cmd = "dir /b c:\\ | find \"Windows\""; 5589 let tmo_max = 180; 5590 // The timeout increase by n*1+n*2+n*3+..., therefore the initial 5591 // interval must be small. 5592 let tmo_int = 2; 5593 let out = ssh_command_ip_with_auth( 5594 cmd, 5595 &self.auth, 5596 &self.guest.network.guest_ip, 5597 { 5598 let mut ret = 1; 5599 let mut tmo_acc = 0; 5600 loop { 5601 tmo_acc += tmo_int * ret; 5602 if tmo_acc >= tmo_max { 5603 break; 5604 } 5605 ret += 1; 5606 } 5607 ret 5608 }, 5609 tmo_int, 5610 ) 5611 .unwrap(); 5612 5613 if "Windows" == out.trim() { 5614 return true; 5615 } 5616 5617 false 5618 } 5619 } 5620 5621 fn vcpu_threads_count(pid: u32) -> u8 { 5622 // ps -T -p 12345 | grep vcpu | wc -l 5623 let out = Command::new("ps") 5624 .args(&["-T", "-p", format!("{}", pid).as_str()]) 5625 .output() 5626 .expect("ps command failed") 5627 .stdout; 5628 return String::from_utf8_lossy(&out).matches("vcpu").count() as u8; 5629 } 5630 5631 fn netdev_ctrl_threads_count(pid: u32) -> u8 { 5632 // ps -T -p 12345 | grep "_net[0-9]*_ctrl" | wc -l 5633 let out = Command::new("ps") 5634 .args(&["-T", "-p", format!("{}", pid).as_str()]) 5635 .output() 5636 .expect("ps command failed") 5637 .stdout; 5638 let mut n = 0; 5639 String::from_utf8_lossy(&out) 5640 .split_whitespace() 5641 .for_each(|s| n += (s.starts_with("_net") && s.ends_with("_ctrl")) as u8); // _net1_ctrl 5642 n 5643 } 5644 5645 fn disk_ctrl_threads_count(pid: u32) -> u8 { 5646 // ps -T -p 15782 | grep "_disk[0-9]*_q0" | wc -l 5647 let out = Command::new("ps") 5648 .args(&["-T", "-p", format!("{}", pid).as_str()]) 5649 .output() 5650 .expect("ps command failed") 5651 .stdout; 5652 let mut n = 0; 5653 String::from_utf8_lossy(&out) 5654 .split_whitespace() 5655 .for_each(|s| n += (s.starts_with("_disk") && s.ends_with("_q0")) as u8); // _disk0_q0, don't care about multiple queues as they're related to the same hdd 5656 n 5657 } 5658 5659 #[test] 5660 fn test_windows_guest() { 5661 let windows_guest = WindowsGuest::new(); 5662 5663 let mut ovmf_path = dirs::home_dir().unwrap(); 5664 ovmf_path.push("workloads"); 5665 ovmf_path.push(OVMF_NAME); 5666 5667 let mut child = GuestCommand::new(windows_guest.guest()) 5668 .args(&["--cpus", "boot=2,kvm_hyperv=on"]) 5669 .args(&["--memory", "size=4G"]) 5670 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 5671 .args(&["--serial", "tty"]) 5672 .args(&["--console", "off"]) 5673 .default_disks() 5674 .default_net() 5675 .capture_output() 5676 .spawn() 5677 .unwrap(); 5678 5679 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 5680 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 5681 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 5682 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 5683 5684 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 5685 5686 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 5687 5688 let r = std::panic::catch_unwind(|| { 5689 // Wait to make sure Windows boots up 5690 assert!(windows_guest.wait_for_boot()); 5691 5692 windows_guest.shutdown(); 5693 }); 5694 5695 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 5696 let _ = child.kill(); 5697 let output = child.wait_with_output().unwrap(); 5698 5699 let _ = child_dnsmasq.kill(); 5700 let _ = child_dnsmasq.wait(); 5701 5702 handle_child_output(r, &output); 5703 } 5704 5705 #[test] 5706 fn test_windows_guest_multiple_queues() { 5707 let windows_guest = WindowsGuest::new(); 5708 5709 let mut ovmf_path = dirs::home_dir().unwrap(); 5710 ovmf_path.push("workloads"); 5711 ovmf_path.push(OVMF_NAME); 5712 5713 let mut child = GuestCommand::new(windows_guest.guest()) 5714 .args(&["--cpus", "boot=4,kvm_hyperv=on"]) 5715 .args(&["--memory", "size=4G"]) 5716 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 5717 .args(&["--serial", "tty"]) 5718 .args(&["--console", "off"]) 5719 .args(&[ 5720 "--disk", 5721 format!( 5722 "path={},num_queues=4", 5723 windows_guest 5724 .guest() 5725 .disk_config 5726 .disk(DiskType::OperatingSystem) 5727 .unwrap() 5728 ) 5729 .as_str(), 5730 ]) 5731 .args(&[ 5732 "--net", 5733 format!( 5734 "tap=,mac={},ip={},mask=255.255.255.0,num_queues=8", 5735 windows_guest.guest().network.guest_mac, 5736 windows_guest.guest().network.host_ip 5737 ) 5738 .as_str(), 5739 ]) 5740 .capture_output() 5741 .spawn() 5742 .unwrap(); 5743 5744 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 5745 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 5746 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 5747 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 5748 5749 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 5750 5751 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 5752 5753 let r = std::panic::catch_unwind(|| { 5754 // Wait to make sure Windows boots up 5755 assert!(windows_guest.wait_for_boot()); 5756 5757 windows_guest.shutdown(); 5758 }); 5759 5760 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 5761 let _ = child.kill(); 5762 let output = child.wait_with_output().unwrap(); 5763 5764 let _ = child_dnsmasq.kill(); 5765 let _ = child_dnsmasq.wait(); 5766 5767 handle_child_output(r, &output); 5768 } 5769 5770 #[test] 5771 #[cfg(not(feature = "mshv"))] 5772 fn test_windows_guest_snapshot_restore() { 5773 let windows_guest = WindowsGuest::new(); 5774 5775 let mut ovmf_path = dirs::home_dir().unwrap(); 5776 ovmf_path.push("workloads"); 5777 ovmf_path.push(OVMF_NAME); 5778 5779 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 5780 let api_socket = temp_api_path(&tmp_dir); 5781 5782 let mut child = GuestCommand::new(windows_guest.guest()) 5783 .args(&["--api-socket", &api_socket]) 5784 .args(&["--cpus", "boot=2,kvm_hyperv=on"]) 5785 .args(&["--memory", "size=4G"]) 5786 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 5787 .args(&["--serial", "tty"]) 5788 .args(&["--console", "off"]) 5789 .default_disks() 5790 .default_net() 5791 .capture_output() 5792 .spawn() 5793 .unwrap(); 5794 5795 let fd = child.stdout.as_ref().unwrap().as_raw_fd(); 5796 let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 5797 let fd = child.stderr.as_ref().unwrap().as_raw_fd(); 5798 let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; 5799 5800 assert!(pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE); 5801 5802 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 5803 5804 // Wait to make sure Windows boots up 5805 assert!(windows_guest.wait_for_boot()); 5806 5807 let snapshot_dir = temp_snapshot_dir_path(&tmp_dir); 5808 5809 // Pause the VM 5810 assert!(remote_command(&api_socket, "pause", None)); 5811 5812 // Take a snapshot from the VM 5813 assert!(remote_command( 5814 &api_socket, 5815 "snapshot", 5816 Some(format!("file://{}", snapshot_dir).as_str()), 5817 )); 5818 5819 // Wait to make sure the snapshot is completed 5820 thread::sleep(std::time::Duration::new(30, 0)); 5821 5822 let _ = child.kill(); 5823 child.wait().unwrap(); 5824 5825 // Restore the VM from the snapshot 5826 let mut child = GuestCommand::new(windows_guest.guest()) 5827 .args(&["--api-socket", &api_socket]) 5828 .args(&[ 5829 "--restore", 5830 format!("source_url=file://{}", snapshot_dir).as_str(), 5831 ]) 5832 .capture_output() 5833 .spawn() 5834 .unwrap(); 5835 5836 // Wait for the VM to be restored 5837 thread::sleep(std::time::Duration::new(20, 0)); 5838 5839 let r = std::panic::catch_unwind(|| { 5840 // Resume the VM 5841 assert!(remote_command(&api_socket, "resume", None)); 5842 5843 windows_guest.shutdown(); 5844 }); 5845 5846 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 5847 let _ = child.kill(); 5848 let output = child.wait_with_output().unwrap(); 5849 5850 let _ = child_dnsmasq.kill(); 5851 let _ = child_dnsmasq.wait(); 5852 5853 handle_child_output(r, &output); 5854 } 5855 5856 #[test] 5857 #[cfg(not(feature = "mshv"))] 5858 fn test_windows_guest_cpu_hotplug() { 5859 let windows_guest = WindowsGuest::new(); 5860 5861 let mut ovmf_path = dirs::home_dir().unwrap(); 5862 ovmf_path.push("workloads"); 5863 ovmf_path.push(OVMF_NAME); 5864 5865 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 5866 let api_socket = temp_api_path(&tmp_dir); 5867 5868 let mut child = GuestCommand::new(windows_guest.guest()) 5869 .args(&["--api-socket", &api_socket]) 5870 .args(&["--cpus", "boot=2,max=8,kvm_hyperv=on"]) 5871 .args(&["--memory", "size=4G"]) 5872 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 5873 .args(&["--serial", "tty"]) 5874 .args(&["--console", "off"]) 5875 .default_disks() 5876 .default_net() 5877 .capture_output() 5878 .spawn() 5879 .unwrap(); 5880 5881 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 5882 5883 let r = std::panic::catch_unwind(|| { 5884 // Wait to make sure Windows boots up 5885 assert!(windows_guest.wait_for_boot()); 5886 5887 let vcpu_num = 2; 5888 // Check the initial number of CPUs the guest sees 5889 assert_eq!(windows_guest.cpu_count(), vcpu_num); 5890 // Check the initial number of vcpu threads in the CH process 5891 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 5892 5893 let vcpu_num = 6; 5894 // Hotplug some CPUs 5895 resize_command(&api_socket, Some(vcpu_num), None, None); 5896 // Wait to make sure CPUs are added 5897 thread::sleep(std::time::Duration::new(10, 0)); 5898 // Check the guest sees the correct number 5899 assert_eq!(windows_guest.cpu_count(), vcpu_num); 5900 // Check the CH process has the correct number of vcpu threads 5901 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 5902 5903 let vcpu_num = 4; 5904 // Remove some CPUs. Note that Windows doesn't support hot-remove. 5905 resize_command(&api_socket, Some(vcpu_num), None, None); 5906 // Wait to make sure CPUs are removed 5907 thread::sleep(std::time::Duration::new(10, 0)); 5908 // Reboot to let Windows catch up 5909 windows_guest.reboot(); 5910 // Wait to make sure Windows completely rebooted 5911 thread::sleep(std::time::Duration::new(60, 0)); 5912 // Check the guest sees the correct number 5913 assert_eq!(windows_guest.cpu_count(), vcpu_num); 5914 // Check the CH process has the correct number of vcpu threads 5915 assert_eq!(vcpu_threads_count(child.id()), vcpu_num); 5916 5917 windows_guest.shutdown(); 5918 }); 5919 5920 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 5921 let _ = child.kill(); 5922 let output = child.wait_with_output().unwrap(); 5923 5924 let _ = child_dnsmasq.kill(); 5925 let _ = child_dnsmasq.wait(); 5926 5927 handle_child_output(r, &output); 5928 } 5929 5930 #[test] 5931 #[cfg(not(feature = "mshv"))] 5932 fn test_windows_guest_ram_hotplug() { 5933 let windows_guest = WindowsGuest::new(); 5934 5935 let mut ovmf_path = dirs::home_dir().unwrap(); 5936 ovmf_path.push("workloads"); 5937 ovmf_path.push(OVMF_NAME); 5938 5939 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 5940 let api_socket = temp_api_path(&tmp_dir); 5941 5942 let mut child = GuestCommand::new(windows_guest.guest()) 5943 .args(&["--api-socket", &api_socket]) 5944 .args(&["--cpus", "boot=2,kvm_hyperv=on"]) 5945 .args(&["--memory", "size=2G,hotplug_size=5G"]) 5946 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 5947 .args(&["--serial", "tty"]) 5948 .args(&["--console", "off"]) 5949 .default_disks() 5950 .default_net() 5951 .capture_output() 5952 .spawn() 5953 .unwrap(); 5954 5955 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 5956 5957 let r = std::panic::catch_unwind(|| { 5958 // Wait to make sure Windows boots up 5959 assert!(windows_guest.wait_for_boot()); 5960 5961 let ram_size = 2 * 1024 * 1024 * 1024; 5962 // Check the initial number of RAM the guest sees 5963 let current_ram_size = windows_guest.ram_size(); 5964 // This size seems to be reserved by the system and thus the 5965 // reported amount differs by this constant value. 5966 let reserved_ram_size = ram_size - current_ram_size; 5967 // Verify that there's not more than 4mb constant diff wasted 5968 // by the reserved ram. 5969 assert!(reserved_ram_size < 4 * 1024 * 1024); 5970 5971 let ram_size = 4 * 1024 * 1024 * 1024; 5972 // Hotplug some RAM 5973 resize_command(&api_socket, None, Some(ram_size), None); 5974 // Wait to make sure RAM has been added 5975 thread::sleep(std::time::Duration::new(10, 0)); 5976 // Check the guest sees the correct number 5977 assert_eq!(windows_guest.ram_size(), ram_size - reserved_ram_size); 5978 5979 let ram_size = 3 * 1024 * 1024 * 1024; 5980 // Unplug some RAM. Note that hot-remove most likely won't work. 5981 resize_command(&api_socket, None, Some(ram_size), None); 5982 // Wait to make sure RAM has been added 5983 thread::sleep(std::time::Duration::new(10, 0)); 5984 // Reboot to let Windows catch up 5985 windows_guest.reboot(); 5986 // Wait to make sure guest completely rebooted 5987 thread::sleep(std::time::Duration::new(60, 0)); 5988 // Check the guest sees the correct number 5989 assert_eq!(windows_guest.ram_size(), ram_size - reserved_ram_size); 5990 5991 windows_guest.shutdown(); 5992 }); 5993 5994 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 5995 let _ = child.kill(); 5996 let output = child.wait_with_output().unwrap(); 5997 5998 let _ = child_dnsmasq.kill(); 5999 let _ = child_dnsmasq.wait(); 6000 6001 handle_child_output(r, &output); 6002 } 6003 6004 #[test] 6005 #[cfg(not(feature = "mshv"))] 6006 fn test_windows_guest_netdev_hotplug() { 6007 let windows_guest = WindowsGuest::new(); 6008 6009 let mut ovmf_path = dirs::home_dir().unwrap(); 6010 ovmf_path.push("workloads"); 6011 ovmf_path.push(OVMF_NAME); 6012 6013 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 6014 let api_socket = temp_api_path(&tmp_dir); 6015 6016 let mut child = GuestCommand::new(windows_guest.guest()) 6017 .args(&["--api-socket", &api_socket]) 6018 .args(&["--cpus", "boot=2,kvm_hyperv=on"]) 6019 .args(&["--memory", "size=4G"]) 6020 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 6021 .args(&["--serial", "tty"]) 6022 .args(&["--console", "off"]) 6023 .default_disks() 6024 .default_net() 6025 .capture_output() 6026 .spawn() 6027 .unwrap(); 6028 6029 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 6030 6031 let r = std::panic::catch_unwind(|| { 6032 // Wait to make sure Windows boots up 6033 assert!(windows_guest.wait_for_boot()); 6034 6035 // Initially present network device 6036 let netdev_num = 1; 6037 assert_eq!(windows_guest.netdev_count(), netdev_num); 6038 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 6039 6040 // Hotplug network device 6041 let (cmd_success, cmd_output) = remote_command_w_output( 6042 &api_socket, 6043 "add-net", 6044 Some(windows_guest.guest().default_net_string().as_str()), 6045 ); 6046 assert!(cmd_success); 6047 assert!(String::from_utf8_lossy(&cmd_output).contains("\"id\":\"_net2\"")); 6048 thread::sleep(std::time::Duration::new(5, 0)); 6049 // Verify the device is on the system 6050 let netdev_num = 2; 6051 assert_eq!(windows_guest.netdev_count(), netdev_num); 6052 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 6053 6054 // Remove network device 6055 let cmd_success = remote_command(&api_socket, "remove-device", Some("_net2")); 6056 assert!(cmd_success); 6057 thread::sleep(std::time::Duration::new(5, 0)); 6058 // Verify the device has been removed 6059 let netdev_num = 1; 6060 assert_eq!(windows_guest.netdev_count(), netdev_num); 6061 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 6062 6063 windows_guest.shutdown(); 6064 }); 6065 6066 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 6067 let _ = child.kill(); 6068 let output = child.wait_with_output().unwrap(); 6069 6070 let _ = child_dnsmasq.kill(); 6071 let _ = child_dnsmasq.wait(); 6072 6073 handle_child_output(r, &output); 6074 } 6075 6076 #[test] 6077 #[cfg(not(feature = "mshv"))] 6078 fn test_windows_guest_disk_hotplug() { 6079 let windows_guest = WindowsGuest::new(); 6080 6081 let mut ovmf_path = dirs::home_dir().unwrap(); 6082 ovmf_path.push("workloads"); 6083 ovmf_path.push(OVMF_NAME); 6084 6085 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 6086 let api_socket = temp_api_path(&tmp_dir); 6087 6088 let mut child = GuestCommand::new(windows_guest.guest()) 6089 .args(&["--api-socket", &api_socket]) 6090 .args(&["--cpus", "boot=2,kvm_hyperv=on"]) 6091 .args(&["--memory", "size=4G"]) 6092 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 6093 .args(&["--serial", "tty"]) 6094 .args(&["--console", "off"]) 6095 .default_disks() 6096 .default_net() 6097 .capture_output() 6098 .spawn() 6099 .unwrap(); 6100 6101 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 6102 6103 let disk = windows_guest.disk_new(WindowsGuest::FS_FAT, 100); 6104 6105 let r = std::panic::catch_unwind(|| { 6106 // Wait to make sure Windows boots up 6107 assert!(windows_guest.wait_for_boot()); 6108 6109 // Initially present disk device 6110 let disk_num = 1; 6111 assert_eq!(windows_guest.disk_count(), disk_num); 6112 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 6113 6114 // Hotplug disk device 6115 let (cmd_success, cmd_output) = remote_command_w_output( 6116 &api_socket, 6117 "add-disk", 6118 Some(format!("path={},readonly=off", disk).as_str()), 6119 ); 6120 assert!(cmd_success); 6121 assert!(String::from_utf8_lossy(&cmd_output).contains("\"id\":\"_disk2\"")); 6122 thread::sleep(std::time::Duration::new(5, 0)); 6123 // Online disk device 6124 windows_guest.disks_set_rw(); 6125 windows_guest.disks_online(); 6126 // Verify the device is on the system 6127 let disk_num = 2; 6128 assert_eq!(windows_guest.disk_count(), disk_num); 6129 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 6130 6131 let data = "hello"; 6132 let fname = "d:\\world"; 6133 windows_guest.disk_file_put(fname, data); 6134 6135 // Unmount disk device 6136 let cmd_success = remote_command(&api_socket, "remove-device", Some("_disk2")); 6137 assert!(cmd_success); 6138 thread::sleep(std::time::Duration::new(5, 0)); 6139 // Verify the device has been removed 6140 let disk_num = 1; 6141 assert_eq!(windows_guest.disk_count(), disk_num); 6142 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 6143 6144 // Remount and check the file exists with the expected contents 6145 let (cmd_success, _cmd_output) = remote_command_w_output( 6146 &api_socket, 6147 "add-disk", 6148 Some(format!("path={},readonly=off", disk).as_str()), 6149 ); 6150 assert!(cmd_success); 6151 thread::sleep(std::time::Duration::new(5, 0)); 6152 let out = windows_guest.disk_file_read(fname); 6153 assert_eq!(data, out.trim()); 6154 6155 // Intentionally no unmount, it'll happen at shutdown. 6156 6157 windows_guest.shutdown(); 6158 }); 6159 6160 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 6161 let _ = child.kill(); 6162 let output = child.wait_with_output().unwrap(); 6163 6164 let _ = child_dnsmasq.kill(); 6165 let _ = child_dnsmasq.wait(); 6166 6167 handle_child_output(r, &output); 6168 } 6169 6170 #[test] 6171 #[cfg(not(feature = "mshv"))] 6172 fn test_windows_guest_disk_hotplug_multi() { 6173 let windows_guest = WindowsGuest::new(); 6174 6175 let mut ovmf_path = dirs::home_dir().unwrap(); 6176 ovmf_path.push("workloads"); 6177 ovmf_path.push(OVMF_NAME); 6178 6179 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 6180 let api_socket = temp_api_path(&tmp_dir); 6181 6182 let mut child = GuestCommand::new(windows_guest.guest()) 6183 .args(&["--api-socket", &api_socket]) 6184 .args(&["--cpus", "boot=2,kvm_hyperv=on"]) 6185 .args(&["--memory", "size=2G"]) 6186 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 6187 .args(&["--serial", "tty"]) 6188 .args(&["--console", "off"]) 6189 .default_disks() 6190 .default_net() 6191 .capture_output() 6192 .spawn() 6193 .unwrap(); 6194 6195 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 6196 6197 // Predefined data to used at various test stages 6198 let disk_test_data: [[String; 4]; 2] = [ 6199 [ 6200 "_disk2".to_string(), 6201 windows_guest.disk_new(WindowsGuest::FS_FAT, 123), 6202 "d:\\world".to_string(), 6203 "hello".to_string(), 6204 ], 6205 [ 6206 "_disk3".to_string(), 6207 windows_guest.disk_new(WindowsGuest::FS_NTFS, 333), 6208 "e:\\hello".to_string(), 6209 "world".to_string(), 6210 ], 6211 ]; 6212 6213 let r = std::panic::catch_unwind(|| { 6214 // Wait to make sure Windows boots up 6215 assert!(windows_guest.wait_for_boot()); 6216 6217 // Initially present disk device 6218 let disk_num = 1; 6219 assert_eq!(windows_guest.disk_count(), disk_num); 6220 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 6221 6222 for it in &disk_test_data { 6223 let disk_id = it[0].as_str(); 6224 let disk = it[1].as_str(); 6225 // Hotplug disk device 6226 let (cmd_success, cmd_output) = remote_command_w_output( 6227 &api_socket, 6228 "add-disk", 6229 Some(format!("path={},readonly=off", disk).as_str()), 6230 ); 6231 assert!(cmd_success); 6232 assert!(String::from_utf8_lossy(&cmd_output) 6233 .contains(format!("\"id\":\"{}\"", disk_id).as_str())); 6234 thread::sleep(std::time::Duration::new(5, 0)); 6235 // Online disk devices 6236 windows_guest.disks_set_rw(); 6237 windows_guest.disks_online(); 6238 } 6239 // Verify the devices are on the system 6240 let disk_num = (disk_test_data.len() + 1) as u8; 6241 assert_eq!(windows_guest.disk_count(), disk_num); 6242 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 6243 6244 // Put test data 6245 for it in &disk_test_data { 6246 let fname = it[2].as_str(); 6247 let data = it[3].as_str(); 6248 windows_guest.disk_file_put(fname, data); 6249 } 6250 6251 // Unmount disk devices 6252 for it in &disk_test_data { 6253 let disk_id = it[0].as_str(); 6254 let cmd_success = remote_command(&api_socket, "remove-device", Some(disk_id)); 6255 assert!(cmd_success); 6256 thread::sleep(std::time::Duration::new(5, 0)); 6257 } 6258 6259 // Verify the devices have been removed 6260 let disk_num = 1; 6261 assert_eq!(windows_guest.disk_count(), disk_num); 6262 assert_eq!(disk_ctrl_threads_count(child.id()), disk_num); 6263 6264 // Remount 6265 for it in &disk_test_data { 6266 let disk = it[1].as_str(); 6267 let (cmd_success, _cmd_output) = remote_command_w_output( 6268 &api_socket, 6269 "add-disk", 6270 Some(format!("path={},readonly=off", disk).as_str()), 6271 ); 6272 assert!(cmd_success); 6273 thread::sleep(std::time::Duration::new(5, 0)); 6274 } 6275 6276 // Check the files exists with the expected contents 6277 for it in &disk_test_data { 6278 let fname = it[2].as_str(); 6279 let data = it[3].as_str(); 6280 let out = windows_guest.disk_file_read(fname); 6281 assert_eq!(data, out.trim()); 6282 } 6283 6284 // Intentionally no unmount, it'll happen at shutdown. 6285 6286 windows_guest.shutdown(); 6287 }); 6288 6289 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 6290 let _ = child.kill(); 6291 let output = child.wait_with_output().unwrap(); 6292 6293 let _ = child_dnsmasq.kill(); 6294 let _ = child_dnsmasq.wait(); 6295 6296 handle_child_output(r, &output); 6297 } 6298 6299 #[test] 6300 #[cfg(not(feature = "mshv"))] 6301 fn test_windows_guest_netdev_multi() { 6302 let windows_guest = WindowsGuest::new(); 6303 6304 let mut ovmf_path = dirs::home_dir().unwrap(); 6305 ovmf_path.push("workloads"); 6306 ovmf_path.push(OVMF_NAME); 6307 6308 let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); 6309 let api_socket = temp_api_path(&tmp_dir); 6310 6311 let mut child = GuestCommand::new(windows_guest.guest()) 6312 .args(&["--api-socket", &api_socket]) 6313 .args(&["--cpus", "boot=2,kvm_hyperv=on"]) 6314 .args(&["--memory", "size=4G"]) 6315 .args(&["--kernel", ovmf_path.to_str().unwrap()]) 6316 .args(&["--serial", "tty"]) 6317 .args(&["--console", "off"]) 6318 .default_disks() 6319 // The multi net dev config is borrowed from test_multiple_network_interfaces 6320 .args(&[ 6321 "--net", 6322 windows_guest.guest().default_net_string().as_str(), 6323 "tap=,mac=8a:6b:6f:5a:de:ac,ip=192.168.3.1,mask=255.255.255.0", 6324 "tap=mytap42,mac=fe:1f:9e:e1:60:f2,ip=192.168.4.1,mask=255.255.255.0", 6325 ]) 6326 .capture_output() 6327 .spawn() 6328 .unwrap(); 6329 6330 let mut child_dnsmasq = windows_guest.run_dnsmasq(); 6331 6332 let r = std::panic::catch_unwind(|| { 6333 // Wait to make sure Windows boots up 6334 assert!(windows_guest.wait_for_boot()); 6335 6336 let netdev_num = 3; 6337 assert_eq!(windows_guest.netdev_count(), netdev_num); 6338 assert_eq!(netdev_ctrl_threads_count(child.id()), netdev_num); 6339 6340 let tap_count = exec_host_command_output("ip link | grep -c mytap42"); 6341 assert_eq!(String::from_utf8_lossy(&tap_count.stdout).trim(), "1"); 6342 6343 windows_guest.shutdown(); 6344 }); 6345 6346 let _ = child.wait_timeout(std::time::Duration::from_secs(60)); 6347 let _ = child.kill(); 6348 let output = child.wait_with_output().unwrap(); 6349 6350 let _ = child_dnsmasq.kill(); 6351 let _ = child_dnsmasq.wait(); 6352 6353 handle_child_output(r, &output); 6354 } 6355 } 6356 6357 #[cfg(target_arch = "x86_64")] 6358 mod sgx { 6359 use crate::tests::*; 6360 6361 #[test] 6362 fn test_sgx() { 6363 let focal = UbuntuDiskConfig::new(FOCAL_SGX_IMAGE_NAME.to_string()); 6364 let guest = Guest::new(Box::new(focal)); 6365 let mut workload_path = dirs::home_dir().unwrap(); 6366 workload_path.push("workloads"); 6367 6368 let mut kernel_path = workload_path; 6369 kernel_path.push("vmlinux_w_sgx"); 6370 6371 let mut child = GuestCommand::new(&guest) 6372 .args(&["--cpus", "boot=1"]) 6373 .args(&["--memory", "size=512M"]) 6374 .args(&["--kernel", kernel_path.to_str().unwrap()]) 6375 .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) 6376 .default_disks() 6377 .default_net() 6378 .args(&["--sgx-epc", "id=epc0,size=64M"]) 6379 .capture_output() 6380 .spawn() 6381 .unwrap(); 6382 6383 let r = std::panic::catch_unwind(|| { 6384 guest.wait_vm_boot(None).unwrap(); 6385 6386 // Check if SGX is correctly detected in the guest. 6387 guest.check_sgx_support().unwrap(); 6388 6389 // Validate the SGX EPC section is 64MiB. 6390 assert_eq!( 6391 guest 6392 .ssh_command("cpuid -l 0x12 -s 2 | grep 'section size' | cut -d '=' -f 2") 6393 .unwrap() 6394 .trim(), 6395 "0x0000000004000000" 6396 ); 6397 6398 // Run a test relying on SGX enclaves and check if it runs 6399 // successfully. 6400 assert!(guest 6401 .ssh_command("cd /linux-sgx/SampleCode/LocalAttestation/bin/ && sudo ./app") 6402 .unwrap() 6403 .trim() 6404 .contains( 6405 "succeed to load enclaves.\nsucceed to \ 6406 establish secure channel.\nSucceed to exchange \ 6407 secure message...\nSucceed to close Session..." 6408 )); 6409 }); 6410 6411 let _ = child.kill(); 6412 let output = child.wait_with_output().unwrap(); 6413 6414 handle_child_output(r, &output); 6415 } 6416 } 6417 6418 #[cfg(target_arch = "x86_64")] 6419 mod vfio { 6420 use crate::tests::*; 6421 6422 fn test_nvidia_card_memory_hotplug(hotplug_method: &str) { 6423 let hirsute = UbuntuDiskConfig::new(HIRSUTE_NVIDIA_IMAGE_NAME.to_string()); 6424 let guest = Guest::new(Box::new(hirsute)); 6425 let api_socket = temp_api_path(&guest.tmp_dir); 6426 6427 let mut child = GuestCommand::new(&guest) 6428 .args(&["--cpus", "boot=4"]) 6429 .args(&[ 6430 "--memory", 6431 format!("size=4G,hotplug_size=4G,hotplug_method={}", hotplug_method).as_str(), 6432 ]) 6433 .args(&["--kernel", guest.fw_path.as_str()]) 6434 .args(&["--device", "path=/sys/bus/pci/devices/0000:31:00.0/"]) 6435 .args(&["--api-socket", &api_socket]) 6436 .default_disks() 6437 .default_net() 6438 .capture_output() 6439 .spawn() 6440 .unwrap(); 6441 6442 let r = std::panic::catch_unwind(|| { 6443 guest.wait_vm_boot(None).unwrap(); 6444 6445 assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); 6446 6447 guest.enable_memory_hotplug(); 6448 6449 // Add RAM to the VM 6450 let desired_ram = 6 << 30; 6451 resize_command(&api_socket, None, Some(desired_ram), None); 6452 thread::sleep(std::time::Duration::new(30, 0)); 6453 assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); 6454 6455 // Check the VFIO device works when RAM is increased to 6GiB 6456 guest.check_nvidia_gpu(); 6457 }); 6458 6459 let _ = child.kill(); 6460 let output = child.wait_with_output().unwrap(); 6461 6462 handle_child_output(r, &output); 6463 } 6464 6465 #[test] 6466 fn test_nvidia_card_memory_hotplug_acpi() { 6467 test_nvidia_card_memory_hotplug("acpi") 6468 } 6469 6470 #[test] 6471 fn test_nvidia_card_memory_hotplug_virtio_mem() { 6472 test_nvidia_card_memory_hotplug("virtio-mem") 6473 } 6474 6475 #[test] 6476 fn test_nvidia_card_pci_hotplug() { 6477 let hirsute = UbuntuDiskConfig::new(HIRSUTE_NVIDIA_IMAGE_NAME.to_string()); 6478 let guest = Guest::new(Box::new(hirsute)); 6479 let api_socket = temp_api_path(&guest.tmp_dir); 6480 6481 let mut child = GuestCommand::new(&guest) 6482 .args(&["--cpus", "boot=4"]) 6483 .args(&["--memory", "size=4G"]) 6484 .args(&["--kernel", guest.fw_path.as_str()]) 6485 .args(&["--api-socket", &api_socket]) 6486 .default_disks() 6487 .default_net() 6488 .capture_output() 6489 .spawn() 6490 .unwrap(); 6491 6492 let r = std::panic::catch_unwind(|| { 6493 guest.wait_vm_boot(None).unwrap(); 6494 6495 // Hotplug the card to the VM 6496 let (cmd_success, cmd_output) = remote_command_w_output( 6497 &api_socket, 6498 "add-device", 6499 Some("id=vfio0,path=/sys/bus/pci/devices/0000:31:00.0/"), 6500 ); 6501 assert!(cmd_success); 6502 assert!(String::from_utf8_lossy(&cmd_output) 6503 .contains("{\"id\":\"vfio0\",\"bdf\":\"0000:00:06.0\"}")); 6504 6505 thread::sleep(std::time::Duration::new(10, 0)); 6506 6507 // Check the VFIO device works after hotplug 6508 guest.check_nvidia_gpu(); 6509 }); 6510 6511 let _ = child.kill(); 6512 let output = child.wait_with_output().unwrap(); 6513 6514 handle_child_output(r, &output); 6515 } 6516 6517 #[test] 6518 fn test_nvidia_card_reboot() { 6519 let hirsute = UbuntuDiskConfig::new(HIRSUTE_NVIDIA_IMAGE_NAME.to_string()); 6520 let guest = Guest::new(Box::new(hirsute)); 6521 let api_socket = temp_api_path(&guest.tmp_dir); 6522 6523 let mut child = GuestCommand::new(&guest) 6524 .args(&["--cpus", "boot=4"]) 6525 .args(&["--memory", "size=4G"]) 6526 .args(&["--kernel", guest.fw_path.as_str()]) 6527 .args(&["--device", "path=/sys/bus/pci/devices/0000:31:00.0/"]) 6528 .args(&["--api-socket", &api_socket]) 6529 .default_disks() 6530 .default_net() 6531 .capture_output() 6532 .spawn() 6533 .unwrap(); 6534 6535 let r = std::panic::catch_unwind(|| { 6536 guest.wait_vm_boot(None).unwrap(); 6537 6538 // Check the VFIO device works after boot 6539 guest.check_nvidia_gpu(); 6540 6541 guest.reboot_linux(0, None); 6542 6543 // Check the VFIO device works after reboot 6544 guest.check_nvidia_gpu(); 6545 }); 6546 6547 let _ = child.kill(); 6548 let output = child.wait_with_output().unwrap(); 6549 6550 handle_child_output(r, &output); 6551 } 6552 } 6553 } 6554