1 // Copyright © 2022 Intel Corporation 2 // 3 // SPDX-License-Identifier: Apache-2.0 4 // 5 6 // Custom harness to run performance tests 7 extern crate test_infra; 8 #[macro_use(crate_authors)] 9 extern crate clap; 10 11 mod performance_tests; 12 13 use clap::{Arg, Command as ClapCommand}; 14 use performance_tests::*; 15 use serde::{Deserialize, Serialize}; 16 use std::{ 17 env, fmt, 18 process::Command, 19 sync::{mpsc::channel, Arc}, 20 thread, 21 time::Duration, 22 }; 23 use thiserror::Error; 24 25 #[derive(Error, Debug)] 26 enum Error { 27 #[error("Error: test timed-out")] 28 TestTimeout, 29 #[error("Error: test failed")] 30 TestFailed, 31 } 32 33 #[derive(Deserialize, Serialize)] 34 pub struct PerformanceTestResult { 35 name: String, 36 mean: f64, 37 std_dev: f64, 38 max: f64, 39 min: f64, 40 } 41 42 #[derive(Deserialize, Serialize)] 43 pub struct MetricsReport { 44 pub git_human_readable: String, 45 pub git_revision: String, 46 pub git_commit_date: String, 47 pub date: String, 48 pub results: Vec<PerformanceTestResult>, 49 } 50 51 impl Default for MetricsReport { 52 fn default() -> Self { 53 let mut git_human_readable = "".to_string(); 54 if let Ok(git_out) = Command::new("git").args(&["describe", "--dirty"]).output() { 55 if git_out.status.success() { 56 git_human_readable = String::from_utf8(git_out.stdout) 57 .unwrap() 58 .trim() 59 .to_string(); 60 } else { 61 eprintln!( 62 "Error generating human readable git reference: {}", 63 String::from_utf8(git_out.stderr).unwrap() 64 ); 65 } 66 } 67 68 let mut git_revision = "".to_string(); 69 if let Ok(git_out) = Command::new("git").args(&["rev-parse", "HEAD"]).output() { 70 if git_out.status.success() { 71 git_revision = String::from_utf8(git_out.stdout) 72 .unwrap() 73 .trim() 74 .to_string(); 75 } else { 76 eprintln!( 77 "Error generating git reference: {}", 78 String::from_utf8(git_out.stderr).unwrap() 79 ); 80 } 81 } 82 83 let mut git_commit_date = "".to_string(); 84 if let Ok(git_out) = Command::new("git") 85 .args(&["show", "-s", "--format=%cd"]) 86 .output() 87 { 88 if git_out.status.success() { 89 git_commit_date = String::from_utf8(git_out.stdout) 90 .unwrap() 91 .trim() 92 .to_string(); 93 } else { 94 eprintln!( 95 "Error generating git commit date: {}", 96 String::from_utf8(git_out.stderr).unwrap() 97 ); 98 } 99 } 100 101 MetricsReport { 102 git_human_readable, 103 git_revision, 104 git_commit_date, 105 date: date(), 106 results: Vec::new(), 107 } 108 } 109 } 110 111 #[derive(Default)] 112 pub struct PerformanceTestOverrides { 113 test_iterations: Option<u32>, 114 } 115 116 impl fmt::Display for PerformanceTestOverrides { 117 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 118 if let Some(test_iterations) = self.test_iterations { 119 write!(f, "test_iterations = {}", test_iterations)?; 120 } 121 122 Ok(()) 123 } 124 } 125 126 pub struct PerformanceTestControl { 127 test_timeout: u32, 128 test_iterations: u32, 129 num_queues: Option<u32>, 130 queue_size: Option<u32>, 131 net_rx: Option<bool>, 132 fio_ops: Option<FioOps>, 133 num_boot_vcpus: Option<u8>, 134 } 135 136 impl fmt::Display for PerformanceTestControl { 137 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 138 let mut output = format!( 139 "test_timeout = {}s, test_iterations = {}", 140 self.test_timeout, self.test_iterations 141 ); 142 if let Some(o) = self.num_queues { 143 output = format!("{}, num_queues = {}", output, o); 144 } 145 if let Some(o) = self.queue_size { 146 output = format!("{}, queue_size = {}", output, o); 147 } 148 if let Some(o) = self.net_rx { 149 output = format!("{}, net_rx = {}", output, o); 150 } 151 if let Some(o) = &self.fio_ops { 152 output = format!("{}, fio_ops = {}", output, o); 153 } 154 155 write!(f, "{}", output) 156 } 157 } 158 159 impl PerformanceTestControl { 160 const fn default() -> Self { 161 Self { 162 test_timeout: 10, 163 test_iterations: 5, 164 num_queues: None, 165 queue_size: None, 166 net_rx: None, 167 fio_ops: None, 168 num_boot_vcpus: Some(1), 169 } 170 } 171 } 172 173 /// A performance test should finish within the a certain time-out and 174 /// return a performance metrics number (including the average number and 175 /// standard deviation) 176 struct PerformanceTest { 177 pub name: &'static str, 178 pub func_ptr: fn(&PerformanceTestControl) -> f64, 179 pub control: PerformanceTestControl, 180 unit_adjuster: fn(f64) -> f64, 181 } 182 183 impl PerformanceTest { 184 pub fn run(&self, overrides: &PerformanceTestOverrides) -> PerformanceTestResult { 185 let mut metrics = Vec::new(); 186 for _ in 0..overrides 187 .test_iterations 188 .unwrap_or(self.control.test_iterations) 189 { 190 metrics.push((self.func_ptr)(&self.control)); 191 } 192 193 let mean = (self.unit_adjuster)(mean(&metrics).unwrap()); 194 let std_dev = (self.unit_adjuster)(std_deviation(&metrics).unwrap()); 195 let max = (self.unit_adjuster)(metrics.clone().into_iter().reduce(f64::max).unwrap()); 196 let min = (self.unit_adjuster)(metrics.clone().into_iter().reduce(f64::min).unwrap()); 197 198 PerformanceTestResult { 199 name: self.name.to_string(), 200 mean, 201 std_dev, 202 max, 203 min, 204 } 205 } 206 207 // Calculate the timeout for each test 208 // Note: To cover the setup/cleanup time, 20s is added for each iteration of the test 209 pub fn calc_timeout(&self, test_iterations: &Option<u32>) -> u64 { 210 ((self.control.test_timeout + 20) * test_iterations.unwrap_or(self.control.test_iterations)) 211 as u64 212 } 213 } 214 215 fn mean(data: &[f64]) -> Option<f64> { 216 let count = data.len(); 217 218 if count > 0 { 219 Some(data.iter().sum::<f64>() / count as f64) 220 } else { 221 None 222 } 223 } 224 225 fn std_deviation(data: &[f64]) -> Option<f64> { 226 let count = data.len(); 227 228 if count > 0 { 229 let mean = mean(data).unwrap(); 230 let variance = data 231 .iter() 232 .map(|value| { 233 let diff = mean - *value; 234 diff * diff 235 }) 236 .sum::<f64>() 237 / count as f64; 238 239 Some(variance.sqrt()) 240 } else { 241 None 242 } 243 } 244 245 mod adjuster { 246 pub fn identity(v: f64) -> f64 { 247 v 248 } 249 250 pub fn s_to_ms(v: f64) -> f64 { 251 v * 1000.0 252 } 253 254 pub fn bps_to_gbps(v: f64) -> f64 { 255 v / (1_000_000_000_f64) 256 } 257 258 #[allow(non_snake_case)] 259 pub fn Bps_to_MiBps(v: f64) -> f64 { 260 v / (1 << 20) as f64 261 } 262 } 263 264 const TEST_LIST: [PerformanceTest; 17] = [ 265 PerformanceTest { 266 name: "boot_time_ms", 267 func_ptr: performance_boot_time, 268 control: PerformanceTestControl { 269 test_timeout: 2, 270 test_iterations: 10, 271 ..PerformanceTestControl::default() 272 }, 273 unit_adjuster: adjuster::s_to_ms, 274 }, 275 PerformanceTest { 276 name: "boot_time_pmem_ms", 277 func_ptr: performance_boot_time_pmem, 278 control: PerformanceTestControl { 279 test_timeout: 2, 280 test_iterations: 10, 281 ..PerformanceTestControl::default() 282 }, 283 unit_adjuster: adjuster::s_to_ms, 284 }, 285 PerformanceTest { 286 name: "boot_time_16_vcpus_ms", 287 func_ptr: performance_boot_time, 288 control: PerformanceTestControl { 289 test_timeout: 2, 290 test_iterations: 10, 291 num_boot_vcpus: Some(16), 292 ..PerformanceTestControl::default() 293 }, 294 unit_adjuster: adjuster::s_to_ms, 295 }, 296 PerformanceTest { 297 name: "boot_time_16_vcpus_pmem_ms", 298 func_ptr: performance_boot_time_pmem, 299 control: PerformanceTestControl { 300 test_timeout: 2, 301 test_iterations: 10, 302 num_boot_vcpus: Some(16), 303 ..PerformanceTestControl::default() 304 }, 305 unit_adjuster: adjuster::s_to_ms, 306 }, 307 PerformanceTest { 308 name: "virtio_net_latency_us", 309 func_ptr: performance_net_latency, 310 control: PerformanceTestControl { 311 num_queues: Some(2), 312 queue_size: Some(256), 313 ..PerformanceTestControl::default() 314 }, 315 unit_adjuster: adjuster::identity, 316 }, 317 PerformanceTest { 318 name: "virtio_net_throughput_single_queue_rx_gbps", 319 func_ptr: performance_net_throughput, 320 control: PerformanceTestControl { 321 num_queues: Some(2), 322 queue_size: Some(256), 323 net_rx: Some(true), 324 ..PerformanceTestControl::default() 325 }, 326 unit_adjuster: adjuster::bps_to_gbps, 327 }, 328 PerformanceTest { 329 name: "virtio_net_throughput_single_queue_tx_gbps", 330 func_ptr: performance_net_throughput, 331 control: PerformanceTestControl { 332 num_queues: Some(2), 333 queue_size: Some(256), 334 net_rx: Some(false), 335 ..PerformanceTestControl::default() 336 }, 337 unit_adjuster: adjuster::bps_to_gbps, 338 }, 339 PerformanceTest { 340 name: "virtio_net_throughput_multi_queue_rx_gbps", 341 func_ptr: performance_net_throughput, 342 control: PerformanceTestControl { 343 num_queues: Some(4), 344 queue_size: Some(256), 345 net_rx: Some(true), 346 ..PerformanceTestControl::default() 347 }, 348 unit_adjuster: adjuster::bps_to_gbps, 349 }, 350 PerformanceTest { 351 name: "virtio_net_throughput_multi_queue_tx_gbps", 352 func_ptr: performance_net_throughput, 353 control: PerformanceTestControl { 354 num_queues: Some(4), 355 queue_size: Some(256), 356 net_rx: Some(false), 357 ..PerformanceTestControl::default() 358 }, 359 unit_adjuster: adjuster::bps_to_gbps, 360 }, 361 PerformanceTest { 362 name: "block_read_MiBps", 363 func_ptr: performance_block_io, 364 control: PerformanceTestControl { 365 num_queues: Some(1), 366 queue_size: Some(128), 367 fio_ops: Some(FioOps::Read), 368 ..PerformanceTestControl::default() 369 }, 370 unit_adjuster: adjuster::Bps_to_MiBps, 371 }, 372 PerformanceTest { 373 name: "block_write_MiBps", 374 func_ptr: performance_block_io, 375 control: PerformanceTestControl { 376 num_queues: Some(1), 377 queue_size: Some(128), 378 fio_ops: Some(FioOps::Write), 379 ..PerformanceTestControl::default() 380 }, 381 unit_adjuster: adjuster::Bps_to_MiBps, 382 }, 383 PerformanceTest { 384 name: "block_random_read_MiBps", 385 func_ptr: performance_block_io, 386 control: PerformanceTestControl { 387 num_queues: Some(1), 388 queue_size: Some(128), 389 fio_ops: Some(FioOps::RandomRead), 390 ..PerformanceTestControl::default() 391 }, 392 unit_adjuster: adjuster::Bps_to_MiBps, 393 }, 394 PerformanceTest { 395 name: "block_random_write_MiBps", 396 func_ptr: performance_block_io, 397 control: PerformanceTestControl { 398 num_queues: Some(1), 399 queue_size: Some(128), 400 fio_ops: Some(FioOps::RandomWrite), 401 ..PerformanceTestControl::default() 402 }, 403 unit_adjuster: adjuster::Bps_to_MiBps, 404 }, 405 PerformanceTest { 406 name: "block_multi_queue_read_MiBps", 407 func_ptr: performance_block_io, 408 control: PerformanceTestControl { 409 num_queues: Some(2), 410 queue_size: Some(128), 411 fio_ops: Some(FioOps::Read), 412 ..PerformanceTestControl::default() 413 }, 414 unit_adjuster: adjuster::Bps_to_MiBps, 415 }, 416 PerformanceTest { 417 name: "block_multi_queue_write_MiBps", 418 func_ptr: performance_block_io, 419 control: PerformanceTestControl { 420 num_queues: Some(2), 421 queue_size: Some(128), 422 fio_ops: Some(FioOps::Write), 423 ..PerformanceTestControl::default() 424 }, 425 unit_adjuster: adjuster::Bps_to_MiBps, 426 }, 427 PerformanceTest { 428 name: "block_multi_queue_random_read_MiBps", 429 func_ptr: performance_block_io, 430 control: PerformanceTestControl { 431 num_queues: Some(2), 432 queue_size: Some(128), 433 fio_ops: Some(FioOps::RandomRead), 434 ..PerformanceTestControl::default() 435 }, 436 unit_adjuster: adjuster::Bps_to_MiBps, 437 }, 438 PerformanceTest { 439 name: "block_multi_queue_random_write_MiBps", 440 func_ptr: performance_block_io, 441 control: PerformanceTestControl { 442 num_queues: Some(2), 443 queue_size: Some(128), 444 fio_ops: Some(FioOps::RandomWrite), 445 ..PerformanceTestControl::default() 446 }, 447 unit_adjuster: adjuster::Bps_to_MiBps, 448 }, 449 ]; 450 451 fn run_test_with_timeout( 452 test: &'static PerformanceTest, 453 overrides: &Arc<PerformanceTestOverrides>, 454 ) -> Result<PerformanceTestResult, Error> { 455 let (sender, receiver) = channel::<Result<PerformanceTestResult, Error>>(); 456 let test_iterations = overrides.test_iterations; 457 let overrides = overrides.clone(); 458 thread::spawn(move || { 459 println!( 460 "Test '{}' running .. (control: {}, overrides: {})", 461 test.name, test.control, overrides 462 ); 463 464 let output = match std::panic::catch_unwind(|| test.run(&overrides)) { 465 Ok(test_result) => { 466 println!( 467 "Test '{}' .. ok: mean = {}, std_dev = {}", 468 test_result.name, test_result.mean, test_result.std_dev 469 ); 470 Ok(test_result) 471 } 472 Err(_) => Err(Error::TestFailed), 473 }; 474 475 let _ = sender.send(output); 476 }); 477 478 // Todo: Need to cleanup/kill all hanging child processes 479 let test_timeout = test.calc_timeout(&test_iterations); 480 receiver 481 .recv_timeout(Duration::from_secs(test_timeout)) 482 .map_err(|_| { 483 eprintln!( 484 "[Error] Test '{}' time-out after {} seconds", 485 test.name, test_timeout 486 ); 487 Error::TestTimeout 488 })? 489 } 490 491 fn date() -> String { 492 let output = test_infra::exec_host_command_output("date"); 493 String::from_utf8_lossy(&output.stdout).trim().to_string() 494 } 495 496 fn main() { 497 let cmd_arguments = ClapCommand::new("performance-metrics") 498 .version(env!("GIT_HUMAN_READABLE")) 499 .author(crate_authors!()) 500 .about("Generate the performance metrics data for Cloud Hypervisor") 501 .arg( 502 Arg::new("test-filter") 503 .long("test-filter") 504 .help("Filter metrics tests to run based on provided keywords") 505 .multiple_occurrences(true) 506 .takes_value(true) 507 .required(false), 508 ) 509 .arg( 510 Arg::new("list-tests") 511 .long("list-tests") 512 .help("Print the list of availale metrics tests") 513 .multiple_occurrences(true) 514 .takes_value(false) 515 .required(false), 516 ) 517 .arg( 518 Arg::new("report-file") 519 .long("report-file") 520 .help("Report file. Standard error is used if not specified") 521 .takes_value(true), 522 ) 523 .arg( 524 Arg::new("iterations") 525 .long("iterations") 526 .help("Override number of test iterations") 527 .takes_value(true), 528 ) 529 .get_matches(); 530 531 // It seems that the tool (ethr) used for testing the virtio-net latency 532 // is not stable on AArch64, and therefore the latency test is currently 533 // skipped on AArch64. 534 let test_list: Vec<&PerformanceTest> = TEST_LIST 535 .iter() 536 .filter(|t| !(cfg!(target_arch = "aarch64") && t.name == "virtio_net_latency_us")) 537 .collect(); 538 539 if cmd_arguments.is_present("list-tests") { 540 for test in test_list.iter() { 541 println!("\"{}\" ({})", test.name, test.control); 542 } 543 544 return; 545 } 546 547 let test_filter = match cmd_arguments.values_of("test-filter") { 548 Some(s) => s.collect(), 549 None => Vec::new(), 550 }; 551 552 // Run performance tests sequentially and report results (in both readable/json format) 553 let mut metrics_report: MetricsReport = Default::default(); 554 555 init_tests(); 556 557 let overrides = Arc::new(PerformanceTestOverrides { 558 test_iterations: cmd_arguments 559 .value_of("iterations") 560 .map(|s| s.parse()) 561 .transpose() 562 .unwrap_or_default(), 563 }); 564 565 for test in test_list.iter() { 566 if test_filter.is_empty() || test_filter.iter().any(|&s| test.name.contains(s)) { 567 match run_test_with_timeout(test, &overrides) { 568 Ok(r) => { 569 metrics_report.results.push(r); 570 } 571 Err(e) => { 572 eprintln!("Aborting test due to error: '{:?}'", e); 573 std::process::exit(1); 574 } 575 }; 576 } 577 } 578 579 cleanup_tests(); 580 581 let mut report_file: Box<dyn std::io::Write + Send> = 582 if let Some(file) = cmd_arguments.value_of("report-file") { 583 Box::new( 584 std::fs::File::create(std::path::Path::new(file)) 585 .map_err(|e| { 586 eprintln!("Error opening report file: {}: {}", file, e); 587 std::process::exit(1); 588 }) 589 .unwrap(), 590 ) 591 } else { 592 Box::new(std::io::stdout()) 593 }; 594 595 report_file 596 .write_all( 597 serde_json::to_string_pretty(&metrics_report) 598 .unwrap() 599 .as_bytes(), 600 ) 601 .map_err(|e| { 602 eprintln!("Error writing report file: {}", e); 603 std::process::exit(1); 604 }) 605 .unwrap(); 606 } 607