1 // Copyright © 2022 Intel Corporation 2 // 3 // SPDX-License-Identifier: Apache-2.0 4 // 5 6 // Custom harness to run performance tests 7 extern crate test_infra; 8 #[macro_use(crate_authors)] 9 extern crate clap; 10 11 mod performance_tests; 12 13 use clap::{Arg, ArgAction, Command as ClapCommand}; 14 use performance_tests::*; 15 use serde::{Deserialize, Serialize}; 16 use std::{ 17 env, fmt, 18 process::Command, 19 sync::{mpsc::channel, Arc}, 20 thread, 21 time::Duration, 22 }; 23 use test_infra::FioOps; 24 use thiserror::Error; 25 26 #[derive(Error, Debug)] 27 enum Error { 28 #[error("Error: test timed-out")] 29 TestTimeout, 30 #[error("Error: test failed")] 31 TestFailed, 32 } 33 34 #[derive(Deserialize, Serialize)] 35 pub struct PerformanceTestResult { 36 name: String, 37 mean: f64, 38 std_dev: f64, 39 max: f64, 40 min: f64, 41 } 42 43 #[derive(Deserialize, Serialize)] 44 pub struct MetricsReport { 45 pub git_human_readable: String, 46 pub git_revision: String, 47 pub git_commit_date: String, 48 pub date: String, 49 pub results: Vec<PerformanceTestResult>, 50 } 51 52 impl Default for MetricsReport { 53 fn default() -> Self { 54 let mut git_human_readable = "".to_string(); 55 if let Ok(git_out) = Command::new("git").args(["describe", "--dirty"]).output() { 56 if git_out.status.success() { 57 git_human_readable = String::from_utf8(git_out.stdout) 58 .unwrap() 59 .trim() 60 .to_string(); 61 } else { 62 eprintln!( 63 "Error generating human readable git reference: {}", 64 String::from_utf8(git_out.stderr).unwrap() 65 ); 66 } 67 } 68 69 let mut git_revision = "".to_string(); 70 if let Ok(git_out) = Command::new("git").args(["rev-parse", "HEAD"]).output() { 71 if git_out.status.success() { 72 git_revision = String::from_utf8(git_out.stdout) 73 .unwrap() 74 .trim() 75 .to_string(); 76 } else { 77 eprintln!( 78 "Error generating git reference: {}", 79 String::from_utf8(git_out.stderr).unwrap() 80 ); 81 } 82 } 83 84 let mut git_commit_date = "".to_string(); 85 if let Ok(git_out) = Command::new("git") 86 .args(["show", "-s", "--format=%cd"]) 87 .output() 88 { 89 if git_out.status.success() { 90 git_commit_date = String::from_utf8(git_out.stdout) 91 .unwrap() 92 .trim() 93 .to_string(); 94 } else { 95 eprintln!( 96 "Error generating git commit date: {}", 97 String::from_utf8(git_out.stderr).unwrap() 98 ); 99 } 100 } 101 102 MetricsReport { 103 git_human_readable, 104 git_revision, 105 git_commit_date, 106 date: date(), 107 results: Vec::new(), 108 } 109 } 110 } 111 112 #[derive(Default)] 113 pub struct PerformanceTestOverrides { 114 test_iterations: Option<u32>, 115 } 116 117 impl fmt::Display for PerformanceTestOverrides { 118 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 119 if let Some(test_iterations) = self.test_iterations { 120 write!(f, "test_iterations = {}", test_iterations)?; 121 } 122 123 Ok(()) 124 } 125 } 126 127 pub struct PerformanceTestControl { 128 test_timeout: u32, 129 test_iterations: u32, 130 num_queues: Option<u32>, 131 queue_size: Option<u32>, 132 net_rx: Option<bool>, 133 fio_ops: Option<FioOps>, 134 num_boot_vcpus: Option<u8>, 135 } 136 137 impl fmt::Display for PerformanceTestControl { 138 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 139 let mut output = format!( 140 "test_timeout = {}s, test_iterations = {}", 141 self.test_timeout, self.test_iterations 142 ); 143 if let Some(o) = self.num_queues { 144 output = format!("{}, num_queues = {}", output, o); 145 } 146 if let Some(o) = self.queue_size { 147 output = format!("{}, queue_size = {}", output, o); 148 } 149 if let Some(o) = self.net_rx { 150 output = format!("{}, net_rx = {}", output, o); 151 } 152 if let Some(o) = &self.fio_ops { 153 output = format!("{}, fio_ops = {}", output, o); 154 } 155 156 write!(f, "{}", output) 157 } 158 } 159 160 impl PerformanceTestControl { 161 const fn default() -> Self { 162 Self { 163 test_timeout: 10, 164 test_iterations: 5, 165 num_queues: None, 166 queue_size: None, 167 net_rx: None, 168 fio_ops: None, 169 num_boot_vcpus: Some(1), 170 } 171 } 172 } 173 174 /// A performance test should finish within the a certain time-out and 175 /// return a performance metrics number (including the average number and 176 /// standard deviation) 177 struct PerformanceTest { 178 pub name: &'static str, 179 pub func_ptr: fn(&PerformanceTestControl) -> f64, 180 pub control: PerformanceTestControl, 181 unit_adjuster: fn(f64) -> f64, 182 } 183 184 impl PerformanceTest { 185 pub fn run(&self, overrides: &PerformanceTestOverrides) -> PerformanceTestResult { 186 let mut metrics = Vec::new(); 187 for _ in 0..overrides 188 .test_iterations 189 .unwrap_or(self.control.test_iterations) 190 { 191 metrics.push((self.func_ptr)(&self.control)); 192 } 193 194 let mean = (self.unit_adjuster)(mean(&metrics).unwrap()); 195 let std_dev = (self.unit_adjuster)(std_deviation(&metrics).unwrap()); 196 let max = (self.unit_adjuster)(metrics.clone().into_iter().reduce(f64::max).unwrap()); 197 let min = (self.unit_adjuster)(metrics.clone().into_iter().reduce(f64::min).unwrap()); 198 199 PerformanceTestResult { 200 name: self.name.to_string(), 201 mean, 202 std_dev, 203 max, 204 min, 205 } 206 } 207 208 // Calculate the timeout for each test 209 // Note: To cover the setup/cleanup time, 20s is added for each iteration of the test 210 pub fn calc_timeout(&self, test_iterations: &Option<u32>) -> u64 { 211 ((self.control.test_timeout + 20) * test_iterations.unwrap_or(self.control.test_iterations)) 212 as u64 213 } 214 } 215 216 fn mean(data: &[f64]) -> Option<f64> { 217 let count = data.len(); 218 219 if count > 0 { 220 Some(data.iter().sum::<f64>() / count as f64) 221 } else { 222 None 223 } 224 } 225 226 fn std_deviation(data: &[f64]) -> Option<f64> { 227 let count = data.len(); 228 229 if count > 0 { 230 let mean = mean(data).unwrap(); 231 let variance = data 232 .iter() 233 .map(|value| { 234 let diff = mean - *value; 235 diff * diff 236 }) 237 .sum::<f64>() 238 / count as f64; 239 240 Some(variance.sqrt()) 241 } else { 242 None 243 } 244 } 245 246 mod adjuster { 247 pub fn identity(v: f64) -> f64 { 248 v 249 } 250 251 pub fn s_to_ms(v: f64) -> f64 { 252 v * 1000.0 253 } 254 255 pub fn bps_to_gbps(v: f64) -> f64 { 256 v / (1_000_000_000_f64) 257 } 258 259 #[allow(non_snake_case)] 260 pub fn Bps_to_MiBps(v: f64) -> f64 { 261 v / (1 << 20) as f64 262 } 263 } 264 265 const TEST_LIST: [PerformanceTest; 17] = [ 266 PerformanceTest { 267 name: "boot_time_ms", 268 func_ptr: performance_boot_time, 269 control: PerformanceTestControl { 270 test_timeout: 2, 271 test_iterations: 10, 272 ..PerformanceTestControl::default() 273 }, 274 unit_adjuster: adjuster::s_to_ms, 275 }, 276 PerformanceTest { 277 name: "boot_time_pmem_ms", 278 func_ptr: performance_boot_time_pmem, 279 control: PerformanceTestControl { 280 test_timeout: 2, 281 test_iterations: 10, 282 ..PerformanceTestControl::default() 283 }, 284 unit_adjuster: adjuster::s_to_ms, 285 }, 286 PerformanceTest { 287 name: "boot_time_16_vcpus_ms", 288 func_ptr: performance_boot_time, 289 control: PerformanceTestControl { 290 test_timeout: 2, 291 test_iterations: 10, 292 num_boot_vcpus: Some(16), 293 ..PerformanceTestControl::default() 294 }, 295 unit_adjuster: adjuster::s_to_ms, 296 }, 297 PerformanceTest { 298 name: "boot_time_16_vcpus_pmem_ms", 299 func_ptr: performance_boot_time_pmem, 300 control: PerformanceTestControl { 301 test_timeout: 2, 302 test_iterations: 10, 303 num_boot_vcpus: Some(16), 304 ..PerformanceTestControl::default() 305 }, 306 unit_adjuster: adjuster::s_to_ms, 307 }, 308 PerformanceTest { 309 name: "virtio_net_latency_us", 310 func_ptr: performance_net_latency, 311 control: PerformanceTestControl { 312 num_queues: Some(2), 313 queue_size: Some(256), 314 ..PerformanceTestControl::default() 315 }, 316 unit_adjuster: adjuster::identity, 317 }, 318 PerformanceTest { 319 name: "virtio_net_throughput_single_queue_rx_gbps", 320 func_ptr: performance_net_throughput, 321 control: PerformanceTestControl { 322 num_queues: Some(2), 323 queue_size: Some(256), 324 net_rx: Some(true), 325 ..PerformanceTestControl::default() 326 }, 327 unit_adjuster: adjuster::bps_to_gbps, 328 }, 329 PerformanceTest { 330 name: "virtio_net_throughput_single_queue_tx_gbps", 331 func_ptr: performance_net_throughput, 332 control: PerformanceTestControl { 333 num_queues: Some(2), 334 queue_size: Some(256), 335 net_rx: Some(false), 336 ..PerformanceTestControl::default() 337 }, 338 unit_adjuster: adjuster::bps_to_gbps, 339 }, 340 PerformanceTest { 341 name: "virtio_net_throughput_multi_queue_rx_gbps", 342 func_ptr: performance_net_throughput, 343 control: PerformanceTestControl { 344 num_queues: Some(4), 345 queue_size: Some(256), 346 net_rx: Some(true), 347 ..PerformanceTestControl::default() 348 }, 349 unit_adjuster: adjuster::bps_to_gbps, 350 }, 351 PerformanceTest { 352 name: "virtio_net_throughput_multi_queue_tx_gbps", 353 func_ptr: performance_net_throughput, 354 control: PerformanceTestControl { 355 num_queues: Some(4), 356 queue_size: Some(256), 357 net_rx: Some(false), 358 ..PerformanceTestControl::default() 359 }, 360 unit_adjuster: adjuster::bps_to_gbps, 361 }, 362 PerformanceTest { 363 name: "block_read_MiBps", 364 func_ptr: performance_block_io, 365 control: PerformanceTestControl { 366 num_queues: Some(1), 367 queue_size: Some(128), 368 fio_ops: Some(FioOps::Read), 369 ..PerformanceTestControl::default() 370 }, 371 unit_adjuster: adjuster::Bps_to_MiBps, 372 }, 373 PerformanceTest { 374 name: "block_write_MiBps", 375 func_ptr: performance_block_io, 376 control: PerformanceTestControl { 377 num_queues: Some(1), 378 queue_size: Some(128), 379 fio_ops: Some(FioOps::Write), 380 ..PerformanceTestControl::default() 381 }, 382 unit_adjuster: adjuster::Bps_to_MiBps, 383 }, 384 PerformanceTest { 385 name: "block_random_read_MiBps", 386 func_ptr: performance_block_io, 387 control: PerformanceTestControl { 388 num_queues: Some(1), 389 queue_size: Some(128), 390 fio_ops: Some(FioOps::RandomRead), 391 ..PerformanceTestControl::default() 392 }, 393 unit_adjuster: adjuster::Bps_to_MiBps, 394 }, 395 PerformanceTest { 396 name: "block_random_write_MiBps", 397 func_ptr: performance_block_io, 398 control: PerformanceTestControl { 399 num_queues: Some(1), 400 queue_size: Some(128), 401 fio_ops: Some(FioOps::RandomWrite), 402 ..PerformanceTestControl::default() 403 }, 404 unit_adjuster: adjuster::Bps_to_MiBps, 405 }, 406 PerformanceTest { 407 name: "block_multi_queue_read_MiBps", 408 func_ptr: performance_block_io, 409 control: PerformanceTestControl { 410 num_queues: Some(2), 411 queue_size: Some(128), 412 fio_ops: Some(FioOps::Read), 413 ..PerformanceTestControl::default() 414 }, 415 unit_adjuster: adjuster::Bps_to_MiBps, 416 }, 417 PerformanceTest { 418 name: "block_multi_queue_write_MiBps", 419 func_ptr: performance_block_io, 420 control: PerformanceTestControl { 421 num_queues: Some(2), 422 queue_size: Some(128), 423 fio_ops: Some(FioOps::Write), 424 ..PerformanceTestControl::default() 425 }, 426 unit_adjuster: adjuster::Bps_to_MiBps, 427 }, 428 PerformanceTest { 429 name: "block_multi_queue_random_read_MiBps", 430 func_ptr: performance_block_io, 431 control: PerformanceTestControl { 432 num_queues: Some(2), 433 queue_size: Some(128), 434 fio_ops: Some(FioOps::RandomRead), 435 ..PerformanceTestControl::default() 436 }, 437 unit_adjuster: adjuster::Bps_to_MiBps, 438 }, 439 PerformanceTest { 440 name: "block_multi_queue_random_write_MiBps", 441 func_ptr: performance_block_io, 442 control: PerformanceTestControl { 443 num_queues: Some(2), 444 queue_size: Some(128), 445 fio_ops: Some(FioOps::RandomWrite), 446 ..PerformanceTestControl::default() 447 }, 448 unit_adjuster: adjuster::Bps_to_MiBps, 449 }, 450 ]; 451 452 fn run_test_with_timeout( 453 test: &'static PerformanceTest, 454 overrides: &Arc<PerformanceTestOverrides>, 455 ) -> Result<PerformanceTestResult, Error> { 456 let (sender, receiver) = channel::<Result<PerformanceTestResult, Error>>(); 457 let test_iterations = overrides.test_iterations; 458 let overrides = overrides.clone(); 459 thread::spawn(move || { 460 println!( 461 "Test '{}' running .. (control: {}, overrides: {})", 462 test.name, test.control, overrides 463 ); 464 465 let output = match std::panic::catch_unwind(|| test.run(&overrides)) { 466 Ok(test_result) => { 467 println!( 468 "Test '{}' .. ok: mean = {}, std_dev = {}", 469 test_result.name, test_result.mean, test_result.std_dev 470 ); 471 Ok(test_result) 472 } 473 Err(_) => Err(Error::TestFailed), 474 }; 475 476 let _ = sender.send(output); 477 }); 478 479 // Todo: Need to cleanup/kill all hanging child processes 480 let test_timeout = test.calc_timeout(&test_iterations); 481 receiver 482 .recv_timeout(Duration::from_secs(test_timeout)) 483 .map_err(|_| { 484 eprintln!( 485 "[Error] Test '{}' time-out after {} seconds", 486 test.name, test_timeout 487 ); 488 Error::TestTimeout 489 })? 490 } 491 492 fn date() -> String { 493 let output = test_infra::exec_host_command_output("date"); 494 String::from_utf8_lossy(&output.stdout).trim().to_string() 495 } 496 497 fn main() { 498 let cmd_arguments = ClapCommand::new("performance-metrics") 499 .version(env!("GIT_HUMAN_READABLE")) 500 .author(crate_authors!()) 501 .about("Generate the performance metrics data for Cloud Hypervisor") 502 .arg( 503 Arg::new("test-filter") 504 .long("test-filter") 505 .help("Filter metrics tests to run based on provided keywords") 506 .num_args(1) 507 .required(false), 508 ) 509 .arg( 510 Arg::new("list-tests") 511 .long("list-tests") 512 .help("Print the list of availale metrics tests") 513 .num_args(0) 514 .action(ArgAction::SetTrue) 515 .required(false), 516 ) 517 .arg( 518 Arg::new("report-file") 519 .long("report-file") 520 .help("Report file. Standard error is used if not specified") 521 .num_args(1), 522 ) 523 .arg( 524 Arg::new("iterations") 525 .long("iterations") 526 .help("Override number of test iterations") 527 .num_args(1), 528 ) 529 .get_matches(); 530 531 // It seems that the tool (ethr) used for testing the virtio-net latency 532 // is not stable on AArch64, and therefore the latency test is currently 533 // skipped on AArch64. 534 let test_list: Vec<&PerformanceTest> = TEST_LIST 535 .iter() 536 .filter(|t| !(cfg!(target_arch = "aarch64") && t.name == "virtio_net_latency_us")) 537 .collect(); 538 539 if cmd_arguments.get_flag("list-tests") { 540 for test in test_list.iter() { 541 println!("\"{}\" ({})", test.name, test.control); 542 } 543 544 return; 545 } 546 547 let test_filter = match cmd_arguments.get_many::<String>("test-filter") { 548 Some(s) => s.collect(), 549 None => Vec::new(), 550 }; 551 552 // Run performance tests sequentially and report results (in both readable/json format) 553 let mut metrics_report: MetricsReport = Default::default(); 554 555 init_tests(); 556 557 let overrides = Arc::new(PerformanceTestOverrides { 558 test_iterations: cmd_arguments 559 .get_one::<String>("iterations") 560 .map(|s| s.parse()) 561 .transpose() 562 .unwrap_or_default(), 563 }); 564 565 for test in test_list.iter() { 566 if test_filter.is_empty() || test_filter.iter().any(|&s| test.name.contains(s)) { 567 match run_test_with_timeout(test, &overrides) { 568 Ok(r) => { 569 metrics_report.results.push(r); 570 } 571 Err(e) => { 572 eprintln!("Aborting test due to error: '{:?}'", e); 573 std::process::exit(1); 574 } 575 }; 576 } 577 } 578 579 cleanup_tests(); 580 581 let mut report_file: Box<dyn std::io::Write + Send> = 582 if let Some(file) = cmd_arguments.get_one::<String>("report-file") { 583 Box::new( 584 std::fs::File::create(std::path::Path::new(file)) 585 .map_err(|e| { 586 eprintln!("Error opening report file: {}: {}", file, e); 587 std::process::exit(1); 588 }) 589 .unwrap(), 590 ) 591 } else { 592 Box::new(std::io::stdout()) 593 }; 594 595 report_file 596 .write_all( 597 serde_json::to_string_pretty(&metrics_report) 598 .unwrap() 599 .as_bytes(), 600 ) 601 .map_err(|e| { 602 eprintln!("Error writing report file: {}", e); 603 std::process::exit(1); 604 }) 605 .unwrap(); 606 } 607