1 // Copyright © 2022 Intel Corporation 2 // 3 // SPDX-License-Identifier: Apache-2.0 4 // 5 6 // Custom harness to run performance tests 7 extern crate test_infra; 8 #[macro_use(crate_authors)] 9 extern crate clap; 10 11 mod performance_tests; 12 13 use clap::{Arg, Command as ClapCommand}; 14 use performance_tests::*; 15 use serde_derive::{Deserialize, Serialize}; 16 use std::{env, fmt, process::Command, sync::mpsc::channel, thread, time::Duration}; 17 use thiserror::Error; 18 19 #[derive(Error, Debug)] 20 enum Error { 21 #[error("Error: test timed-out")] 22 TestTimeout, 23 #[error("Error: test failed")] 24 TestFailed, 25 } 26 27 #[derive(Deserialize, Serialize)] 28 pub struct PerformanceTestResult { 29 name: String, 30 mean: f64, 31 std_dev: f64, 32 max: f64, 33 min: f64, 34 } 35 36 #[derive(Deserialize, Serialize)] 37 pub struct MetricsReport { 38 pub git_human_readable: String, 39 pub git_revision: String, 40 pub git_commit_date: String, 41 pub date: String, 42 pub results: Vec<PerformanceTestResult>, 43 } 44 45 impl Default for MetricsReport { 46 fn default() -> Self { 47 let mut git_human_readable = "".to_string(); 48 if let Ok(git_out) = Command::new("git").args(&["describe", "--dirty"]).output() { 49 if git_out.status.success() { 50 if let Ok(git_out_str) = String::from_utf8(git_out.stdout) { 51 git_human_readable = git_out_str.trim().to_string(); 52 } 53 } 54 } 55 56 let mut git_revision = "".to_string(); 57 if let Ok(git_out) = Command::new("git").args(&["rev-parse", "HEAD"]).output() { 58 if git_out.status.success() { 59 if let Ok(git_out_str) = String::from_utf8(git_out.stdout) { 60 git_revision = git_out_str.trim().to_string(); 61 } 62 } 63 } 64 65 let mut git_commit_date = "".to_string(); 66 if let Ok(git_out) = Command::new("git") 67 .args(&["show", "-s", "--format=%cd"]) 68 .output() 69 { 70 if git_out.status.success() { 71 if let Ok(git_out_str) = String::from_utf8(git_out.stdout) { 72 git_commit_date = git_out_str.trim().to_string(); 73 } 74 } 75 } 76 77 MetricsReport { 78 git_human_readable, 79 git_revision, 80 git_commit_date, 81 date: date(), 82 results: Vec::new(), 83 } 84 } 85 } 86 87 pub struct PerformanceTestControl { 88 test_timeout: u32, 89 test_iterations: u32, 90 num_queues: Option<u32>, 91 queue_size: Option<u32>, 92 net_rx: Option<bool>, 93 fio_ops: Option<FioOps>, 94 } 95 96 impl fmt::Display for PerformanceTestControl { 97 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 98 let mut output = format!( 99 "test_timeout = {}s, test_iterations = {}", 100 self.test_timeout, self.test_iterations 101 ); 102 if let Some(o) = self.num_queues { 103 output = format!("{}, num_queues = {}", output, o); 104 } 105 if let Some(o) = self.queue_size { 106 output = format!("{}, queue_size = {}", output, o); 107 } 108 if let Some(o) = self.net_rx { 109 output = format!("{}, net_rx = {}", output, o); 110 } 111 if let Some(o) = &self.fio_ops { 112 output = format!("{}, fio_ops = {}", output, o); 113 } 114 115 write!(f, "{}", output) 116 } 117 } 118 119 impl PerformanceTestControl { 120 const fn default() -> Self { 121 Self { 122 test_timeout: 10, 123 test_iterations: 5, 124 num_queues: None, 125 queue_size: None, 126 net_rx: None, 127 fio_ops: None, 128 } 129 } 130 } 131 132 /// A performance test should finish within the a certain time-out and 133 /// return a performance metrics number (including the average number and 134 /// standard deviation) 135 struct PerformanceTest { 136 pub name: &'static str, 137 pub func_ptr: fn(&PerformanceTestControl) -> f64, 138 pub control: PerformanceTestControl, 139 unit_adjuster: fn(f64) -> f64, 140 } 141 142 impl PerformanceTest { 143 pub fn run(&self) -> PerformanceTestResult { 144 let mut metrics = Vec::new(); 145 for _ in 0..self.control.test_iterations { 146 metrics.push((self.func_ptr)(&self.control)); 147 } 148 149 let mean = (self.unit_adjuster)(mean(&metrics).unwrap()); 150 let std_dev = (self.unit_adjuster)(std_deviation(&metrics).unwrap()); 151 let max = (self.unit_adjuster)(metrics.clone().into_iter().reduce(f64::max).unwrap()); 152 let min = (self.unit_adjuster)(metrics.clone().into_iter().reduce(f64::min).unwrap()); 153 154 PerformanceTestResult { 155 name: self.name.to_string(), 156 mean, 157 std_dev, 158 max, 159 min, 160 } 161 } 162 163 // Calculate the timeout for each test 164 // Note: To cover the setup/cleanup time, 20s is added for each iteration of the test 165 pub fn calc_timeout(&self) -> u64 { 166 ((self.control.test_timeout + 20) * self.control.test_iterations) as u64 167 } 168 } 169 170 fn mean(data: &[f64]) -> Option<f64> { 171 let count = data.len(); 172 173 if count > 0 { 174 Some(data.iter().sum::<f64>() / count as f64) 175 } else { 176 None 177 } 178 } 179 180 fn std_deviation(data: &[f64]) -> Option<f64> { 181 let count = data.len(); 182 183 if count > 0 { 184 let mean = mean(data).unwrap(); 185 let variance = data 186 .iter() 187 .map(|value| { 188 let diff = mean - *value; 189 diff * diff 190 }) 191 .sum::<f64>() 192 / count as f64; 193 194 Some(variance.sqrt()) 195 } else { 196 None 197 } 198 } 199 200 mod adjuster { 201 pub fn identity(v: f64) -> f64 { 202 v 203 } 204 205 pub fn s_to_ms(v: f64) -> f64 { 206 v * 1000.0 207 } 208 209 pub fn bps_to_gbps(v: f64) -> f64 { 210 v / (1_000_000_000_f64) 211 } 212 213 #[allow(non_snake_case)] 214 pub fn Bps_to_MiBps(v: f64) -> f64 { 215 v / (1 << 20) as f64 216 } 217 } 218 219 const TEST_LIST: [PerformanceTest; 15] = [ 220 PerformanceTest { 221 name: "boot_time_ms", 222 func_ptr: performance_boot_time, 223 control: PerformanceTestControl { 224 test_timeout: 2, 225 test_iterations: 10, 226 ..PerformanceTestControl::default() 227 }, 228 unit_adjuster: adjuster::s_to_ms, 229 }, 230 PerformanceTest { 231 name: "boot_time_pmem_ms", 232 func_ptr: performance_boot_time_pmem, 233 control: PerformanceTestControl { 234 test_timeout: 2, 235 test_iterations: 10, 236 ..PerformanceTestControl::default() 237 }, 238 unit_adjuster: adjuster::s_to_ms, 239 }, 240 PerformanceTest { 241 name: "virtio_net_latency_us", 242 func_ptr: performance_net_latency, 243 control: PerformanceTestControl { 244 num_queues: Some(2), 245 queue_size: Some(256), 246 ..PerformanceTestControl::default() 247 }, 248 unit_adjuster: adjuster::identity, 249 }, 250 PerformanceTest { 251 name: "virtio_net_throughput_single_queue_rx_gbps", 252 func_ptr: performance_net_throughput, 253 control: PerformanceTestControl { 254 num_queues: Some(2), 255 queue_size: Some(256), 256 net_rx: Some(true), 257 ..PerformanceTestControl::default() 258 }, 259 unit_adjuster: adjuster::bps_to_gbps, 260 }, 261 PerformanceTest { 262 name: "virtio_net_throughput_single_queue_tx_gbps", 263 func_ptr: performance_net_throughput, 264 control: PerformanceTestControl { 265 num_queues: Some(2), 266 queue_size: Some(256), 267 net_rx: Some(false), 268 ..PerformanceTestControl::default() 269 }, 270 unit_adjuster: adjuster::bps_to_gbps, 271 }, 272 PerformanceTest { 273 name: "virtio_net_throughput_multi_queue_rx_gbps", 274 func_ptr: performance_net_throughput, 275 control: PerformanceTestControl { 276 num_queues: Some(4), 277 queue_size: Some(256), 278 net_rx: Some(true), 279 ..PerformanceTestControl::default() 280 }, 281 unit_adjuster: adjuster::bps_to_gbps, 282 }, 283 PerformanceTest { 284 name: "virtio_net_throughput_multi_queue_tx_gbps", 285 func_ptr: performance_net_throughput, 286 control: PerformanceTestControl { 287 num_queues: Some(4), 288 queue_size: Some(256), 289 net_rx: Some(false), 290 ..PerformanceTestControl::default() 291 }, 292 unit_adjuster: adjuster::bps_to_gbps, 293 }, 294 PerformanceTest { 295 name: "block_read_MiBps", 296 func_ptr: performance_block_io, 297 control: PerformanceTestControl { 298 num_queues: Some(1), 299 queue_size: Some(128), 300 fio_ops: Some(FioOps::Read), 301 ..PerformanceTestControl::default() 302 }, 303 unit_adjuster: adjuster::Bps_to_MiBps, 304 }, 305 PerformanceTest { 306 name: "block_write_MiBps", 307 func_ptr: performance_block_io, 308 control: PerformanceTestControl { 309 num_queues: Some(1), 310 queue_size: Some(128), 311 fio_ops: Some(FioOps::Write), 312 ..PerformanceTestControl::default() 313 }, 314 unit_adjuster: adjuster::Bps_to_MiBps, 315 }, 316 PerformanceTest { 317 name: "block_random_read_MiBps", 318 func_ptr: performance_block_io, 319 control: PerformanceTestControl { 320 num_queues: Some(1), 321 queue_size: Some(128), 322 fio_ops: Some(FioOps::RandomRead), 323 ..PerformanceTestControl::default() 324 }, 325 unit_adjuster: adjuster::Bps_to_MiBps, 326 }, 327 PerformanceTest { 328 name: "block_random_write_MiBps", 329 func_ptr: performance_block_io, 330 control: PerformanceTestControl { 331 num_queues: Some(1), 332 queue_size: Some(128), 333 fio_ops: Some(FioOps::RandomWrite), 334 ..PerformanceTestControl::default() 335 }, 336 unit_adjuster: adjuster::Bps_to_MiBps, 337 }, 338 PerformanceTest { 339 name: "block_multi_queue_read_MiBps", 340 func_ptr: performance_block_io, 341 control: PerformanceTestControl { 342 num_queues: Some(2), 343 queue_size: Some(128), 344 fio_ops: Some(FioOps::Read), 345 ..PerformanceTestControl::default() 346 }, 347 unit_adjuster: adjuster::Bps_to_MiBps, 348 }, 349 PerformanceTest { 350 name: "block_multi_queue_write_MiBps", 351 func_ptr: performance_block_io, 352 control: PerformanceTestControl { 353 num_queues: Some(2), 354 queue_size: Some(128), 355 fio_ops: Some(FioOps::Write), 356 ..PerformanceTestControl::default() 357 }, 358 unit_adjuster: adjuster::Bps_to_MiBps, 359 }, 360 PerformanceTest { 361 name: "block_multi_queue_random_read_MiBps", 362 func_ptr: performance_block_io, 363 control: PerformanceTestControl { 364 num_queues: Some(2), 365 queue_size: Some(128), 366 fio_ops: Some(FioOps::RandomRead), 367 ..PerformanceTestControl::default() 368 }, 369 unit_adjuster: adjuster::Bps_to_MiBps, 370 }, 371 PerformanceTest { 372 name: "block_multi_queue_random_write_MiBps", 373 func_ptr: performance_block_io, 374 control: PerformanceTestControl { 375 num_queues: Some(2), 376 queue_size: Some(128), 377 fio_ops: Some(FioOps::RandomWrite), 378 ..PerformanceTestControl::default() 379 }, 380 unit_adjuster: adjuster::Bps_to_MiBps, 381 }, 382 ]; 383 384 fn run_test_with_timeout(test: &'static PerformanceTest) -> Result<PerformanceTestResult, Error> { 385 let (sender, receiver) = channel::<Result<PerformanceTestResult, Error>>(); 386 thread::spawn(move || { 387 println!("Test '{}' running .. ({})", test.name, test.control); 388 389 let output = match std::panic::catch_unwind(|| test.run()) { 390 Ok(test_result) => { 391 println!( 392 "Test '{}' .. ok: mean = {}, std_dev = {}", 393 test_result.name, test_result.mean, test_result.std_dev 394 ); 395 Ok(test_result) 396 } 397 Err(_) => Err(Error::TestFailed), 398 }; 399 400 let _ = sender.send(output); 401 }); 402 403 // Todo: Need to cleanup/kill all hanging child processes 404 let test_timeout = test.calc_timeout(); 405 receiver 406 .recv_timeout(Duration::from_secs(test_timeout)) 407 .map_err(|_| { 408 eprintln!( 409 "[Error] Test '{}' time-out after {} seconds", 410 test.name, test_timeout 411 ); 412 Error::TestTimeout 413 })? 414 } 415 416 fn date() -> String { 417 let output = test_infra::exec_host_command_output("date"); 418 String::from_utf8_lossy(&output.stdout).trim().to_string() 419 } 420 421 fn main() { 422 let cmd_arguments = ClapCommand::new("performance-metrics") 423 .version(env!("GIT_HUMAN_READABLE")) 424 .author(crate_authors!()) 425 .about("Generate the performance metrics data for Cloud Hypervisor") 426 .arg( 427 Arg::new("test-filter") 428 .long("test-filter") 429 .help("Filter metrics tests to run based on provided keywords") 430 .multiple_occurrences(true) 431 .takes_value(true) 432 .required(false), 433 ) 434 .arg( 435 Arg::new("list-tests") 436 .long("list-tests") 437 .help("Print the list of availale metrics tests") 438 .multiple_occurrences(true) 439 .takes_value(false) 440 .required(false), 441 ) 442 .arg( 443 Arg::new("report-file") 444 .long("report-file") 445 .help("Report file. Standard error is used if not specified") 446 .takes_value(true), 447 ) 448 .get_matches(); 449 450 // It seems that the tool (ethr) used for testing the virtio-net latency 451 // is not stable on AArch64, and therefore the latency test is currently 452 // skipped on AArch64. 453 let test_list: Vec<&PerformanceTest> = TEST_LIST 454 .iter() 455 .filter(|t| !(cfg!(target_arch = "aarch64") && t.name == "virtio_net_latency_us")) 456 .collect(); 457 458 if cmd_arguments.is_present("list-tests") { 459 for test in test_list.iter() { 460 println!("\"{}\" ({})", test.name, test.control); 461 } 462 463 return; 464 } 465 466 let test_filter = match cmd_arguments.values_of("test-filter") { 467 Some(s) => s.collect(), 468 None => Vec::new(), 469 }; 470 471 // Run performance tests sequentially and report results (in both readable/json format) 472 let mut metrics_report: MetricsReport = Default::default(); 473 474 init_tests(); 475 476 for test in test_list.iter() { 477 if test_filter.is_empty() || test_filter.iter().any(|&s| test.name.contains(s)) { 478 match run_test_with_timeout(test) { 479 Ok(r) => { 480 metrics_report.results.push(r); 481 } 482 Err(e) => { 483 eprintln!("Aborting test due to error: '{:?}'", e); 484 std::process::exit(1); 485 } 486 }; 487 } 488 } 489 490 cleanup_tests(); 491 492 let mut report_file: Box<dyn std::io::Write + Send> = 493 if let Some(file) = cmd_arguments.value_of("report-file") { 494 Box::new( 495 std::fs::File::create(std::path::Path::new(file)) 496 .map_err(|e| { 497 eprintln!("Error opening report file: {}: {}", file, e); 498 std::process::exit(1); 499 }) 500 .unwrap(), 501 ) 502 } else { 503 Box::new(std::io::stdout()) 504 }; 505 506 report_file 507 .write_all( 508 serde_json::to_string_pretty(&metrics_report) 509 .unwrap() 510 .as_bytes(), 511 ) 512 .map_err(|e| { 513 eprintln!("Error writing report file: {}", e); 514 std::process::exit(1); 515 }) 516 .unwrap(); 517 } 518