xref: /cloud-hypervisor/performance-metrics/src/performance_tests.rs (revision 20296e909a88ab1dd9d94b6bd2160ca84c430f5f)
1 // Copyright © 2022 Intel Corporation
2 //
3 // SPDX-License-Identifier: Apache-2.0
4 //
5 
6 // Performance tests
7 
8 use std::path::PathBuf;
9 use std::time::Duration;
10 use std::{fs, thread};
11 
12 use test_infra::{Error as InfraError, *};
13 use thiserror::Error;
14 
15 use crate::{mean, PerformanceTestControl};
16 
17 #[cfg(target_arch = "x86_64")]
18 pub const FOCAL_IMAGE_NAME: &str = "focal-server-cloudimg-amd64-custom-20210609-0.raw";
19 #[cfg(target_arch = "aarch64")]
20 pub const FOCAL_IMAGE_NAME: &str = "focal-server-cloudimg-arm64-custom-20210929-0-update-tool.raw";
21 
22 #[allow(dead_code)]
23 #[derive(Error, Debug)]
24 enum Error {
25     #[error("boot time could not be parsed")]
26     BootTimeParse,
27     #[error("infrastructure failure: {0}")]
28     Infra(#[source] InfraError),
29     #[error("restore time could not be parsed")]
30     RestoreTimeParse,
31 }
32 
33 impl From<InfraError> for Error {
34     fn from(e: InfraError) -> Self {
35         Self::Infra(e)
36     }
37 }
38 
39 const BLK_IO_TEST_IMG: &str = "/var/tmp/ch-blk-io-test.img";
40 
41 pub fn init_tests() {
42     // The test image cannot be created on tmpfs (e.g. /tmp) filesystem,
43     // as tmpfs does not support O_DIRECT
44     assert!(exec_host_command_output(&format!(
45         "dd if=/dev/zero of={BLK_IO_TEST_IMG} bs=1M count=4096"
46     ))
47     .status
48     .success());
49 }
50 
51 pub fn cleanup_tests() {
52     fs::remove_file(BLK_IO_TEST_IMG)
53         .unwrap_or_else(|_| panic!("Failed to remove file '{BLK_IO_TEST_IMG}'."));
54 }
55 
56 // Performance tests are expected to be executed sequentially, so we can
57 // start VM guests with the same IP while putting them on a different
58 // private network. The default constructor "Guest::new()" does not work
59 // well, as we can easily create more than 256 VMs from repeating various
60 // performance tests dozens times in a single run.
61 fn performance_test_new_guest(disk_config: Box<dyn DiskConfig>) -> Guest {
62     Guest::new_from_ip_range(disk_config, "172.19", 0)
63 }
64 
65 const DIRECT_KERNEL_BOOT_CMDLINE: &str =
66     "root=/dev/vda1 console=hvc0 rw systemd.journald.forward_to_console=1";
67 
68 // Creates the path for direct kernel boot and return the path.
69 // For x86_64, this function returns the vmlinux kernel path.
70 // For AArch64, this function returns the PE kernel path.
71 fn direct_kernel_boot_path() -> PathBuf {
72     let mut workload_path = dirs::home_dir().unwrap();
73     workload_path.push("workloads");
74 
75     let mut kernel_path = workload_path;
76     #[cfg(target_arch = "x86_64")]
77     kernel_path.push("vmlinux");
78     #[cfg(target_arch = "aarch64")]
79     kernel_path.push("Image");
80 
81     kernel_path
82 }
83 
84 fn remote_command(api_socket: &str, command: &str, arg: Option<&str>) -> bool {
85     let mut cmd = std::process::Command::new(clh_command("ch-remote"));
86     cmd.args([&format!("--api-socket={api_socket}"), command]);
87 
88     if let Some(arg) = arg {
89         cmd.arg(arg);
90     }
91     let output = cmd.output().unwrap();
92     if output.status.success() {
93         true
94     } else {
95         eprintln!(
96             "Error running ch-remote command: {:?}\nstderr: {}",
97             &cmd,
98             String::from_utf8_lossy(&output.stderr)
99         );
100         false
101     }
102 }
103 
104 pub fn performance_net_throughput(control: &PerformanceTestControl) -> f64 {
105     let test_timeout = control.test_timeout;
106     let (rx, bandwidth) = control.net_control.unwrap();
107 
108     let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string());
109     let guest = performance_test_new_guest(Box::new(focal));
110 
111     let num_queues = control.num_queues.unwrap();
112     let queue_size = control.queue_size.unwrap();
113     let net_params = format!(
114         "tap=,mac={},ip={},mask=255.255.255.0,num_queues={},queue_size={}",
115         guest.network.guest_mac, guest.network.host_ip, num_queues, queue_size,
116     );
117 
118     let mut child = GuestCommand::new(&guest)
119         .args(["--cpus", &format!("boot={num_queues}")])
120         .args(["--memory", "size=4G"])
121         .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()])
122         .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE])
123         .default_disks()
124         .args(["--net", net_params.as_str()])
125         .capture_output()
126         .verbosity(VerbosityLevel::Warn)
127         .set_print_cmd(false)
128         .spawn()
129         .unwrap();
130 
131     let r = std::panic::catch_unwind(|| {
132         guest.wait_vm_boot(None).unwrap();
133         measure_virtio_net_throughput(test_timeout, num_queues / 2, &guest, rx, bandwidth).unwrap()
134     });
135 
136     let _ = child.kill();
137     let output = child.wait_with_output().unwrap();
138 
139     match r {
140         Ok(r) => r,
141         Err(e) => {
142             handle_child_output(Err(e), &output);
143             panic!("test failed!");
144         }
145     }
146 }
147 
148 pub fn performance_net_latency(control: &PerformanceTestControl) -> f64 {
149     let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string());
150     let guest = performance_test_new_guest(Box::new(focal));
151 
152     let num_queues = control.num_queues.unwrap();
153     let queue_size = control.queue_size.unwrap();
154     let net_params = format!(
155         "tap=,mac={},ip={},mask=255.255.255.0,num_queues={},queue_size={}",
156         guest.network.guest_mac, guest.network.host_ip, num_queues, queue_size,
157     );
158 
159     let mut child = GuestCommand::new(&guest)
160         .args(["--cpus", &format!("boot={num_queues}")])
161         .args(["--memory", "size=4G"])
162         .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()])
163         .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE])
164         .default_disks()
165         .args(["--net", net_params.as_str()])
166         .capture_output()
167         .verbosity(VerbosityLevel::Warn)
168         .set_print_cmd(false)
169         .spawn()
170         .unwrap();
171 
172     let r = std::panic::catch_unwind(|| {
173         guest.wait_vm_boot(None).unwrap();
174 
175         // 'ethr' tool will measure the latency multiple times with provided test time
176         let latency = measure_virtio_net_latency(&guest, control.test_timeout).unwrap();
177         mean(&latency).unwrap()
178     });
179 
180     let _ = child.kill();
181     let output = child.wait_with_output().unwrap();
182 
183     match r {
184         Ok(r) => r,
185         Err(e) => {
186             handle_child_output(Err(e), &output);
187             panic!("test failed!");
188         }
189     }
190 }
191 
192 fn parse_boot_time_output(output: &[u8]) -> Result<f64, Error> {
193     std::panic::catch_unwind(|| {
194         let l: Vec<String> = String::from_utf8_lossy(output)
195             .lines()
196             .filter(|l| l.contains("Debug I/O port: Kernel code"))
197             .map(|l| l.to_string())
198             .collect();
199 
200         assert_eq!(
201             l.len(),
202             2,
203             "Expecting two matching lines for 'Debug I/O port: Kernel code'"
204         );
205 
206         let time_stamp_kernel_start = {
207             let s = l[0].split("--").collect::<Vec<&str>>();
208             assert_eq!(
209                 s.len(),
210                 2,
211                 "Expecting '--' for the matching line of 'Debug I/O port' output"
212             );
213 
214             // Sample output: "[Debug I/O port: Kernel code 0x40] 0.096537 seconds"
215             assert!(
216                 s[1].contains("0x40"),
217                 "Expecting kernel code '0x40' for 'linux_kernel_start' time stamp output"
218             );
219             let t = s[1].split_whitespace().collect::<Vec<&str>>();
220             assert_eq!(
221                 t.len(),
222                 8,
223                 "Expecting exact '8' words from the 'Debug I/O port' output"
224             );
225             assert!(
226                 t[7].eq("seconds"),
227                 "Expecting 'seconds' as the last word of the 'Debug I/O port' output"
228             );
229 
230             t[6].parse::<f64>().unwrap()
231         };
232 
233         let time_stamp_user_start = {
234             let s = l[1].split("--").collect::<Vec<&str>>();
235             assert_eq!(
236                 s.len(),
237                 2,
238                 "Expecting '--' for the matching line of 'Debug I/O port' output"
239             );
240 
241             // Sample output: "Debug I/O port: Kernel code 0x41] 0.198980 seconds"
242             assert!(
243                 s[1].contains("0x41"),
244                 "Expecting kernel code '0x41' for 'linux_kernel_start' time stamp output"
245             );
246             let t = s[1].split_whitespace().collect::<Vec<&str>>();
247             assert_eq!(
248                 t.len(),
249                 8,
250                 "Expecting exact '8' words from the 'Debug I/O port' output"
251             );
252             assert!(
253                 t[7].eq("seconds"),
254                 "Expecting 'seconds' as the last word of the 'Debug I/O port' output"
255             );
256 
257             t[6].parse::<f64>().unwrap()
258         };
259 
260         time_stamp_user_start - time_stamp_kernel_start
261     })
262     .map_err(|_| {
263         eprintln!(
264             "=============== boot-time output ===============\n\n{}\n\n===========end============\n\n",
265             String::from_utf8_lossy(output)
266         );
267         Error::BootTimeParse
268     })
269 }
270 
271 fn measure_boot_time(cmd: &mut GuestCommand, test_timeout: u32) -> Result<f64, Error> {
272     let mut child = cmd
273         .capture_output()
274         .verbosity(VerbosityLevel::Warn)
275         .set_print_cmd(false)
276         .spawn()
277         .unwrap();
278 
279     thread::sleep(Duration::new(test_timeout as u64, 0));
280     let _ = child.kill();
281     let output = child.wait_with_output().unwrap();
282 
283     parse_boot_time_output(&output.stderr).inspect_err(|_| {
284         eprintln!(
285             "\n\n==== Start child stdout ====\n\n{}\n\n==== End child stdout ====",
286             String::from_utf8_lossy(&output.stdout)
287         );
288         eprintln!(
289             "\n\n==== Start child stderr ====\n\n{}\n\n==== End child stderr ====",
290             String::from_utf8_lossy(&output.stderr)
291         );
292     })
293 }
294 
295 pub fn performance_boot_time(control: &PerformanceTestControl) -> f64 {
296     let r = std::panic::catch_unwind(|| {
297         let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string());
298         let guest = performance_test_new_guest(Box::new(focal));
299         let mut cmd = GuestCommand::new(&guest);
300 
301         let c = cmd
302             .args([
303                 "--cpus",
304                 &format!("boot={}", control.num_boot_vcpus.unwrap_or(1)),
305             ])
306             .args(["--memory", "size=1G"])
307             .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()])
308             .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE])
309             .args(["--console", "off"])
310             .default_disks();
311 
312         measure_boot_time(c, control.test_timeout).unwrap()
313     });
314 
315     match r {
316         Ok(r) => r,
317         Err(_) => {
318             panic!("test failed!");
319         }
320     }
321 }
322 
323 pub fn performance_boot_time_pmem(control: &PerformanceTestControl) -> f64 {
324     let r = std::panic::catch_unwind(|| {
325         let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string());
326         let guest = performance_test_new_guest(Box::new(focal));
327         let mut cmd = GuestCommand::new(&guest);
328         let c = cmd
329             .args([
330                 "--cpus",
331                 &format!("boot={}", control.num_boot_vcpus.unwrap_or(1)),
332             ])
333             .args(["--memory", "size=1G,hugepages=on"])
334             .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()])
335             .args(["--cmdline", "root=/dev/pmem0p1 console=ttyS0 quiet rw"])
336             .args(["--console", "off"])
337             .args([
338                 "--pmem",
339                 format!(
340                     "file={}",
341                     guest.disk_config.disk(DiskType::OperatingSystem).unwrap()
342                 )
343                 .as_str(),
344             ]);
345 
346         measure_boot_time(c, control.test_timeout).unwrap()
347     });
348 
349     match r {
350         Ok(r) => r,
351         Err(_) => {
352             panic!("test failed!");
353         }
354     }
355 }
356 
357 pub fn performance_block_io(control: &PerformanceTestControl) -> f64 {
358     let test_timeout = control.test_timeout;
359     let num_queues = control.num_queues.unwrap();
360     let (fio_ops, bandwidth) = control.fio_control.as_ref().unwrap();
361 
362     let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string());
363     let guest = performance_test_new_guest(Box::new(focal));
364     let api_socket = guest
365         .tmp_dir
366         .as_path()
367         .join("cloud-hypervisor.sock")
368         .to_str()
369         .unwrap()
370         .to_string();
371 
372     let mut child = GuestCommand::new(&guest)
373         .args(["--cpus", &format!("boot={num_queues}")])
374         .args(["--memory", "size=4G"])
375         .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()])
376         .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE])
377         .args([
378             "--disk",
379             format!(
380                 "path={}",
381                 guest.disk_config.disk(DiskType::OperatingSystem).unwrap()
382             )
383             .as_str(),
384             format!(
385                 "path={}",
386                 guest.disk_config.disk(DiskType::CloudInit).unwrap()
387             )
388             .as_str(),
389             format!("path={BLK_IO_TEST_IMG}").as_str(),
390         ])
391         .default_net()
392         .args(["--api-socket", &api_socket])
393         .capture_output()
394         .verbosity(VerbosityLevel::Warn)
395         .set_print_cmd(false)
396         .spawn()
397         .unwrap();
398 
399     let r = std::panic::catch_unwind(|| {
400         guest.wait_vm_boot(None).unwrap();
401 
402         let fio_command = format!(
403             "sudo fio --filename=/dev/vdc --name=test --output-format=json \
404             --direct=1 --bs=4k --ioengine=io_uring --iodepth=64 \
405             --rw={fio_ops} --runtime={test_timeout} --numjobs={num_queues}"
406         );
407         let output = guest
408             .ssh_command(&fio_command)
409             .map_err(InfraError::SshCommand)
410             .unwrap();
411 
412         // Parse fio output
413         if *bandwidth {
414             parse_fio_output(&output, fio_ops, num_queues).unwrap()
415         } else {
416             parse_fio_output_iops(&output, fio_ops, num_queues).unwrap()
417         }
418     });
419 
420     let _ = child.kill();
421     let output = child.wait_with_output().unwrap();
422 
423     match r {
424         Ok(r) => r,
425         Err(e) => {
426             handle_child_output(Err(e), &output);
427             panic!("test failed!");
428         }
429     }
430 }
431 
432 // Parse the event_monitor file based on the format that each event
433 // is followed by a double newline
434 fn parse_event_file(event_file: &str) -> Vec<serde_json::Value> {
435     let content = fs::read(event_file).unwrap();
436     let mut ret = Vec::new();
437     for entry in String::from_utf8_lossy(&content)
438         .trim()
439         .split("\n\n")
440         .collect::<Vec<&str>>()
441     {
442         ret.push(serde_json::from_str(entry).unwrap());
443     }
444     ret
445 }
446 
447 fn parse_restore_time_output(events: &[serde_json::Value]) -> Result<f64, Error> {
448     for entry in events.iter() {
449         if entry["event"].as_str().unwrap() == "restored" {
450             let duration = entry["timestamp"]["secs"].as_u64().unwrap() as f64 * 1_000f64
451                 + entry["timestamp"]["nanos"].as_u64().unwrap() as f64 / 1_000_000f64;
452             return Ok(duration);
453         }
454     }
455     Err(Error::RestoreTimeParse)
456 }
457 
458 fn measure_restore_time(
459     cmd: &mut GuestCommand,
460     event_file: &str,
461     test_timeout: u32,
462 ) -> Result<f64, Error> {
463     let mut child = cmd
464         .capture_output()
465         .verbosity(VerbosityLevel::Warn)
466         .set_print_cmd(false)
467         .spawn()
468         .unwrap();
469 
470     thread::sleep(Duration::new((test_timeout / 2) as u64, 0));
471     let _ = child.kill();
472     let output = child.wait_with_output().unwrap();
473 
474     let json_events = parse_event_file(event_file);
475 
476     parse_restore_time_output(&json_events).inspect_err(|_| {
477         eprintln!(
478             "\n\n==== Start child stdout ====\n\n{}\n\n==== End child stdout ====\
479             \n\n==== Start child stderr ====\n\n{}\n\n==== End child stderr ====",
480             String::from_utf8_lossy(&output.stdout),
481             String::from_utf8_lossy(&output.stderr)
482         )
483     })
484 }
485 
486 pub fn performance_restore_latency(control: &PerformanceTestControl) -> f64 {
487     let r = std::panic::catch_unwind(|| {
488         let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string());
489         let guest = performance_test_new_guest(Box::new(focal));
490         let api_socket_source = String::from(
491             guest
492                 .tmp_dir
493                 .as_path()
494                 .join("cloud-hypervisor.sock")
495                 .to_str()
496                 .unwrap(),
497         );
498 
499         let mut child = GuestCommand::new(&guest)
500             .args(["--api-socket", &api_socket_source])
501             .args([
502                 "--cpus",
503                 &format!("boot={}", control.num_boot_vcpus.unwrap_or(1)),
504             ])
505             .args(["--memory", "size=256M"])
506             .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()])
507             .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE])
508             .args(["--console", "off"])
509             .default_disks()
510             .set_print_cmd(false)
511             .spawn()
512             .unwrap();
513 
514         thread::sleep(Duration::new((control.test_timeout / 2) as u64, 0));
515         let snapshot_dir = String::from(guest.tmp_dir.as_path().join("snapshot").to_str().unwrap());
516         std::fs::create_dir(&snapshot_dir).unwrap();
517         assert!(remote_command(&api_socket_source, "pause", None));
518         assert!(remote_command(
519             &api_socket_source,
520             "snapshot",
521             Some(format!("file://{snapshot_dir}").as_str()),
522         ));
523 
524         let _ = child.kill();
525         child.wait().unwrap();
526 
527         let event_path = String::from(guest.tmp_dir.as_path().join("event.json").to_str().unwrap());
528         let mut cmd = GuestCommand::new(&guest);
529         let c = cmd
530             .args([
531                 "--restore",
532                 format!("source_url=file://{snapshot_dir}").as_str(),
533             ])
534             .args(["--event-monitor", format!("path={event_path}").as_str()]);
535 
536         measure_restore_time(c, event_path.as_str(), control.test_timeout).unwrap()
537     });
538 
539     match r {
540         Ok(r) => r,
541         Err(_) => {
542             panic!("test failed!");
543         }
544     }
545 }
546 
547 #[cfg(test)]
548 mod tests {
549     use super::*;
550 
551     #[test]
552     fn test_parse_iperf3_output() {
553         let output = r#"
554 {
555 	"end":	{
556 		"sum_sent":	{
557 			"start":	0,
558 			"end":	5.000196,
559 			"seconds":	5.000196,
560 			"bytes":	14973836248,
561 			"bits_per_second":	23957198874.604115,
562 			"retransmits":	0,
563 			"sender":	false
564 		}
565 	}
566 }
567        "#;
568         assert_eq!(
569             parse_iperf3_output(output.as_bytes(), true, true).unwrap(),
570             23957198874.604115
571         );
572 
573         let output = r#"
574 {
575 	"end":	{
576 		"sum_received":	{
577 			"start":	0,
578 			"end":	5.000626,
579 			"seconds":	5.000626,
580 			"bytes":	24703557800,
581 			"bits_per_second":	39520744482.79,
582 			"sender":	true
583 		}
584 	}
585 }
586               "#;
587         assert_eq!(
588             parse_iperf3_output(output.as_bytes(), false, true).unwrap(),
589             39520744482.79
590         );
591         let output = r#"
592 {
593     "end":	{
594         "sum":  {
595             "start":        0,
596             "end":  5.000036,
597             "seconds":      5.000036,
598             "bytes":        29944971264,
599             "bits_per_second":      47911877363.396217,
600             "jitter_ms":    0.0038609822983198556,
601             "lost_packets": 16,
602             "packets":      913848,
603             "lost_percent": 0.0017508382137948542,
604             "sender":       true
605         }
606     }
607 }
608               "#;
609         assert_eq!(
610             parse_iperf3_output(output.as_bytes(), true, false).unwrap(),
611             182765.08409139456
612         );
613     }
614 
615     #[test]
616     fn test_parse_ethr_latency_output() {
617         let output = r#"{"Time":"2022-02-08T03:52:50Z","Title":"","Type":"INFO","Message":"Using destination: 192.168.249.2, ip: 192.168.249.2, port: 8888"}
618 {"Time":"2022-02-08T03:52:51Z","Title":"","Type":"INFO","Message":"Running latency test: 1000, 1"}
619 {"Time":"2022-02-08T03:52:51Z","Title":"","Type":"LatencyResult","RemoteAddr":"192.168.249.2","Protocol":"TCP","Avg":"80.712us","Min":"61.677us","P50":"257.014us","P90":"74.418us","P95":"107.283us","P99":"119.309us","P999":"142.100us","P9999":"216.341us","Max":"216.341us"}
620 {"Time":"2022-02-08T03:52:52Z","Title":"","Type":"LatencyResult","RemoteAddr":"192.168.249.2","Protocol":"TCP","Avg":"79.826us","Min":"55.129us","P50":"598.996us","P90":"73.849us","P95":"106.552us","P99":"122.152us","P999":"142.459us","P9999":"474.280us","Max":"474.280us"}
621 {"Time":"2022-02-08T03:52:53Z","Title":"","Type":"LatencyResult","RemoteAddr":"192.168.249.2","Protocol":"TCP","Avg":"78.239us","Min":"56.999us","P50":"396.820us","P90":"69.469us","P95":"115.421us","P99":"119.404us","P999":"130.158us","P9999":"258.686us","Max":"258.686us"}"#;
622 
623         let ret = parse_ethr_latency_output(output.as_bytes()).unwrap();
624         let reference = vec![80.712_f64, 79.826_f64, 78.239_f64];
625         assert_eq!(ret, reference);
626     }
627 
628     #[test]
629     fn test_parse_boot_time_output() {
630         let output = r#"
631 cloud-hypervisor: 161.167103ms: <vcpu0> INFO:vmm/src/vm.rs:392 -- [Debug I/O port: Kernel code 0x40] 0.132 seconds
632 cloud-hypervisor: 613.57361ms: <vcpu0> INFO:vmm/src/vm.rs:392 -- [Debug I/O port: Kernel code 0x41] 0.5845 seconds
633         "#;
634 
635         assert_eq!(parse_boot_time_output(output.as_bytes()).unwrap(), 0.4525);
636     }
637     #[test]
638     fn test_parse_restore_time_output() {
639         let output = r#"
640 {
641   "timestamp": {
642     "secs": 0,
643     "nanos": 4664404
644   },
645   "source": "virtio-device",
646   "event": "activated",
647   "properties": {
648     "id": "__rng"
649   }
650 }
651 
652 {
653   "timestamp": {
654     "secs": 0,
655     "nanos": 5505133
656   },
657   "source": "vm",
658   "event": "restored",
659   "properties": null
660 }
661 "#;
662         let mut ret = Vec::new();
663         for entry in String::from(output)
664             .trim()
665             .split("\n\n")
666             .collect::<Vec<&str>>()
667         {
668             ret.push(serde_json::from_str(entry).unwrap());
669         }
670 
671         assert_eq!(parse_restore_time_output(&ret).unwrap(), 5.505133_f64);
672     }
673     #[test]
674     fn test_parse_fio_output() {
675         let output = r#"
676 {
677   "jobs" : [
678     {
679       "read" : {
680         "io_bytes" : 1965273088,
681         "io_kbytes" : 1919212,
682         "bw_bytes" : 392976022,
683         "bw" : 383765,
684         "iops" : 95941.411718,
685         "runtime" : 5001,
686         "total_ios" : 479803,
687         "short_ios" : 0,
688         "drop_ios" : 0
689       }
690     }
691   ]
692 }
693 "#;
694 
695         let bps = 1965273088_f64 / (5001_f64 / 1000_f64);
696         assert_eq!(
697             parse_fio_output(output, &FioOps::RandomRead, 1).unwrap(),
698             bps
699         );
700         assert_eq!(parse_fio_output(output, &FioOps::Read, 1).unwrap(), bps);
701 
702         let output = r#"
703 {
704   "jobs" : [
705     {
706       "write" : {
707         "io_bytes" : 1172783104,
708         "io_kbytes" : 1145296,
709         "bw_bytes" : 234462835,
710         "bw" : 228967,
711         "iops" : 57241.903239,
712         "runtime" : 5002,
713         "total_ios" : 286324,
714         "short_ios" : 0,
715         "drop_ios" : 0
716       }
717     },
718     {
719       "write" : {
720         "io_bytes" : 1172234240,
721         "io_kbytes" : 1144760,
722         "bw_bytes" : 234353106,
723         "bw" : 228860,
724         "iops" : 57215.113954,
725         "runtime" : 5002,
726         "total_ios" : 286190,
727         "short_ios" : 0,
728         "drop_ios" : 0
729       }
730     }
731   ]
732 }
733 "#;
734 
735         let bps = 1172783104_f64 / (5002_f64 / 1000_f64) + 1172234240_f64 / (5002_f64 / 1000_f64);
736         assert_eq!(
737             parse_fio_output(output, &FioOps::RandomWrite, 2).unwrap(),
738             bps
739         );
740         assert_eq!(parse_fio_output(output, &FioOps::Write, 2).unwrap(), bps);
741     }
742 }
743