xref: /cloud-hypervisor/performance-metrics/src/performance_tests.rs (revision b440cb7d2330770cd415b63544a371d4caa2db3a)
1 // Copyright © 2022 Intel Corporation
2 //
3 // SPDX-License-Identifier: Apache-2.0
4 //
5 
6 // Performance tests
7 
8 use crate::{mean, PerformanceTestControl};
9 use serde_json::Value;
10 use std::path::{Path, PathBuf};
11 use std::process::{Child, Command, Stdio};
12 use std::string::String;
13 use std::thread;
14 use std::time::Duration;
15 use std::{fmt, fs};
16 use test_infra::Error as InfraError;
17 use test_infra::*;
18 use wait_timeout::ChildExt;
19 
20 #[cfg(target_arch = "x86_64")]
21 pub const FOCAL_IMAGE_NAME: &str = "focal-server-cloudimg-amd64-custom-20210609-0.raw";
22 #[cfg(target_arch = "aarch64")]
23 pub const FOCAL_IMAGE_NAME: &str = "focal-server-cloudimg-arm64-custom-20210929-0-update-tool.raw";
24 
25 #[derive(Debug)]
26 enum WaitTimeoutError {
27     Timedout,
28     ExitStatus,
29     General(std::io::Error),
30 }
31 
32 #[derive(Debug)]
33 enum Error {
34     BootTimeParse,
35     EthrLogFile(std::io::Error),
36     EthrLogParse,
37     FioOutputParse,
38     Iperf3Parse,
39     Infra(InfraError),
40     Spawn(std::io::Error),
41     Scp(SshCommandError),
42     WaitTimeout(WaitTimeoutError),
43 }
44 
45 impl From<InfraError> for Error {
46     fn from(e: InfraError) -> Self {
47         Self::Infra(e)
48     }
49 }
50 
51 const BLK_IO_TEST_IMG: &str = "/var/tmp/ch-blk-io-test.img";
52 
53 pub fn init_tests() {
54     // The test image can not be created on tmpfs (e.g. /tmp) filesystem,
55     // as tmpfs does not support O_DIRECT
56     assert!(exec_host_command_output(&format!(
57         "dd if=/dev/zero of={} bs=1M count=4096",
58         BLK_IO_TEST_IMG
59     ))
60     .status
61     .success());
62 }
63 
64 pub fn cleanup_tests() {
65     fs::remove_file(BLK_IO_TEST_IMG)
66         .unwrap_or_else(|_| panic!("Failed to remove file '{}'.", BLK_IO_TEST_IMG));
67 }
68 
69 // Performance tests are expected to be executed sequentially, so we can
70 // start VM guests with the same IP while putting them on a different
71 // private network. The default constructor "Guest::new()" does not work
72 // well, as we can easily create more than 256 VMs from repeating various
73 // performance tests dozens times in a single run.
74 fn performance_test_new_guest(disk_config: Box<dyn DiskConfig>) -> Guest {
75     Guest::new_from_ip_range(disk_config, "172.19", 0)
76 }
77 
78 const DIRECT_KERNEL_BOOT_CMDLINE: &str =
79     "root=/dev/vda1 console=hvc0 rw systemd.journald.forward_to_console=1";
80 
81 // Creates the path for direct kernel boot and return the path.
82 // For x86_64, this function returns the vmlinux kernel path.
83 // For AArch64, this function returns the PE kernel path.
84 fn direct_kernel_boot_path() -> PathBuf {
85     let mut workload_path = dirs::home_dir().unwrap();
86     workload_path.push("workloads");
87 
88     let mut kernel_path = workload_path;
89     #[cfg(target_arch = "x86_64")]
90     kernel_path.push("vmlinux");
91     #[cfg(target_arch = "aarch64")]
92     kernel_path.push("Image");
93 
94     kernel_path
95 }
96 
97 // Wait the child process for a given timeout
98 fn child_wait_timeout(child: &mut Child, timeout: u64) -> Result<(), WaitTimeoutError> {
99     match child.wait_timeout(Duration::from_secs(timeout)) {
100         Err(e) => {
101             return Err(WaitTimeoutError::General(e));
102         }
103         Ok(s) => match s {
104             None => {
105                 return Err(WaitTimeoutError::Timedout);
106             }
107             Some(s) => {
108                 if !s.success() {
109                     return Err(WaitTimeoutError::ExitStatus);
110                 }
111             }
112         },
113     }
114 
115     Ok(())
116 }
117 
118 fn parse_iperf3_output(output: &[u8], sender: bool) -> Result<f64, Error> {
119     std::panic::catch_unwind(|| {
120         let s = String::from_utf8_lossy(output);
121         let v: Value = serde_json::from_str(&s).expect("'iperf3' parse error: invalid json output");
122 
123         let bps: f64 = if sender {
124             v["end"]["sum_sent"]["bits_per_second"]
125                 .as_f64()
126                 .expect("'iperf3' parse error: missing entry 'end.sum_sent.bits_per_second'")
127         } else {
128             v["end"]["sum_received"]["bits_per_second"]
129                 .as_f64()
130                 .expect("'iperf3' parse error: missing entry 'end.sum_received.bits_per_second'")
131         };
132 
133         bps
134     })
135     .map_err(|_| {
136         eprintln!(
137             "=============== iperf3 output ===============\n\n{}\n\n===========end============\n\n",
138             String::from_utf8_lossy(output)
139         );
140         Error::Iperf3Parse
141     })
142 }
143 
144 fn measure_virtio_net_throughput(
145     test_timeout: u32,
146     queue_pairs: u32,
147     guest: &Guest,
148     receive: bool,
149 ) -> Result<f64, Error> {
150     let default_port = 5201;
151 
152     // 1. start the iperf3 server on the guest
153     for n in 0..queue_pairs {
154         guest
155             .ssh_command(&format!("iperf3 -s -p {} -D", default_port + n))
156             .map_err(InfraError::SshCommand)?;
157     }
158 
159     thread::sleep(Duration::new(1, 0));
160 
161     // 2. start the iperf3 client on host to measure RX through-put
162     let mut clients = Vec::new();
163     for n in 0..queue_pairs {
164         let mut cmd = Command::new("iperf3");
165         cmd.args(&[
166             "-J", // Output in JSON format
167             "-c",
168             &guest.network.guest_ip,
169             "-p",
170             &format!("{}", default_port + n),
171             "-t",
172             &format!("{}", test_timeout),
173         ]);
174         // For measuring the guest transmit throughput (as a sender),
175         // use reverse mode of the iperf3 client on the host
176         if !receive {
177             cmd.args(&["-R"]);
178         }
179         let client = cmd
180             .stderr(Stdio::piped())
181             .stdout(Stdio::piped())
182             .spawn()
183             .map_err(Error::Spawn)?;
184 
185         clients.push(client);
186     }
187 
188     let mut err: Option<Error> = None;
189     let mut bps = Vec::new();
190     let mut failed = false;
191     for c in clients {
192         let mut c = c;
193         if let Err(e) = child_wait_timeout(&mut c, test_timeout as u64 + 5) {
194             err = Some(Error::WaitTimeout(e));
195             failed = true;
196         }
197 
198         if !failed {
199             // Safe to unwrap as we know the child has terminated succesffully
200             let output = c.wait_with_output().unwrap();
201             bps.push(parse_iperf3_output(&output.stdout, receive)?);
202         } else {
203             let _ = c.kill();
204             let output = c.wait_with_output().unwrap();
205             println!(
206                 "=============== Client output [Error] ===============\n\n{}\n\n===========end============\n\n",
207                 String::from_utf8_lossy(&output.stdout)
208             );
209         }
210     }
211 
212     if let Some(e) = err {
213         Err(e)
214     } else {
215         Ok(bps.iter().sum())
216     }
217 }
218 
219 pub fn performance_net_throughput(control: &PerformanceTestControl) -> f64 {
220     let test_timeout = control.test_timeout;
221     let rx = control.net_rx.unwrap();
222 
223     let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string());
224     let guest = performance_test_new_guest(Box::new(focal));
225 
226     let num_queues = control.num_queues.unwrap();
227     let queue_size = control.queue_size.unwrap();
228     let net_params = format!(
229         "tap=,mac={},ip={},mask=255.255.255.0,num_queues={},queue_size={}",
230         guest.network.guest_mac, guest.network.host_ip, num_queues, queue_size,
231     );
232 
233     let mut child = GuestCommand::new(&guest)
234         .args(&["--cpus", &format!("boot={}", num_queues)])
235         .args(&["--memory", "size=4G"])
236         .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()])
237         .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE])
238         .default_disks()
239         .args(&["--net", net_params.as_str()])
240         .capture_output()
241         .verbosity(VerbosityLevel::Warn)
242         .set_print_cmd(false)
243         .spawn()
244         .unwrap();
245 
246     let r = std::panic::catch_unwind(|| {
247         guest.wait_vm_boot(None).unwrap();
248         measure_virtio_net_throughput(test_timeout, num_queues / 2, &guest, rx).unwrap()
249     });
250 
251     let _ = child.kill();
252     let output = child.wait_with_output().unwrap();
253 
254     match r {
255         Ok(r) => r,
256         Err(e) => {
257             handle_child_output(Err(e), &output);
258             panic!("test failed!");
259         }
260     }
261 }
262 
263 fn parse_ethr_latency_output(output: &[u8]) -> Result<Vec<f64>, Error> {
264     std::panic::catch_unwind(|| {
265         let s = String::from_utf8_lossy(output);
266         let mut latency = Vec::new();
267         for l in s.lines() {
268             let v: Value = serde_json::from_str(l).expect("'ethr' parse error: invalid json line");
269             // Skip header/summary lines
270             if let Some(avg) = v["Avg"].as_str() {
271                 // Assume the latency unit is always "us"
272                 latency.push(
273                     avg.split("us").collect::<Vec<&str>>()[0]
274                         .parse::<f64>()
275                         .expect("'ethr' parse error: invalid 'Avg' entry"),
276                 );
277             }
278         }
279 
280         assert!(
281             !latency.is_empty(),
282             "'ethr' parse error: no valid latency data found"
283         );
284 
285         latency
286     })
287     .map_err(|_| {
288         eprintln!(
289             "=============== ethr output ===============\n\n{}\n\n===========end============\n\n",
290             String::from_utf8_lossy(output)
291         );
292         Error::EthrLogParse
293     })
294 }
295 
296 fn measure_virtio_net_latency(guest: &Guest, test_timeout: u32) -> Result<Vec<f64>, Error> {
297     // copy the 'ethr' tool to the guest image
298     let ethr_path = "/usr/local/bin/ethr";
299     let ethr_remote_path = "/tmp/ethr";
300     scp_to_guest(
301         Path::new(ethr_path),
302         Path::new(ethr_remote_path),
303         &guest.network.guest_ip,
304         //DEFAULT_SSH_RETRIES,
305         1,
306         DEFAULT_SSH_TIMEOUT,
307     )
308     .map_err(Error::Scp)?;
309 
310     // Start the ethr server on the guest
311     guest
312         .ssh_command(&format!("{} -s &> /dev/null &", ethr_remote_path))
313         .map_err(InfraError::SshCommand)?;
314 
315     thread::sleep(Duration::new(1, 0));
316 
317     // Start the ethr client on the host
318     let log_file = guest
319         .tmp_dir
320         .as_path()
321         .join("ethr.client.log")
322         .to_str()
323         .unwrap()
324         .to_string();
325     let mut c = Command::new(ethr_path)
326         .args(&[
327             "-c",
328             &guest.network.guest_ip,
329             "-t",
330             "l",
331             "-o",
332             &log_file, // file output is JSON format
333             "-d",
334             &format!("{}s", test_timeout),
335         ])
336         .stderr(Stdio::piped())
337         .stdout(Stdio::piped())
338         .spawn()
339         .map_err(Error::Spawn)?;
340 
341     if let Err(e) = child_wait_timeout(&mut c, test_timeout as u64 + 5).map_err(Error::WaitTimeout)
342     {
343         let _ = c.kill();
344         return Err(e);
345     }
346 
347     // Parse the ethr latency test output
348     let content = fs::read(log_file).map_err(Error::EthrLogFile)?;
349     parse_ethr_latency_output(&content)
350 }
351 
352 pub fn performance_net_latency(control: &PerformanceTestControl) -> f64 {
353     let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string());
354     let guest = performance_test_new_guest(Box::new(focal));
355 
356     let num_queues = control.num_queues.unwrap();
357     let queue_size = control.queue_size.unwrap();
358     let net_params = format!(
359         "tap=,mac={},ip={},mask=255.255.255.0,num_queues={},queue_size={}",
360         guest.network.guest_mac, guest.network.host_ip, num_queues, queue_size,
361     );
362 
363     let mut child = GuestCommand::new(&guest)
364         .args(&["--cpus", &format!("boot={}", num_queues)])
365         .args(&["--memory", "size=4G"])
366         .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()])
367         .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE])
368         .default_disks()
369         .args(&["--net", net_params.as_str()])
370         .capture_output()
371         .verbosity(VerbosityLevel::Warn)
372         .set_print_cmd(false)
373         .spawn()
374         .unwrap();
375 
376     let r = std::panic::catch_unwind(|| {
377         guest.wait_vm_boot(None).unwrap();
378 
379         // 'ethr' tool will measure the latency multiple times with provided test time
380         let latency = measure_virtio_net_latency(&guest, control.test_timeout).unwrap();
381         mean(&latency).unwrap()
382     });
383 
384     let _ = child.kill();
385     let output = child.wait_with_output().unwrap();
386 
387     match r {
388         Ok(r) => r,
389         Err(e) => {
390             handle_child_output(Err(e), &output);
391             panic!("test failed!");
392         }
393     }
394 }
395 
396 fn parse_boot_time_output(output: &[u8]) -> Result<f64, Error> {
397     std::panic::catch_unwind(|| {
398         let l: Vec<String> = String::from_utf8_lossy(output)
399             .lines()
400             .into_iter()
401             .filter(|l| l.contains("Debug I/O port: Kernel code"))
402             .map(|l| l.to_string())
403             .collect();
404 
405         assert_eq!(
406             l.len(),
407             2,
408             "Expecting two matching lines for 'Debug I/O port: Kernel code'"
409         );
410 
411         let time_stamp_kernel_start = {
412             let s = l[0].split("--").collect::<Vec<&str>>();
413             assert_eq!(
414                 s.len(),
415                 2,
416                 "Expecting '--' for the matching line of 'Debug I/O port' output"
417             );
418 
419             // Sample output: "[Debug I/O port: Kernel code 0x40] 0.096537 seconds"
420             assert!(
421                 s[1].contains("0x40"),
422                 "Expecting kernel code '0x40' for 'linux_kernel_start' time stamp output"
423             );
424             let t = s[1].split_whitespace().collect::<Vec<&str>>();
425             assert_eq!(
426                 t.len(),
427                 8,
428                 "Expecting exact '8' words from the 'Debug I/O port' output"
429             );
430             assert!(
431                 t[7].eq("seconds"),
432                 "Expecting 'seconds' as the the last word of the 'Debug I/O port' output"
433             );
434 
435             t[6].parse::<f64>().unwrap()
436         };
437 
438         let time_stamp_user_start = {
439             let s = l[1].split("--").collect::<Vec<&str>>();
440             assert_eq!(
441                 s.len(),
442                 2,
443                 "Expecting '--' for the matching line of 'Debug I/O port' output"
444             );
445 
446             // Sample output: "Debug I/O port: Kernel code 0x41] 0.198980 seconds"
447             assert!(
448                 s[1].contains("0x41"),
449                 "Expecting kernel code '0x41' for 'linux_kernel_start' time stamp output"
450             );
451             let t = s[1].split_whitespace().collect::<Vec<&str>>();
452             assert_eq!(
453                 t.len(),
454                 8,
455                 "Expecting exact '8' words from the 'Debug I/O port' output"
456             );
457             assert!(
458                 t[7].eq("seconds"),
459                 "Expecting 'seconds' as the the last word of the 'Debug I/O port' output"
460             );
461 
462             t[6].parse::<f64>().unwrap()
463         };
464 
465         time_stamp_user_start - time_stamp_kernel_start
466     })
467     .map_err(|_| {
468         eprintln!(
469             "=============== boot-time output ===============\n\n{}\n\n===========end============\n\n",
470             String::from_utf8_lossy(output)
471         );
472         Error::BootTimeParse
473     })
474 }
475 
476 fn measure_boot_time(cmd: &mut GuestCommand, test_timeout: u32) -> Result<f64, Error> {
477     let mut child = cmd
478         .capture_output()
479         .verbosity(VerbosityLevel::Warn)
480         .set_print_cmd(false)
481         .spawn()
482         .unwrap();
483 
484     thread::sleep(Duration::new(test_timeout as u64, 0));
485     let _ = child.kill();
486     let output = child.wait_with_output().unwrap();
487 
488     parse_boot_time_output(&output.stderr).map_err(|e| {
489         eprintln!(
490             "\n\n==== Start child stdout ====\n\n{}\n\n==== End child stdout ====",
491             String::from_utf8_lossy(&output.stdout)
492         );
493         eprintln!(
494             "\n\n==== Start child stderr ====\n\n{}\n\n==== End child stderr ====",
495             String::from_utf8_lossy(&output.stderr)
496         );
497 
498         e
499     })
500 }
501 
502 pub fn performance_boot_time(control: &PerformanceTestControl) -> f64 {
503     let r = std::panic::catch_unwind(|| {
504         let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string());
505         let guest = performance_test_new_guest(Box::new(focal));
506         let mut cmd = GuestCommand::new(&guest);
507 
508         let c = cmd
509             .args(&[
510                 "--cpus",
511                 &format!("boot={}", control.num_boot_vcpus.unwrap_or(1)),
512             ])
513             .args(&["--memory", "size=1G"])
514             .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()])
515             .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE])
516             .args(&["--console", "off"])
517             .default_disks();
518 
519         measure_boot_time(c, control.test_timeout).unwrap()
520     });
521 
522     match r {
523         Ok(r) => r,
524         Err(_) => {
525             panic!("test failed!");
526         }
527     }
528 }
529 
530 pub fn performance_boot_time_pmem(control: &PerformanceTestControl) -> f64 {
531     let r = std::panic::catch_unwind(|| {
532         let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string());
533         let guest = performance_test_new_guest(Box::new(focal));
534         let mut cmd = GuestCommand::new(&guest);
535         let c = cmd
536             .args(&[
537                 "--cpus",
538                 &format!("boot={}", control.num_boot_vcpus.unwrap_or(1)),
539             ])
540             .args(&["--memory", "size=1G,hugepages=on"])
541             .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()])
542             .args(&["--cmdline", "root=/dev/pmem0p1 console=ttyS0 quiet rw"])
543             .args(&["--console", "off"])
544             .args(&[
545                 "--pmem",
546                 format!(
547                     "file={}",
548                     guest.disk_config.disk(DiskType::OperatingSystem).unwrap()
549                 )
550                 .as_str(),
551             ]);
552 
553         measure_boot_time(c, control.test_timeout).unwrap()
554     });
555 
556     match r {
557         Ok(r) => r,
558         Err(_) => {
559             panic!("test failed!");
560         }
561     }
562 }
563 
564 pub enum FioOps {
565     Read,
566     RandomRead,
567     Write,
568     RandomWrite,
569 }
570 
571 impl fmt::Display for FioOps {
572     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
573         match self {
574             FioOps::Read => write!(f, "read"),
575             FioOps::RandomRead => write!(f, "randread"),
576             FioOps::Write => write!(f, "write"),
577             FioOps::RandomWrite => write!(f, "randwrite"),
578         }
579     }
580 }
581 
582 fn parse_fio_output(output: &str, fio_ops: &FioOps, num_jobs: u32) -> Result<f64, Error> {
583     std::panic::catch_unwind(|| {
584         let v: Value =
585             serde_json::from_str(output).expect("'fio' parse error: invalid json output");
586         let jobs = v["jobs"]
587             .as_array()
588             .expect("'fio' parse error: missing entry 'jobs'");
589         assert_eq!(
590             jobs.len(),
591             num_jobs as usize,
592             "'fio' parse error: Unexpected number of 'fio' jobs."
593         );
594 
595         let read = match fio_ops {
596             FioOps::Read | FioOps::RandomRead => true,
597             FioOps::Write | FioOps::RandomWrite => false,
598         };
599 
600         let mut total_bps = 0_f64;
601         for j in jobs {
602             if read {
603                 let bytes = j["read"]["io_bytes"]
604                     .as_u64()
605                     .expect("'fio' parse error: missing entry 'read.io_bytes'");
606                 let runtime = j["read"]["runtime"]
607                     .as_u64()
608                     .expect("'fio' parse error: missing entry 'read.runtime'")
609                     as f64
610                     / 1000_f64;
611                 total_bps += bytes as f64 / runtime as f64;
612             } else {
613                 let bytes = j["write"]["io_bytes"]
614                     .as_u64()
615                     .expect("'fio' parse error: missing entry 'write.io_bytes'");
616                 let runtime = j["write"]["runtime"]
617                     .as_u64()
618                     .expect("'fio' parse error: missing entry 'write.runtime'")
619                     as f64
620                     / 1000_f64;
621                 total_bps += bytes as f64 / runtime as f64;
622             }
623         }
624 
625         total_bps
626     })
627     .map_err(|_| {
628         eprintln!(
629             "=============== Fio output ===============\n\n{}\n\n===========end============\n\n",
630             output
631         );
632         Error::FioOutputParse
633     })
634 }
635 
636 pub fn performance_block_io(control: &PerformanceTestControl) -> f64 {
637     let test_timeout = control.test_timeout;
638     let num_queues = control.num_queues.unwrap();
639     let fio_ops = control.fio_ops.as_ref().unwrap();
640 
641     let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string());
642     let guest = performance_test_new_guest(Box::new(focal));
643     let api_socket = guest
644         .tmp_dir
645         .as_path()
646         .join("cloud-hypervisor.sock")
647         .to_str()
648         .unwrap()
649         .to_string();
650 
651     let mut child = GuestCommand::new(&guest)
652         .args(&["--cpus", &format!("boot={}", num_queues)])
653         .args(&["--memory", "size=4G"])
654         .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()])
655         .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE])
656         .args(&[
657             "--disk",
658             format!(
659                 "path={}",
660                 guest.disk_config.disk(DiskType::OperatingSystem).unwrap()
661             )
662             .as_str(),
663             format!(
664                 "path={}",
665                 guest.disk_config.disk(DiskType::CloudInit).unwrap()
666             )
667             .as_str(),
668             format!("path={}", BLK_IO_TEST_IMG).as_str(),
669         ])
670         .default_net()
671         .args(&["--api-socket", &api_socket])
672         .capture_output()
673         .verbosity(VerbosityLevel::Warn)
674         .set_print_cmd(false)
675         .spawn()
676         .unwrap();
677 
678     let r = std::panic::catch_unwind(|| {
679         guest.wait_vm_boot(None).unwrap();
680 
681         let fio_command = format!(
682             "sudo fio --filename=/dev/vdc --name=test --output-format=json \
683             --direct=1 --bs=4k --ioengine=io_uring --iodepth=64 \
684             --rw={} --runtime={} --numjobs={}",
685             fio_ops, test_timeout, num_queues
686         );
687         let output = guest
688             .ssh_command(&fio_command)
689             .map_err(InfraError::SshCommand)
690             .unwrap();
691 
692         // Parse fio output
693         parse_fio_output(&output, fio_ops, num_queues).unwrap()
694     });
695 
696     let _ = child.kill();
697     let output = child.wait_with_output().unwrap();
698 
699     match r {
700         Ok(r) => r,
701         Err(e) => {
702             handle_child_output(Err(e), &output);
703             panic!("test failed!");
704         }
705     }
706 }
707 
708 #[cfg(test)]
709 mod tests {
710     use super::*;
711 
712     #[test]
713     fn test_parse_iperf3_output() {
714         let output = r#"
715 {
716 	"end":	{
717 		"sum_sent":	{
718 			"start":	0,
719 			"end":	5.000196,
720 			"seconds":	5.000196,
721 			"bytes":	14973836248,
722 			"bits_per_second":	23957198874.604115,
723 			"retransmits":	0,
724 			"sender":	false
725 		}
726 	}
727 }
728        "#;
729         assert_eq!(
730             parse_iperf3_output(output.as_bytes(), true).unwrap(),
731             23957198874.604115
732         );
733 
734         let output = r#"
735 {
736 	"end":	{
737 		"sum_received":	{
738 			"start":	0,
739 			"end":	5.000626,
740 			"seconds":	5.000626,
741 			"bytes":	24703557800,
742 			"bits_per_second":	39520744482.79,
743 			"sender":	true
744 		}
745 	}
746 }
747               "#;
748         assert_eq!(
749             parse_iperf3_output(output.as_bytes(), false).unwrap(),
750             39520744482.79
751         );
752     }
753 
754     #[test]
755     fn test_parse_ethr_latency_output() {
756         let output = r#"{"Time":"2022-02-08T03:52:50Z","Title":"","Type":"INFO","Message":"Using destination: 192.168.249.2, ip: 192.168.249.2, port: 8888"}
757 {"Time":"2022-02-08T03:52:51Z","Title":"","Type":"INFO","Message":"Running latency test: 1000, 1"}
758 {"Time":"2022-02-08T03:52:51Z","Title":"","Type":"LatencyResult","RemoteAddr":"192.168.249.2","Protocol":"TCP","Avg":"80.712us","Min":"61.677us","P50":"257.014us","P90":"74.418us","P95":"107.283us","P99":"119.309us","P999":"142.100us","P9999":"216.341us","Max":"216.341us"}
759 {"Time":"2022-02-08T03:52:52Z","Title":"","Type":"LatencyResult","RemoteAddr":"192.168.249.2","Protocol":"TCP","Avg":"79.826us","Min":"55.129us","P50":"598.996us","P90":"73.849us","P95":"106.552us","P99":"122.152us","P999":"142.459us","P9999":"474.280us","Max":"474.280us"}
760 {"Time":"2022-02-08T03:52:53Z","Title":"","Type":"LatencyResult","RemoteAddr":"192.168.249.2","Protocol":"TCP","Avg":"78.239us","Min":"56.999us","P50":"396.820us","P90":"69.469us","P95":"115.421us","P99":"119.404us","P999":"130.158us","P9999":"258.686us","Max":"258.686us"}"#;
761 
762         let ret = parse_ethr_latency_output(output.as_bytes()).unwrap();
763         let reference = vec![80.712_f64, 79.826_f64, 78.239_f64];
764         assert_eq!(ret, reference);
765     }
766 
767     #[test]
768     fn test_parse_boot_time_output() {
769         let output = r#"
770 cloud-hypervisor: 161.167103ms: <vcpu0> INFO:vmm/src/vm.rs:392 -- [Debug I/O port: Kernel code 0x40] 0.132 seconds
771 cloud-hypervisor: 613.57361ms: <vcpu0> INFO:vmm/src/vm.rs:392 -- [Debug I/O port: Kernel code 0x41] 0.5845 seconds
772         "#;
773 
774         assert_eq!(parse_boot_time_output(output.as_bytes()).unwrap(), 0.4525);
775     }
776 
777     #[test]
778     fn test_parse_fio_output() {
779         let output = r#"
780 {
781   "jobs" : [
782     {
783       "read" : {
784         "io_bytes" : 1965273088,
785         "io_kbytes" : 1919212,
786         "bw_bytes" : 392976022,
787         "bw" : 383765,
788         "iops" : 95941.411718,
789         "runtime" : 5001,
790         "total_ios" : 479803,
791         "short_ios" : 0,
792         "drop_ios" : 0
793       }
794     }
795   ]
796 }
797 "#;
798 
799         let bps = 1965273088_f64 / (5001_f64 / 1000_f64);
800         assert_eq!(
801             parse_fio_output(output, &FioOps::RandomRead, 1).unwrap(),
802             bps
803         );
804         assert_eq!(parse_fio_output(output, &FioOps::Read, 1).unwrap(), bps);
805 
806         let output = r#"
807 {
808   "jobs" : [
809     {
810       "write" : {
811         "io_bytes" : 1172783104,
812         "io_kbytes" : 1145296,
813         "bw_bytes" : 234462835,
814         "bw" : 228967,
815         "iops" : 57241.903239,
816         "runtime" : 5002,
817         "total_ios" : 286324,
818         "short_ios" : 0,
819         "drop_ios" : 0
820       }
821     },
822     {
823       "write" : {
824         "io_bytes" : 1172234240,
825         "io_kbytes" : 1144760,
826         "bw_bytes" : 234353106,
827         "bw" : 228860,
828         "iops" : 57215.113954,
829         "runtime" : 5002,
830         "total_ios" : 286190,
831         "short_ios" : 0,
832         "drop_ios" : 0
833       }
834     }
835   ]
836 }
837 "#;
838 
839         let bps = 1172783104_f64 / (5002_f64 / 1000_f64) + 1172234240_f64 / (5002_f64 / 1000_f64);
840         assert_eq!(
841             parse_fio_output(output, &FioOps::RandomWrite, 2).unwrap(),
842             bps
843         );
844         assert_eq!(parse_fio_output(output, &FioOps::Write, 2).unwrap(), bps);
845     }
846 }
847