xref: /cloud-hypervisor/performance-metrics/src/performance_tests.rs (revision 7d7bfb2034001d4cb15df2ddc56d2d350c8da30f)
1 // Copyright © 2022 Intel Corporation
2 //
3 // SPDX-License-Identifier: Apache-2.0
4 //
5 
6 // Performance tests
7 
8 use crate::{mean, PerformanceTestControl};
9 use serde_json::Value;
10 use std::path::{Path, PathBuf};
11 use std::process::{Child, Command, Stdio};
12 use std::string::String;
13 use std::thread;
14 use std::time::Duration;
15 use std::{fmt, fs};
16 use test_infra::Error as InfraError;
17 use test_infra::*;
18 use wait_timeout::ChildExt;
19 
20 #[cfg(target_arch = "x86_64")]
21 pub const FOCAL_IMAGE_NAME: &str = "focal-server-cloudimg-amd64-custom-20210609-0.raw";
22 #[cfg(target_arch = "aarch64")]
23 pub const FOCAL_IMAGE_NAME: &str = "focal-server-cloudimg-arm64-custom-20210929-0-update-tool.raw";
24 
25 #[derive(Debug)]
26 enum WaitTimeoutError {
27     Timedout,
28     ExitStatus,
29     General(std::io::Error),
30 }
31 
32 #[derive(Debug)]
33 enum Error {
34     BootTimeParse,
35     EthrLogFile(std::io::Error),
36     EthrLogParse,
37     FioOutputParse,
38     Iperf3Parse,
39     Infra(InfraError),
40     Spawn(std::io::Error),
41     Scp(SshCommandError),
42     WaitTimeout(WaitTimeoutError),
43 }
44 
45 impl From<InfraError> for Error {
46     fn from(e: InfraError) -> Self {
47         Self::Infra(e)
48     }
49 }
50 
51 const BLK_IO_TEST_IMG: &str = "/var/tmp/ch-blk-io-test.img";
52 
53 pub fn init_tests() {
54     // The test image can not be created on tmpfs (e.g. /tmp) filesystem,
55     // as tmpfs does not support O_DIRECT
56     assert!(exec_host_command_output(&format!(
57         "dd if=/dev/zero of={} bs=1M count=4096",
58         BLK_IO_TEST_IMG
59     ))
60     .status
61     .success());
62 }
63 
64 pub fn cleanup_tests() {
65     fs::remove_file(BLK_IO_TEST_IMG)
66         .unwrap_or_else(|_| panic!("Failed to remove file '{}'.", BLK_IO_TEST_IMG));
67 }
68 
69 // Performance tests are expected to be executed sequentially, so we can
70 // start VM guests with the same IP while putting them on a different
71 // private network. The default constructor "Guest::new()" does not work
72 // well, as we can easily create more than 256 VMs from repeating various
73 // performance tests dozens times in a single run.
74 fn performance_test_new_guest(disk_config: Box<dyn DiskConfig>) -> Guest {
75     Guest::new_from_ip_range(disk_config, "172.19", 0)
76 }
77 
78 const DIRECT_KERNEL_BOOT_CMDLINE: &str =
79     "root=/dev/vda1 console=hvc0 rw systemd.journald.forward_to_console=1";
80 
81 // Creates the path for direct kernel boot and return the path.
82 // For x86_64, this function returns the vmlinux kernel path.
83 // For AArch64, this function returns the PE kernel path.
84 fn direct_kernel_boot_path() -> PathBuf {
85     let mut workload_path = dirs::home_dir().unwrap();
86     workload_path.push("workloads");
87 
88     let mut kernel_path = workload_path;
89     #[cfg(target_arch = "x86_64")]
90     kernel_path.push("vmlinux");
91     #[cfg(target_arch = "aarch64")]
92     kernel_path.push("Image");
93 
94     kernel_path
95 }
96 
97 // Wait the child process for a given timeout
98 fn child_wait_timeout(child: &mut Child, timeout: u64) -> Result<(), WaitTimeoutError> {
99     match child.wait_timeout(Duration::from_secs(timeout)) {
100         Err(e) => {
101             return Err(WaitTimeoutError::General(e));
102         }
103         Ok(s) => match s {
104             None => {
105                 return Err(WaitTimeoutError::Timedout);
106             }
107             Some(s) => {
108                 if !s.success() {
109                     return Err(WaitTimeoutError::ExitStatus);
110                 }
111             }
112         },
113     }
114 
115     Ok(())
116 }
117 
118 fn parse_iperf3_output(output: &[u8], sender: bool) -> Result<f64, Error> {
119     std::panic::catch_unwind(|| {
120         let s = String::from_utf8_lossy(output);
121         let v: Value = serde_json::from_str(&s).expect("'iperf3' parse error: invalid json output");
122 
123         let bps: f64 = if sender {
124             v["end"]["sum_sent"]["bits_per_second"]
125                 .as_f64()
126                 .expect("'iperf3' parse error: missing entry 'end.sum_sent.bits_per_second'")
127         } else {
128             v["end"]["sum_received"]["bits_per_second"]
129                 .as_f64()
130                 .expect("'iperf3' parse error: missing entry 'end.sum_received.bits_per_second'")
131         };
132 
133         bps
134     })
135     .map_err(|_| {
136         eprintln!(
137             "=============== iperf3 output ===============\n\n{}\n\n===========end============\n\n",
138             String::from_utf8_lossy(output)
139         );
140         Error::Iperf3Parse
141     })
142 }
143 
144 fn measure_virtio_net_throughput(
145     test_timeout: u32,
146     queue_pairs: u32,
147     guest: &Guest,
148     receive: bool,
149 ) -> Result<f64, Error> {
150     let default_port = 5201;
151 
152     // 1. start the iperf3 server on the guest
153     for n in 0..queue_pairs {
154         guest
155             .ssh_command(&format!("iperf3 -s -p {} -D", default_port + n))
156             .map_err(InfraError::SshCommand)?;
157     }
158 
159     thread::sleep(Duration::new(1, 0));
160 
161     // 2. start the iperf3 client on host to measure RX through-put
162     let mut clients = Vec::new();
163     for n in 0..queue_pairs {
164         let mut cmd = Command::new("iperf3");
165         cmd.args(&[
166             "-J", // Output in JSON format
167             "-c",
168             &guest.network.guest_ip,
169             "-p",
170             &format!("{}", default_port + n),
171             "-t",
172             &format!("{}", test_timeout),
173         ]);
174         // For measuring the guest transmit throughput (as a sender),
175         // use reverse mode of the iperf3 client on the host
176         if !receive {
177             cmd.args(&["-R"]);
178         }
179         let client = cmd
180             .stderr(Stdio::piped())
181             .stdout(Stdio::piped())
182             .spawn()
183             .map_err(Error::Spawn)?;
184 
185         clients.push(client);
186     }
187 
188     let mut err: Option<Error> = None;
189     let mut bps = Vec::new();
190     let mut failed = false;
191     for c in clients {
192         let mut c = c;
193         if let Err(e) = child_wait_timeout(&mut c, test_timeout as u64 + 5) {
194             err = Some(Error::WaitTimeout(e));
195             failed = true;
196         }
197 
198         if !failed {
199             // Safe to unwrap as we know the child has terminated succesffully
200             let output = c.wait_with_output().unwrap();
201             bps.push(parse_iperf3_output(&output.stdout, receive)?);
202         } else {
203             let _ = c.kill();
204             let output = c.wait_with_output().unwrap();
205             println!(
206                 "=============== Client output [Error] ===============\n\n{}\n\n===========end============\n\n",
207                 String::from_utf8_lossy(&output.stdout)
208             );
209         }
210     }
211 
212     if let Some(e) = err {
213         Err(e)
214     } else {
215         Ok(bps.iter().sum())
216     }
217 }
218 
219 pub fn performance_net_throughput(control: &PerformanceTestControl) -> f64 {
220     let test_timeout = control.test_timeout;
221     let rx = control.net_rx.unwrap();
222 
223     let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string());
224     let guest = performance_test_new_guest(Box::new(focal));
225 
226     let num_queues = control.num_queues.unwrap();
227     let queue_size = control.queue_size.unwrap();
228     let net_params = format!(
229         "tap=,mac={},ip={},mask=255.255.255.0,num_queues={},queue_size={}",
230         guest.network.guest_mac, guest.network.host_ip, num_queues, queue_size,
231     );
232 
233     let mut child = GuestCommand::new(&guest)
234         .args(&["--cpus", &format!("boot={}", num_queues)])
235         .args(&["--memory", "size=4G"])
236         .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()])
237         .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE])
238         .default_disks()
239         .args(&["--net", net_params.as_str()])
240         .capture_output()
241         .set_print_cmd(false)
242         .spawn()
243         .unwrap();
244 
245     let r = std::panic::catch_unwind(|| {
246         guest.wait_vm_boot(None).unwrap();
247         measure_virtio_net_throughput(test_timeout, num_queues / 2, &guest, rx).unwrap()
248     });
249 
250     let _ = child.kill();
251     let output = child.wait_with_output().unwrap();
252 
253     match r {
254         Ok(r) => r,
255         Err(e) => {
256             handle_child_output(Err(e), &output);
257             panic!("test failed!");
258         }
259     }
260 }
261 
262 fn parse_ethr_latency_output(output: &[u8]) -> Result<Vec<f64>, Error> {
263     std::panic::catch_unwind(|| {
264         let s = String::from_utf8_lossy(output);
265         let mut latency = Vec::new();
266         for l in s.lines() {
267             let v: Value = serde_json::from_str(l).expect("'ethr' parse error: invalid json line");
268             // Skip header/summary lines
269             if let Some(avg) = v["Avg"].as_str() {
270                 // Assume the latency unit is always "us"
271                 latency.push(
272                     avg.split("us").collect::<Vec<&str>>()[0]
273                         .parse::<f64>()
274                         .expect("'ethr' parse error: invalid 'Avg' entry"),
275                 );
276             }
277         }
278 
279         assert!(
280             !latency.is_empty(),
281             "'ethr' parse error: no valid latency data found"
282         );
283 
284         latency
285     })
286     .map_err(|_| {
287         eprintln!(
288             "=============== ethr output ===============\n\n{}\n\n===========end============\n\n",
289             String::from_utf8_lossy(output)
290         );
291         Error::EthrLogParse
292     })
293 }
294 
295 fn measure_virtio_net_latency(guest: &Guest, test_timeout: u32) -> Result<Vec<f64>, Error> {
296     // copy the 'ethr' tool to the guest image
297     let ethr_path = "/usr/local/bin/ethr";
298     let ethr_remote_path = "/tmp/ethr";
299     scp_to_guest(
300         Path::new(ethr_path),
301         Path::new(ethr_remote_path),
302         &guest.network.guest_ip,
303         //DEFAULT_SSH_RETRIES,
304         1,
305         DEFAULT_SSH_TIMEOUT,
306     )
307     .map_err(Error::Scp)?;
308 
309     // Start the ethr server on the guest
310     guest
311         .ssh_command(&format!("{} -s &> /dev/null &", ethr_remote_path))
312         .map_err(InfraError::SshCommand)?;
313 
314     thread::sleep(Duration::new(1, 0));
315 
316     // Start the ethr client on the host
317     let log_file = guest
318         .tmp_dir
319         .as_path()
320         .join("ethr.client.log")
321         .to_str()
322         .unwrap()
323         .to_string();
324     let mut c = Command::new(ethr_path)
325         .args(&[
326             "-c",
327             &guest.network.guest_ip,
328             "-t",
329             "l",
330             "-o",
331             &log_file, // file output is JSON format
332             "-d",
333             &format!("{}s", test_timeout),
334         ])
335         .stderr(Stdio::piped())
336         .stdout(Stdio::piped())
337         .spawn()
338         .map_err(Error::Spawn)?;
339 
340     if let Err(e) = child_wait_timeout(&mut c, test_timeout as u64 + 5).map_err(Error::WaitTimeout)
341     {
342         let _ = c.kill();
343         return Err(e);
344     }
345 
346     // Parse the ethr latency test output
347     let content = fs::read(log_file).map_err(Error::EthrLogFile)?;
348     parse_ethr_latency_output(&content)
349 }
350 
351 pub fn performance_net_latency(control: &PerformanceTestControl) -> f64 {
352     let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string());
353     let guest = performance_test_new_guest(Box::new(focal));
354 
355     let num_queues = control.num_queues.unwrap();
356     let queue_size = control.queue_size.unwrap();
357     let net_params = format!(
358         "tap=,mac={},ip={},mask=255.255.255.0,num_queues={},queue_size={}",
359         guest.network.guest_mac, guest.network.host_ip, num_queues, queue_size,
360     );
361 
362     let mut child = GuestCommand::new(&guest)
363         .args(&["--cpus", &format!("boot={}", num_queues)])
364         .args(&["--memory", "size=4G"])
365         .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()])
366         .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE])
367         .default_disks()
368         .args(&["--net", net_params.as_str()])
369         .capture_output()
370         .set_print_cmd(false)
371         .spawn()
372         .unwrap();
373 
374     let r = std::panic::catch_unwind(|| {
375         guest.wait_vm_boot(None).unwrap();
376 
377         // 'ethr' tool will measure the latency multiple times with provided test time
378         let latency = measure_virtio_net_latency(&guest, control.test_timeout).unwrap();
379         mean(&latency).unwrap()
380     });
381 
382     let _ = child.kill();
383     let output = child.wait_with_output().unwrap();
384 
385     match r {
386         Ok(r) => r,
387         Err(e) => {
388             handle_child_output(Err(e), &output);
389             panic!("test failed!");
390         }
391     }
392 }
393 
394 fn parse_boot_time_output(output: &[u8]) -> Result<f64, Error> {
395     std::panic::catch_unwind(|| {
396         let l: Vec<String> = String::from_utf8_lossy(output)
397             .lines()
398             .into_iter()
399             .filter(|l| l.contains("Debug I/O port: Kernel code"))
400             .map(|l| l.to_string())
401             .collect();
402 
403         assert_eq!(
404             l.len(),
405             2,
406             "Expecting two matching lines for 'Debug I/O port: Kernel code'"
407         );
408 
409         let time_stamp_kernel_start = {
410             let s = l[0].split("--").collect::<Vec<&str>>();
411             assert_eq!(
412                 s.len(),
413                 2,
414                 "Expecting '--' for the matching line of 'Debug I/O port' output"
415             );
416 
417             // Sample output: "[Debug I/O port: Kernel code 0x40] 0.096537 seconds"
418             assert!(
419                 s[1].contains("0x40"),
420                 "Expecting kernel code '0x40' for 'linux_kernel_start' time stamp output"
421             );
422             let t = s[1].split_whitespace().collect::<Vec<&str>>();
423             assert_eq!(
424                 t.len(),
425                 8,
426                 "Expecting exact '8' words from the 'Debug I/O port' output"
427             );
428             assert!(
429                 t[7].eq("seconds"),
430                 "Expecting 'seconds' as the the last word of the 'Debug I/O port' output"
431             );
432 
433             t[6].parse::<f64>().unwrap()
434         };
435 
436         let time_stamp_user_start = {
437             let s = l[1].split("--").collect::<Vec<&str>>();
438             assert_eq!(
439                 s.len(),
440                 2,
441                 "Expecting '--' for the matching line of 'Debug I/O port' output"
442             );
443 
444             // Sample output: "Debug I/O port: Kernel code 0x41] 0.198980 seconds"
445             assert!(
446                 s[1].contains("0x41"),
447                 "Expecting kernel code '0x41' for 'linux_kernel_start' time stamp output"
448             );
449             let t = s[1].split_whitespace().collect::<Vec<&str>>();
450             assert_eq!(
451                 t.len(),
452                 8,
453                 "Expecting exact '8' words from the 'Debug I/O port' output"
454             );
455             assert!(
456                 t[7].eq("seconds"),
457                 "Expecting 'seconds' as the the last word of the 'Debug I/O port' output"
458             );
459 
460             t[6].parse::<f64>().unwrap()
461         };
462 
463         time_stamp_user_start - time_stamp_kernel_start
464     })
465     .map_err(|_| {
466         eprintln!(
467             "=============== boot-time output ===============\n\n{}\n\n===========end============\n\n",
468             String::from_utf8_lossy(output)
469         );
470         Error::BootTimeParse
471     })
472 }
473 
474 fn measure_boot_time(cmd: &mut GuestCommand, test_timeout: u32) -> Result<f64, Error> {
475     let mut child = cmd.capture_output().set_print_cmd(false).spawn().unwrap();
476 
477     thread::sleep(Duration::new(test_timeout as u64, 0));
478     let _ = child.kill();
479     let output = child.wait_with_output().unwrap();
480 
481     parse_boot_time_output(&output.stderr).map_err(|e| {
482         eprintln!(
483             "\n\n==== Start child stdout ====\n\n{}\n\n==== End child stdout ====",
484             String::from_utf8_lossy(&output.stdout)
485         );
486         eprintln!(
487             "\n\n==== Start child stderr ====\n\n{}\n\n==== End child stderr ====",
488             String::from_utf8_lossy(&output.stderr)
489         );
490 
491         e
492     })
493 }
494 
495 pub fn performance_boot_time(control: &PerformanceTestControl) -> f64 {
496     let r = std::panic::catch_unwind(|| {
497         let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string());
498         let guest = performance_test_new_guest(Box::new(focal));
499         let mut cmd = GuestCommand::new(&guest);
500 
501         let c = cmd
502             .args(&["--memory", "size=1G"])
503             .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()])
504             .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE])
505             .args(&["--console", "off"])
506             .default_disks();
507 
508         measure_boot_time(c, control.test_timeout).unwrap()
509     });
510 
511     match r {
512         Ok(r) => r,
513         Err(_) => {
514             panic!("test failed!");
515         }
516     }
517 }
518 
519 pub fn performance_boot_time_pmem(control: &PerformanceTestControl) -> f64 {
520     let r = std::panic::catch_unwind(|| {
521         let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string());
522         let guest = performance_test_new_guest(Box::new(focal));
523         let mut cmd = GuestCommand::new(&guest);
524         let c = cmd
525             .args(&["--memory", "size=1G,hugepages=on"])
526             .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()])
527             .args(&["--cmdline", "root=/dev/pmem0p1 console=ttyS0 quiet rw"])
528             .args(&["--console", "off"])
529             .args(&[
530                 "--pmem",
531                 format!(
532                     "file={}",
533                     guest.disk_config.disk(DiskType::OperatingSystem).unwrap()
534                 )
535                 .as_str(),
536             ]);
537 
538         measure_boot_time(c, control.test_timeout).unwrap()
539     });
540 
541     match r {
542         Ok(r) => r,
543         Err(_) => {
544             panic!("test failed!");
545         }
546     }
547 }
548 
549 pub enum FioOps {
550     Read,
551     RandomRead,
552     Write,
553     RandomWrite,
554 }
555 
556 impl fmt::Display for FioOps {
557     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
558         match self {
559             FioOps::Read => write!(f, "read"),
560             FioOps::RandomRead => write!(f, "randread"),
561             FioOps::Write => write!(f, "write"),
562             FioOps::RandomWrite => write!(f, "randwrite"),
563         }
564     }
565 }
566 
567 fn parse_fio_output(output: &str, fio_ops: &FioOps, num_jobs: u32) -> Result<f64, Error> {
568     std::panic::catch_unwind(|| {
569         let v: Value =
570             serde_json::from_str(output).expect("'fio' parse error: invalid json output");
571         let jobs = v["jobs"]
572             .as_array()
573             .expect("'fio' parse error: missing entry 'jobs'");
574         assert_eq!(
575             jobs.len(),
576             num_jobs as usize,
577             "'fio' parse error: Unexpected number of 'fio' jobs."
578         );
579 
580         let read = match fio_ops {
581             FioOps::Read | FioOps::RandomRead => true,
582             FioOps::Write | FioOps::RandomWrite => false,
583         };
584 
585         let mut total_bps = 0_f64;
586         for j in jobs {
587             if read {
588                 let bytes = j["read"]["io_bytes"]
589                     .as_u64()
590                     .expect("'fio' parse error: missing entry 'read.io_bytes'");
591                 let runtime = j["read"]["runtime"]
592                     .as_u64()
593                     .expect("'fio' parse error: missing entry 'read.runtime'")
594                     as f64
595                     / 1000_f64;
596                 total_bps += bytes as f64 / runtime as f64;
597             } else {
598                 let bytes = j["write"]["io_bytes"]
599                     .as_u64()
600                     .expect("'fio' parse error: missing entry 'write.io_bytes'");
601                 let runtime = j["write"]["runtime"]
602                     .as_u64()
603                     .expect("'fio' parse error: missing entry 'write.runtime'")
604                     as f64
605                     / 1000_f64;
606                 total_bps += bytes as f64 / runtime as f64;
607             }
608         }
609 
610         total_bps
611     })
612     .map_err(|_| {
613         eprintln!(
614             "=============== Fio output ===============\n\n{}\n\n===========end============\n\n",
615             output
616         );
617         Error::FioOutputParse
618     })
619 }
620 
621 pub fn performance_block_io(control: &PerformanceTestControl) -> f64 {
622     let test_timeout = control.test_timeout;
623     let num_queues = control.num_queues.unwrap();
624     let fio_ops = control.fio_ops.as_ref().unwrap();
625 
626     let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string());
627     let guest = performance_test_new_guest(Box::new(focal));
628     let api_socket = guest
629         .tmp_dir
630         .as_path()
631         .join("cloud-hypervisor.sock")
632         .to_str()
633         .unwrap()
634         .to_string();
635 
636     let mut child = GuestCommand::new(&guest)
637         .args(&["--cpus", &format!("boot={}", num_queues)])
638         .args(&["--memory", "size=4G"])
639         .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()])
640         .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE])
641         .args(&[
642             "--disk",
643             format!(
644                 "path={}",
645                 guest.disk_config.disk(DiskType::OperatingSystem).unwrap()
646             )
647             .as_str(),
648             format!(
649                 "path={}",
650                 guest.disk_config.disk(DiskType::CloudInit).unwrap()
651             )
652             .as_str(),
653             format!("path={}", BLK_IO_TEST_IMG).as_str(),
654         ])
655         .default_net()
656         .args(&["--api-socket", &api_socket])
657         .capture_output()
658         .set_print_cmd(false)
659         .spawn()
660         .unwrap();
661 
662     let r = std::panic::catch_unwind(|| {
663         guest.wait_vm_boot(None).unwrap();
664 
665         let fio_command = format!(
666             "sudo fio --filename=/dev/vdc --name=test --output-format=json \
667             --direct=1 --bs=4k --ioengine=io_uring --iodepth=64 \
668             --rw={} --runtime={} --numjobs={}",
669             fio_ops, test_timeout, num_queues
670         );
671         let output = guest
672             .ssh_command(&fio_command)
673             .map_err(InfraError::SshCommand)
674             .unwrap();
675 
676         // Parse fio output
677         parse_fio_output(&output, fio_ops, num_queues).unwrap()
678     });
679 
680     let _ = child.kill();
681     let output = child.wait_with_output().unwrap();
682 
683     match r {
684         Ok(r) => r,
685         Err(e) => {
686             handle_child_output(Err(e), &output);
687             panic!("test failed!");
688         }
689     }
690 }
691 
692 #[cfg(test)]
693 mod tests {
694     use super::*;
695 
696     #[test]
697     fn test_parse_iperf3_output() {
698         let output = r#"
699 {
700 	"end":	{
701 		"sum_sent":	{
702 			"start":	0,
703 			"end":	5.000196,
704 			"seconds":	5.000196,
705 			"bytes":	14973836248,
706 			"bits_per_second":	23957198874.604115,
707 			"retransmits":	0,
708 			"sender":	false
709 		}
710 	}
711 }
712        "#;
713         assert_eq!(
714             parse_iperf3_output(output.as_bytes(), true).unwrap(),
715             23957198874.604115
716         );
717 
718         let output = r#"
719 {
720 	"end":	{
721 		"sum_received":	{
722 			"start":	0,
723 			"end":	5.000626,
724 			"seconds":	5.000626,
725 			"bytes":	24703557800,
726 			"bits_per_second":	39520744482.79,
727 			"sender":	true
728 		}
729 	}
730 }
731               "#;
732         assert_eq!(
733             parse_iperf3_output(output.as_bytes(), false).unwrap(),
734             39520744482.79
735         );
736     }
737 
738     #[test]
739     fn test_parse_ethr_latency_output() {
740         let output = r#"{"Time":"2022-02-08T03:52:50Z","Title":"","Type":"INFO","Message":"Using destination: 192.168.249.2, ip: 192.168.249.2, port: 8888"}
741 {"Time":"2022-02-08T03:52:51Z","Title":"","Type":"INFO","Message":"Running latency test: 1000, 1"}
742 {"Time":"2022-02-08T03:52:51Z","Title":"","Type":"LatencyResult","RemoteAddr":"192.168.249.2","Protocol":"TCP","Avg":"80.712us","Min":"61.677us","P50":"257.014us","P90":"74.418us","P95":"107.283us","P99":"119.309us","P999":"142.100us","P9999":"216.341us","Max":"216.341us"}
743 {"Time":"2022-02-08T03:52:52Z","Title":"","Type":"LatencyResult","RemoteAddr":"192.168.249.2","Protocol":"TCP","Avg":"79.826us","Min":"55.129us","P50":"598.996us","P90":"73.849us","P95":"106.552us","P99":"122.152us","P999":"142.459us","P9999":"474.280us","Max":"474.280us"}
744 {"Time":"2022-02-08T03:52:53Z","Title":"","Type":"LatencyResult","RemoteAddr":"192.168.249.2","Protocol":"TCP","Avg":"78.239us","Min":"56.999us","P50":"396.820us","P90":"69.469us","P95":"115.421us","P99":"119.404us","P999":"130.158us","P9999":"258.686us","Max":"258.686us"}"#;
745 
746         let ret = parse_ethr_latency_output(output.as_bytes()).unwrap();
747         let reference = vec![80.712_f64, 79.826_f64, 78.239_f64];
748         assert_eq!(ret, reference);
749     }
750 
751     #[test]
752     fn test_parse_boot_time_output() {
753         let output = r#"
754 cloud-hypervisor: 161.167103ms: <vcpu0> INFO:vmm/src/vm.rs:392 -- [Debug I/O port: Kernel code 0x40] 0.132 seconds
755 cloud-hypervisor: 613.57361ms: <vcpu0> INFO:vmm/src/vm.rs:392 -- [Debug I/O port: Kernel code 0x41] 0.5845 seconds
756         "#;
757 
758         assert_eq!(parse_boot_time_output(output.as_bytes()).unwrap(), 0.4525);
759     }
760 
761     #[test]
762     fn test_parse_fio_output() {
763         let output = r#"
764 {
765   "jobs" : [
766     {
767       "read" : {
768         "io_bytes" : 1965273088,
769         "io_kbytes" : 1919212,
770         "bw_bytes" : 392976022,
771         "bw" : 383765,
772         "iops" : 95941.411718,
773         "runtime" : 5001,
774         "total_ios" : 479803,
775         "short_ios" : 0,
776         "drop_ios" : 0
777       }
778     }
779   ]
780 }
781 "#;
782 
783         let bps = 1965273088_f64 / (5001_f64 / 1000_f64);
784         assert_eq!(
785             parse_fio_output(output, &FioOps::RandomRead, 1).unwrap(),
786             bps
787         );
788         assert_eq!(parse_fio_output(output, &FioOps::Read, 1).unwrap(), bps);
789 
790         let output = r#"
791 {
792   "jobs" : [
793     {
794       "write" : {
795         "io_bytes" : 1172783104,
796         "io_kbytes" : 1145296,
797         "bw_bytes" : 234462835,
798         "bw" : 228967,
799         "iops" : 57241.903239,
800         "runtime" : 5002,
801         "total_ios" : 286324,
802         "short_ios" : 0,
803         "drop_ios" : 0
804       }
805     },
806     {
807       "write" : {
808         "io_bytes" : 1172234240,
809         "io_kbytes" : 1144760,
810         "bw_bytes" : 234353106,
811         "bw" : 228860,
812         "iops" : 57215.113954,
813         "runtime" : 5002,
814         "total_ios" : 286190,
815         "short_ios" : 0,
816         "drop_ios" : 0
817       }
818     }
819   ]
820 }
821 "#;
822 
823         let bps = 1172783104_f64 / (5002_f64 / 1000_f64) + 1172234240_f64 / (5002_f64 / 1000_f64);
824         assert_eq!(
825             parse_fio_output(output, &FioOps::RandomWrite, 2).unwrap(),
826             bps
827         );
828         assert_eq!(parse_fio_output(output, &FioOps::Write, 2).unwrap(), bps);
829     }
830 }
831