1#!/usr/bin/env python 2# 3# Simple benchmarking framework 4# 5# Copyright (c) 2019 Virtuozzo International GmbH. 6# 7# This program is free software; you can redistribute it and/or modify 8# it under the terms of the GNU General Public License as published by 9# the Free Software Foundation; either version 2 of the License, or 10# (at your option) any later version. 11# 12# This program is distributed in the hope that it will be useful, 13# but WITHOUT ANY WARRANTY; without even the implied warranty of 14# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15# GNU General Public License for more details. 16# 17# You should have received a copy of the GNU General Public License 18# along with this program. If not, see <http://www.gnu.org/licenses/>. 19# 20 21import statistics 22 23 24def bench_one(test_func, test_env, test_case, count=5, initial_run=True): 25 """Benchmark one test-case 26 27 test_func -- benchmarking function with prototype 28 test_func(env, case), which takes test_env and test_case 29 arguments and on success returns dict with 'seconds' or 30 'iops' (or both) fields, specifying the benchmark result. 31 If both 'iops' and 'seconds' provided, the 'iops' is 32 considered the main, and 'seconds' is just an additional 33 info. On failure test_func should return {'error': str}. 34 Returned dict may contain any other additional fields. 35 test_env -- test environment - opaque first argument for test_func 36 test_case -- test case - opaque second argument for test_func 37 count -- how many times to call test_func, to calculate average 38 initial_run -- do initial run of test_func, which don't get into result 39 40 Returns dict with the following fields: 41 'runs': list of test_func results 42 'dimension': dimension of results, may be 'seconds' or 'iops' 43 'average': average value (iops or seconds) per run (exists only if at 44 least one run succeeded) 45 'stdev': standard deviation of results 46 (exists only if at least one run succeeded) 47 'n-failed': number of failed runs (exists only if at least one run 48 failed) 49 """ 50 if initial_run: 51 print(' #initial run:') 52 print(' ', test_func(test_env, test_case)) 53 54 runs = [] 55 for i in range(count): 56 print(' #run {}'.format(i+1)) 57 res = test_func(test_env, test_case) 58 print(' ', res) 59 runs.append(res) 60 61 result = {'runs': runs} 62 63 succeeded = [r for r in runs if ('seconds' in r or 'iops' in r)] 64 if succeeded: 65 if 'iops' in succeeded[0]: 66 assert all('iops' in r for r in succeeded) 67 dim = 'iops' 68 else: 69 assert all('seconds' in r for r in succeeded) 70 assert all('iops' not in r for r in succeeded) 71 dim = 'seconds' 72 result['dimension'] = dim 73 result['average'] = statistics.mean(r[dim] for r in succeeded) 74 result['stdev'] = statistics.stdev(r[dim] for r in succeeded) 75 76 if len(succeeded) < count: 77 result['n-failed'] = count - len(succeeded) 78 79 return result 80 81 82def ascii_one(result): 83 """Return ASCII representation of bench_one() returned dict.""" 84 if 'average' in result: 85 s = '{:.2f} +- {:.2f}'.format(result['average'], result['stdev']) 86 if 'n-failed' in result: 87 s += '\n({} failed)'.format(result['n-failed']) 88 return s 89 else: 90 return 'FAILED' 91 92 93def bench(test_func, test_envs, test_cases, *args, **vargs): 94 """Fill benchmark table 95 96 test_func -- benchmarking function, see bench_one for description 97 test_envs -- list of test environments, see bench_one 98 test_cases -- list of test cases, see bench_one 99 args, vargs -- additional arguments for bench_one 100 101 Returns dict with the following fields: 102 'envs': test_envs 103 'cases': test_cases 104 'tab': filled 2D array, where cell [i][j] is bench_one result for 105 test_cases[i] for test_envs[j] (i.e., rows are test cases and 106 columns are test environments) 107 """ 108 tab = {} 109 results = { 110 'envs': test_envs, 111 'cases': test_cases, 112 'tab': tab 113 } 114 n = 1 115 n_tests = len(test_envs) * len(test_cases) 116 for env in test_envs: 117 for case in test_cases: 118 print('Testing {}/{}: {} :: {}'.format(n, n_tests, 119 env['id'], case['id'])) 120 if case['id'] not in tab: 121 tab[case['id']] = {} 122 tab[case['id']][env['id']] = bench_one(test_func, env, case, 123 *args, **vargs) 124 n += 1 125 126 print('Done') 127 return results 128 129 130def ascii(results): 131 """Return ASCII representation of bench() returned dict.""" 132 from tabulate import tabulate 133 134 dim = None 135 tab = [[""] + [c['id'] for c in results['envs']]] 136 for case in results['cases']: 137 row = [case['id']] 138 for env in results['envs']: 139 res = results['tab'][case['id']][env['id']] 140 if dim is None: 141 dim = res['dimension'] 142 else: 143 assert dim == res['dimension'] 144 row.append(ascii_one(res)) 145 tab.append(row) 146 147 return f'All results are in {dim}\n\n' + tabulate(tab) 148