1# Simple benchmarking framework 2# 3# Copyright (c) 2019 Virtuozzo International GmbH. 4# 5# This program is free software; you can redistribute it and/or modify 6# it under the terms of the GNU General Public License as published by 7# the Free Software Foundation; either version 2 of the License, or 8# (at your option) any later version. 9# 10# This program is distributed in the hope that it will be useful, 11# but WITHOUT ANY WARRANTY; without even the implied warranty of 12# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13# GNU General Public License for more details. 14# 15# You should have received a copy of the GNU General Public License 16# along with this program. If not, see <http://www.gnu.org/licenses/>. 17# 18 19import math 20import tabulate 21 22# We want leading whitespace for difference row cells (see below) 23tabulate.PRESERVE_WHITESPACE = True 24 25 26def format_value(x, stdev): 27 stdev_pr = stdev / x * 100 28 if stdev_pr < 1.5: 29 # don't care too much 30 return f'{x:.2g}' 31 else: 32 return f'{x:.2g} ± {math.ceil(stdev_pr)}%' 33 34 35def result_to_text(result): 36 """Return text representation of bench_one() returned dict.""" 37 if 'average' in result: 38 s = format_value(result['average'], result['stdev']) 39 if 'n-failed' in result: 40 s += '\n({} failed)'.format(result['n-failed']) 41 return s 42 else: 43 return 'FAILED' 44 45 46def results_dimension(results): 47 dim = None 48 for case in results['cases']: 49 for env in results['envs']: 50 res = results['tab'][case['id']][env['id']] 51 if dim is None: 52 dim = res['dimension'] 53 else: 54 assert dim == res['dimension'] 55 56 assert dim in ('iops', 'seconds') 57 58 return dim 59 60 61def results_to_text(results): 62 """Return text representation of bench() returned dict.""" 63 n_columns = len(results['envs']) 64 named_columns = n_columns > 2 65 dim = results_dimension(results) 66 tab = [] 67 68 if named_columns: 69 # Environment columns are named A, B, ... 70 tab.append([''] + [chr(ord('A') + i) for i in range(n_columns)]) 71 72 tab.append([''] + [c['id'] for c in results['envs']]) 73 74 for case in results['cases']: 75 row = [case['id']] 76 case_results = results['tab'][case['id']] 77 for env in results['envs']: 78 res = case_results[env['id']] 79 row.append(result_to_text(res)) 80 tab.append(row) 81 82 # Add row of difference between columns. For each column starting from 83 # B we calculate difference with all previous columns. 84 row = ['', ''] # case name and first column 85 for i in range(1, n_columns): 86 cell = '' 87 env = results['envs'][i] 88 res = case_results[env['id']] 89 90 if 'average' not in res: 91 # Failed result 92 row.append(cell) 93 continue 94 95 for j in range(0, i): 96 env_j = results['envs'][j] 97 res_j = case_results[env_j['id']] 98 cell += ' ' 99 100 if 'average' not in res_j: 101 # Failed result 102 cell += '--' 103 continue 104 105 col_j = tab[0][j + 1] if named_columns else '' 106 diff_pr = round((res['average'] - res_j['average']) / 107 res_j['average'] * 100) 108 cell += f' {col_j}{diff_pr:+}%' 109 row.append(cell) 110 tab.append(row) 111 112 return f'All results are in {dim}\n\n' + tabulate.tabulate(tab) 113