1#!/usr/bin/env python 2# 3# Simple benchmarking framework 4# 5# Copyright (c) 2019 Virtuozzo International GmbH. 6# 7# This program is free software; you can redistribute it and/or modify 8# it under the terms of the GNU General Public License as published by 9# the Free Software Foundation; either version 2 of the License, or 10# (at your option) any later version. 11# 12# This program is distributed in the hope that it will be useful, 13# but WITHOUT ANY WARRANTY; without even the implied warranty of 14# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15# GNU General Public License for more details. 16# 17# You should have received a copy of the GNU General Public License 18# along with this program. If not, see <http://www.gnu.org/licenses/>. 19# 20 21import statistics 22import time 23 24 25def bench_one(test_func, test_env, test_case, count=5, initial_run=True, 26 slow_limit=100): 27 """Benchmark one test-case 28 29 test_func -- benchmarking function with prototype 30 test_func(env, case), which takes test_env and test_case 31 arguments and on success returns dict with 'seconds' or 32 'iops' (or both) fields, specifying the benchmark result. 33 If both 'iops' and 'seconds' provided, the 'iops' is 34 considered the main, and 'seconds' is just an additional 35 info. On failure test_func should return {'error': str}. 36 Returned dict may contain any other additional fields. 37 test_env -- test environment - opaque first argument for test_func 38 test_case -- test case - opaque second argument for test_func 39 count -- how many times to call test_func, to calculate average 40 initial_run -- do initial run of test_func, which don't get into result 41 slow_limit -- stop at slow run (that exceedes the slow_limit by seconds). 42 (initial run is not measured) 43 44 Returns dict with the following fields: 45 'runs': list of test_func results 46 'dimension': dimension of results, may be 'seconds' or 'iops' 47 'average': average value (iops or seconds) per run (exists only if at 48 least one run succeeded) 49 'stdev': standard deviation of results 50 (exists only if at least one run succeeded) 51 'n-failed': number of failed runs (exists only if at least one run 52 failed) 53 """ 54 if initial_run: 55 print(' #initial run:') 56 print(' ', test_func(test_env, test_case)) 57 58 runs = [] 59 for i in range(count): 60 t = time.time() 61 62 print(' #run {}'.format(i+1)) 63 res = test_func(test_env, test_case) 64 print(' ', res) 65 runs.append(res) 66 67 if time.time() - t > slow_limit: 68 print(' - run is too slow, stop here') 69 break 70 71 count = len(runs) 72 73 result = {'runs': runs} 74 75 succeeded = [r for r in runs if ('seconds' in r or 'iops' in r)] 76 if succeeded: 77 if 'iops' in succeeded[0]: 78 assert all('iops' in r for r in succeeded) 79 dim = 'iops' 80 else: 81 assert all('seconds' in r for r in succeeded) 82 assert all('iops' not in r for r in succeeded) 83 dim = 'seconds' 84 result['dimension'] = dim 85 result['average'] = statistics.mean(r[dim] for r in succeeded) 86 if len(succeeded) == 1: 87 result['stdev'] = 0 88 else: 89 result['stdev'] = statistics.stdev(r[dim] for r in succeeded) 90 91 if len(succeeded) < count: 92 result['n-failed'] = count - len(succeeded) 93 94 return result 95 96 97def bench(test_func, test_envs, test_cases, *args, **vargs): 98 """Fill benchmark table 99 100 test_func -- benchmarking function, see bench_one for description 101 test_envs -- list of test environments, see bench_one 102 test_cases -- list of test cases, see bench_one 103 args, vargs -- additional arguments for bench_one 104 105 Returns dict with the following fields: 106 'envs': test_envs 107 'cases': test_cases 108 'tab': filled 2D array, where cell [i][j] is bench_one result for 109 test_cases[i] for test_envs[j] (i.e., rows are test cases and 110 columns are test environments) 111 """ 112 tab = {} 113 results = { 114 'envs': test_envs, 115 'cases': test_cases, 116 'tab': tab 117 } 118 n = 1 119 n_tests = len(test_envs) * len(test_cases) 120 for env in test_envs: 121 for case in test_cases: 122 print('Testing {}/{}: {} :: {}'.format(n, n_tests, 123 env['id'], case['id'])) 124 if case['id'] not in tab: 125 tab[case['id']] = {} 126 tab[case['id']][env['id']] = bench_one(test_func, env, case, 127 *args, **vargs) 128 n += 1 129 130 print('Done') 131 return results 132