modelverse/performance/perf_stats.py

"""Computes statistiscs from raw performance results."""

import sys
import argparse
from collections import defaultdict
import utils

def main():
    arg_parser = argparse.ArgumentParser()
    arg_parser.add_argument('input', help='The performance data file.')

    args = arg_parser.parse_args()

    perf_data = utils.parse_perf_data(args.input)

    for quantity, opt_level_dict in perf_data.items():
        for opt_level, data_points in opt_level_dict:
            perf_per_test = defaultdict(list)
            for test_name, measurement in data_points:
                perf_per_test[test_name].append(measurement)

            for test_name, measurement_list in perf_per_test.items():
                avg = utils.mean(measurement_list)
                utils.write_perf_entry_to_stream(
                    test_name, opt_level, quantity, avg, sys.stdout)
                if len(measurement_list) > 1:
                    standard_dev = utils.standard_deviation(measurement_list)
                    utils.write_perf_entry_to_stream(
                        test_name, opt_level, quantity + '-standard-deviation',
                        standard_dev, sys.stdout)
                    utils.write_perf_entry_to_stream(
                        test_name, opt_level, quantity + '-relative-standard-deviation',
                        standard_dev / avg if avg != 0.0 else 0.0, sys.stdout)

                utils.write_perf_entry_to_stream(
                    test_name, opt_level, quantity + '-sample-size',
                    len(measurement_list), sys.stdout)

if __name__ == '__main__':
    main()