Add the -C switch to perf2tex, which computes confidence intervals

performance/perf2tex.py

@@ -45,7 +45,8 @@ def assemble_latex_chart(optimization_levels,
                          test_names,
                          data,
                          embed=False,
-                         bar_width=DEFAULT_BAR_WIDTH):
+                         bar_width=DEFAULT_BAR_WIDTH,
+                         confidence_intervals=None):
     """Assembles a LaTeX chart from the given components."""
     lines = []
     if not embed:
@@ -79,11 +80,23 @@ def assemble_latex_chart(optimization_levels,
         every node near coord/.append style={rotate=90, anchor=west}
     ]""" % (bar_width, ','.join(map(encode_latex_string, test_names))))
     for color_name, points in data:
-        lines.append(r"""
-        \addplot[style={%s,fill=%s,mark=none}]
-            coordinates {%s};""" % (color_name, color_name, ' '.join(
-            [('(%s,%s)' % (encode_latex_string(name), measurement))
-             for name, measurement in points])))
+        if confidence_intervals is None:
+            lines.append(r"""
+            \addplot[style={%s,fill=%s,mark=none}]
+                coordinates {%s};""" % (color_name, color_name, ' '.join(
+                [('(%s,%s)' % (encode_latex_string(name), measurement))
+                 for name, measurement in points])))
+        else:
+            lines.append(r"""
+            \addplot[style={%s,fill=%s,mark=none},error bars/.cd, y dir=both, y explicit,error bar style=darkgray]
+                coordinates {%s};""" % (color_name, color_name, ' '.join(
+                [('(%s,%s) += (%s,%s) -= (%s,%s)' %
+                  (encode_latex_string(name), measurement,
+                   encode_latex_string(name),
+                   confidence_intervals[color_name][name][0],
+                   encode_latex_string(name),
+                   confidence_intervals[color_name][name][1]))
+                 for name, measurement in points])))
     lines.append(r"""
         \legend{%s}""" %
                  ','.join(map(encode_latex_string, optimization_levels)))
@@ -186,7 +199,8 @@ def assemble_stacked_latex_chart(optimization_levels,
 def create_latex_chart(perf_data,
                        sorted_opt_levels=None,
                        embed=False,
-                       bar_width=DEFAULT_BAR_WIDTH):
+                       bar_width=DEFAULT_BAR_WIDTH,
+                       confidence_intervals=None):
     """Creates a LaTeX chart for the given performance data."""
     if sorted_opt_levels is None:
         sorted_opt_levels = sort_by_runtime(perf_data)
@@ -194,6 +208,7 @@ def create_latex_chart(perf_data,
     color_scheme = generate_color_scheme(sorted_opt_levels)
     opt_levels = []
     color_defs = []
+    color_conf_intervals = {}
     test_names = []
     data = []
     for i, optimization_level in enumerate(sorted_opt_levels):
@@ -203,12 +218,17 @@ def create_latex_chart(perf_data,
         opt_levels.append(optimization_level)
         color_defs.append((color_name, color))
         data.append((color_name, measurements.items()))
+        color_conf_intervals[color_name] = {}
         for name, _ in measurements.items():
+            if confidence_intervals is not None:
+                color_conf_intervals[color_name][name] = confidence_intervals[
+                    optimization_level][name]
             if name not in test_names:
                 test_names.append(name)
 
-    return assemble_latex_chart(opt_levels, color_defs, test_names, data,
-                                embed, bar_width)
+    return assemble_latex_chart(
+        opt_levels, color_defs, test_names, data, embed, bar_width, None
+        if confidence_intervals is None else color_conf_intervals)
 
 
 def create_stacked_latex_chart(perf_data,
@@ -446,6 +466,14 @@ def main():
         nargs='*',
         help="Picks other quantities which are subtracted from the "
         "main quantity and then combined in a stacked chart.")
+    arg_parser.add_argument(
+        '-C',
+        '--confidence',
+        action='store_const',
+        const=True,
+        help="Computes 95% confidence intervals from standard deviation and "
+        "sample size quantities.",
+        default=False)
 
     args = arg_parser.parse_args()
 
@@ -487,8 +515,27 @@ def main():
                        for q, split_data in split_perf_data]
 
     if len(split_perf_data) == 0:
+        if args.confidence:
+            sd_data = perf_list_to_dict(
+                all_perf_data[args.quantity + '-standard-deviation'])
+            if args.relative:
+                sd_data = get_relative_measurements(
+                    sd_data, baseline_opt_level, main_perf_data)
+            sample_size_data = perf_list_to_dict(
+                all_perf_data[args.quantity + '-sample-size'])
+            conf_intervals = {
+                opt_level: {
+                    name: 2 * (1.96 * sd_data[opt_level][name] /
+                               (sample_size_data[opt_level][name]**0.5), )
+                    for name, mean in measurements.items()
+                }
+                for opt_level, measurements in sub_perf_data.items()
+            }
+        else:
+            conf_intervals = None
+
         print(create_latex_chart(sub_perf_data, sorted_opt_levels, args.embed,
-                                 args.bar_width))
+                                 args.bar_width, conf_intervals))
     else:
         print(create_stacked_latex_chart([
             (args.quantity, perf_dict_to_list(sub_perf_data))