Turn one of the test_mvc tests into a performance test

performance/perf2tex.py

@@ -99,4 +99,6 @@ def create_latex_chart(perf_data):
     return assemble_latex_chart(opt_levels, color_defs, test_names, data)
 
 if __name__ == '__main__':
-    print(create_latex_chart(utils.parse_perf_data(utils.DEFAULT_PERF_FILE_NAME)))
+    print(
+        create_latex_chart(
+            utils.parse_perf_data(utils.DEFAULT_PERF_FILE_NAME)[utils.TOTAL_TIME_QUANTITY]))

performance/test_mvc.py

@@ -0,0 +1,334 @@
+import unittest
+import utils
+
+all_files = [
+    "core/mini_modify.alc",
+    "core/core_formalism.mvc",
+    "core/core_algorithm.alc",
+    "primitives.alc",
+    "object_operations.alc",
+    "conformance_scd.alc",
+    "library.alc",
+    "transform.alc",
+    "model_management.alc",
+    "ramify.alc",
+    "metamodels.alc",
+    "random.alc",
+    "constructors.alc",
+    "modelling.alc",
+    "compilation_manager.alc",
+]
+
+class TestMvC(unittest.TestCase):
+    def transform_add_MT_pn_simulate_larger(self, optimization_level):
+        def step_and_print():
+            return [
+                # transformation_execute (pn_step)
+                "Which transformation do you want to execute?",
+                "Which model to bind for source element PetriNets_Runtime",
+                "Which model to create for target element PetriNets_Runtime",
+                "Transformation executed with result: True",
+                "Ready for command...",
+                # transformation_execute (pn_runtime_to_design)
+                "Which transformation do you want to execute?",
+                "Which model to bind for source element PetriNets_Runtime",
+                "Which model to create for target element PetriNets",
+                "Transformation executed with result: True",
+                "Ready for command...",
+                # transformation_execute (pn_print)
+                "Which transformation do you want to execute?",
+                "Which model to bind for source element PetriNets",
+                None,
+                None,
+                None,
+                "Transformation executed with result: True",
+                "Ready for command...",
+            ]
+
+        utils.write_total_runtime_to_file(
+            'test_mvc', optimization_level,
+            utils.run_correctness_test(all_files,
+            ["root", "root", "root",
+                "model_add",
+                    "SimpleClassDiagrams",
+                    "PetriNets",
+                    ] + utils.get_model_constructor(open("integration/code/pn_design.mvc", "r").read()) + [
+                "model_add",
+                    "SimpleClassDiagrams",
+                    "PetriNets_Runtime",
+                    ] + utils.get_model_constructor(open("integration/code/pn_runtime.mvc", "r").read()) + [
+                "model_add",
+                    "PetriNets",
+                    "my_pn",
+                    ] + utils.get_model_constructor(open("integration/code/pn_design_model_larger.mvc", "r").read()) + [
+                "model_list",
+                "transformation_add_MT_language",
+                    "PetriNets_Runtime",
+                    "PetriNets",
+                    "",
+                    "PetriNets_RAM",
+                "model_list",
+                "model_modify",
+                    "__merged_PetriNets_RAM",
+                        "instantiate",
+                            "Association",
+                            "D2R_PlaceLink",
+                            "PetriNets/Place",
+                            "PetriNets_Runtime/Place",
+                        "instantiate",
+                            "Association",
+                            "D2R_TransitionLink",
+                            "PetriNets/Transition",
+                            "PetriNets_Runtime/Transition",
+                        "instantiate",
+                            "Association",
+                            "R2D_PlaceLink",
+                            "PetriNets_Runtime/Place",
+                            "PetriNets/Place",
+                        "instantiate",
+                            "Association",
+                            "R2D_TransitionLink",
+                            "PetriNets_Runtime/Transition",
+                            "PetriNets/Transition",
+                        "exit",
+                "transformation_RAMify",
+                    "__merged_PetriNets_RAM",
+                    "PetriNets_RAM",
+                "transformation_add_MT",
+                    "PetriNets_RAM",
+                    "PetriNets",
+                    "",
+                    "PetriNets_Runtime",
+                    "",
+                    "pn_design_to_runtime",
+                    ] + utils.get_model_constructor(open("integration/code/pn_design_to_runtime.mvc", "r").read()) + [
+                "transformation_add_MT",
+                    "PetriNets_RAM",
+                    "PetriNets_Runtime",
+                    "",
+                    "PetriNets",
+                    "",
+                    "pn_runtime_to_design",
+                    ] + utils.get_model_constructor(open("integration/code/pn_runtime_to_design.mvc", "r").read()) + [
+                "transformation_add_MT",
+                    "PetriNets_RAM",
+                    "PetriNets_Runtime",
+                    "",
+                    "PetriNets_Runtime",
+                    "",
+                    "pn_step",
+                    ] + utils.get_model_constructor(open("integration/code/pn_simulate.mvc", "r").read()) + [
+                "transformation_add_MT",
+                    "PetriNets_RAM",
+                    "PetriNets",
+                    "",
+                    "",
+                    "pn_print",
+                    ] + utils.get_model_constructor(open("integration/code/pn_print.mvc", "r").read()) + [
+                "model_list",
+                "transformation_list",
+                "transformation_execute",
+                "pn_print",
+                "my_pn",
+                "transformation_execute",
+                "pn_design_to_runtime",
+                "my_pn",
+                "my_pn_runtime", ] + [
+                    "transformation_execute",
+                    "pn_step",
+                    "my_pn_runtime",
+                    "my_pn_runtime",
+                    "transformation_execute",
+                    "pn_runtime_to_design",
+                    "my_pn_runtime",
+                    "my_pn",
+                    "transformation_execute",
+                    "pn_print",
+                    "my_pn",
+                    ] * 10 + [
+            ],
+            [   # bootup phase
+                "Desired username for admin user?",
+                "Desired password for admin user?",
+                "Please repeat the password",
+                "Passwords match!",
+                "Welcome to the Model Management Interface v2.0!",
+                "Use the 'help' command for a list of possible commands",
+                "Ready for command...",
+                # model_add
+                "Creating new model!",
+                "Model type?",
+                "Model name?",
+                "Waiting for model constructors...",
+                "Model upload success!",
+                "Ready for command...",
+                # model_add
+                "Creating new model!",
+                "Model type?",
+                "Model name?",
+                "Waiting for model constructors...",
+                "Model upload success!",
+                "Ready for command...",
+                # model_add
+                "Creating new model!",
+                "Model type?",
+                "Model name?",
+                "Waiting for model constructors...",
+                "Model upload success!",
+                "Ready for command...",
+                # model_list
+                set(["  SimpleClassDiagrams : SimpleClassDiagrams",
+                     "  CoreFormalism : SimpleClassDiagrams",
+                     "  PetriNets : SimpleClassDiagrams",
+                     "  my_pn : PetriNets",
+                     "  PetriNets_Runtime : SimpleClassDiagrams",
+                     "  core : CoreFormalism"]),
+                "Ready for command...",
+                # transformation_add_MT_language
+                "Formalisms to include (terminate with empty string)?",
+                "Name of the RAMified transformation metamodel?",
+                "Ready for command...",
+                # model_list
+                set(["  SimpleClassDiagrams : SimpleClassDiagrams",
+                     "  CoreFormalism : SimpleClassDiagrams",
+                     "  PetriNets_Runtime : SimpleClassDiagrams",
+                     "  PetriNets : SimpleClassDiagrams",
+                     "  __merged_PetriNets_RAM : SimpleClassDiagrams",
+                     "  PetriNets_RAM : SimpleClassDiagrams",
+                     "  my_pn : PetriNets",
+                     "  core : CoreFormalism"]),
+                "Ready for command...",
+                # model_modify
+                "Which model do you want to modify?",
+                "Model loaded, ready for commands!",
+                "Use 'help' command for a list of possible commands",
+                "Please give your command.",
+                # instantiate 1
+                "Type to instantiate?",
+                "Name of new element?",
+                "Source name?",
+                "Destination name?",
+                "Instantiation successful!",
+                "Please give your command.",
+                # instantiate 2
+                "Type to instantiate?",
+                "Name of new element?",
+                "Source name?",
+                "Destination name?",
+                "Instantiation successful!",
+                "Please give your command.",
+                # instantiate 3
+                "Type to instantiate?",
+                "Name of new element?",
+                "Source name?",
+                "Destination name?",
+                "Instantiation successful!",
+                "Please give your command.",
+                # instantiate 4
+                "Type to instantiate?",
+                "Name of new element?",
+                "Source name?",
+                "Destination name?",
+                "Instantiation successful!",
+                "Please give your command.",
+                "Ready for command...",
+                # transformation_RAMify
+                "Which metamodel do you want to RAMify?",
+                "Where do you want to store the RAMified metamodel?",
+                "Ready for command...",
+                # transformation_add_MT
+                "RAMified metamodel to use?",
+                "Supported metamodels:",
+                set(["  PetriNets",
+                     "  PetriNets_Runtime",
+                    ]),
+                "",
+                "Which ones do you want to use as source (empty string to finish)?",
+                "Model added as source",
+                "Which ones do you want to use as target (empty string to finish)?",
+                "Model added as target",
+                "Name of new transformation?",
+                "Waiting for model constructors...",
+                "Ready for command...",
+                # transformation_add_MT
+                "RAMified metamodel to use?",
+                "Supported metamodels:",
+                set(["  PetriNets",
+                     "  PetriNets_Runtime",
+                    ]),
+                "",
+                "Which ones do you want to use as source (empty string to finish)?",
+                "Model added as source",
+                "Which ones do you want to use as target (empty string to finish)?",
+                "Model added as target",
+                "Name of new transformation?",
+                "Waiting for model constructors...",
+                "Ready for command...",
+                # transformation_add_MT
+                "RAMified metamodel to use?",
+                "Supported metamodels:",
+                set(["  PetriNets",
+                     "  PetriNets_Runtime",
+                    ]),
+                "",
+                "Which ones do you want to use as source (empty string to finish)?",
+                "Model added as source",
+                "Which ones do you want to use as target (empty string to finish)?",
+                "Model added as target",
+                "Name of new transformation?",
+                "Waiting for model constructors...",
+                "Ready for command...",
+                # transformation_add_MT
+                "RAMified metamodel to use?",
+                "Supported metamodels:",
+                set(["  PetriNets",
+                     "  PetriNets_Runtime",
+                    ]),
+                "",
+                "Which ones do you want to use as source (empty string to finish)?",
+                "Model added as source",
+                "Which ones do you want to use as target (empty string to finish)?",
+                "Name of new transformation?",
+                "Waiting for model constructors...",
+                "Ready for command...",
+                # model_list
+                set(["  SimpleClassDiagrams : SimpleClassDiagrams",
+                     "  CoreFormalism : SimpleClassDiagrams",
+                     "  PetriNets_Runtime : SimpleClassDiagrams",
+                     "  PetriNets : SimpleClassDiagrams",
+                     "  pn_print : PetriNets_RAM",
+                     "  pn_design_to_runtime : PetriNets_RAM",
+                     "  pn_runtime_to_design : PetriNets_RAM",
+                     "  pn_step : PetriNets_RAM",
+                     "  __merged_PetriNets_RAM : SimpleClassDiagrams",
+                     "  PetriNets_RAM : SimpleClassDiagrams",
+                     "  my_pn : PetriNets",
+                     "  core : CoreFormalism"]),
+                "Ready for command...",
+                # transformation_list
+                set(["[ModelTransformation] pn_print : PetriNets_RAM",
+                     "[ModelTransformation] pn_design_to_runtime : PetriNets_RAM",
+                     "[ModelTransformation] pn_runtime_to_design : PetriNets_RAM",
+                     "[ModelTransformation] pn_step : PetriNets_RAM"]),
+                "Ready for command...",
+                # transformation_execute (pn_print)
+                "Which transformation do you want to execute?",
+                "Which model to bind for source element PetriNets",
+                set(['"lock_available" --> 1',
+                     '"critical_section_1" --> 0',
+                     '"critical_section_2" --> 0',
+                    ]),
+                "Transformation executed with result: True",
+                "Ready for command...",
+                # transformation_execute (pn_design_to_runtime)
+                "Which transformation do you want to execute?",
+                "Which model to bind for source element PetriNets",
+                "Which model to create for target element PetriNets_Runtime",
+                "Transformation executed with result: True",
+                "Ready for command...",
+                ] + \
+                    step_and_print() * 10 +
+                [],
+            optimization_level))
+
+utils.define_perf_tests(TestMvC, TestMvC.transform_add_MT_pn_simulate_larger)

performance/utils.py

@@ -12,6 +12,8 @@ import signal
 import random
 import operator
 
+from collections import defaultdict
+
 sys.path.append("interface/HUTN")
 sys.path.append("scripts")
 from hutn_compiler.compiler import main as do_compile
@@ -135,6 +137,49 @@ def compile_file(address, mod_filename, filename, mode, proc):
         except UnboundLocalError:
             pass
 
+def compile_files(address, process, files, mode):
+    """Compiles the given files in the given mode."""
+    threads = []
+    mod_files = []
+    for filename in files:
+        if os.path.isfile(filename):
+            mod_filename = filename
+        elif os.path.isfile("%s/%s" % (get_code_folder_name(), filename)):
+            mod_filename = "%s/%s" % (get_code_folder_name(), filename)
+        elif os.path.isfile("%s/%s" % (BOOTSTRAP_FOLDER_NAME, filename)):
+            mod_filename = "%s/%s" % (BOOTSTRAP_FOLDER_NAME, filename)
+        else:
+            raise Exception("File not found: %s" % filename)
+        mod_files.append(mod_filename)
+
+    to_compile = to_recompile(address, mod_files)
+
+    for mod_filename in to_compile:
+        if mod_filename.endswith(".mvc"):
+            model_mode = "MO"
+            mod_files.remove(mod_filename)
+        else:
+            model_mode = mode
+        if PARALLEL_PUSH:
+            import threading
+            threads.append(
+                threading.Thread(
+                    target=compile_file,
+                    args=[address, mod_filename, mod_filename, model_mode, process]))
+            threads[-1].start()
+        else:
+            compile_file(address, mod_filename, mod_filename, model_mode, process)
+
+    if PARALLEL_PUSH:
+        for t in threads:
+            t.join()
+
+    if mode[-1] == "O":
+        # Fire up the linker
+        val = execute("link_and_load", [address, USERNAME] + mod_files, wait=True)
+        if val != 0:
+            raise Exception("Linking error")
+
 def run_file(files, parameters, mode, handle_output, optimization_level=None):
     """Compiles the given sequence of files, feeds them the given input in the given mode,
        and handles their output."""
@@ -151,39 +196,8 @@ def run_file(files, parameters, mode, handle_output, optimization_level=None):
             modelverse_args.append('--kernel=%s' % optimization_level)
         proc = execute("run_local_modelverse", modelverse_args, wait=False)
 
-        threads = []
-        mod_files = []
-        for filename in files:
-            if os.path.isfile("%s/%s" % (get_code_folder_name(), filename)):
-                mod_filename = "%s/%s" % (get_code_folder_name(), filename)
-            elif os.path.isfile("%s/%s" % (BOOTSTRAP_FOLDER_NAME, filename)):
-                mod_filename = "%s/%s" % (BOOTSTRAP_FOLDER_NAME, filename)
-            else:
-                raise Exception("File not found: %s" % filename)
-            mod_files.append(mod_filename)
-
-        to_compile = to_recompile(address, mod_files)
-
-        for mod_filename in to_compile:
-            if PARALLEL_PUSH:
-                import threading
-                threads.append(
-                    threading.Thread(
-                        target=compile_file,
-                        args=[address, mod_filename, mod_filename, mode, proc]))
-                threads[-1].start()
-            else:
-                compile_file(address, mod_filename, mod_filename, mode, proc)
-
-        if PARALLEL_PUSH:
-            for t in threads:
-                t.join()
-
-        if mode[-1] == "O":
-            # Fire up the linker
-            val = execute("link_and_load", [address, USERNAME] + mod_files, wait=True)
-            if val != 0:
-                raise Exception("Linking error")
+        # Compile, push and link the source code files.
+        compile_files(address, proc, files, mode)
 
         # Send the request ...
         set_input_data(address, parameters)
@@ -248,17 +262,71 @@ def run_file_single_output(files, parameters, mode, optimization_level=None):
        and then collects and returns a single output."""
     return run_file_fixed_output_count(files, parameters, mode, 1, optimization_level)[0]
 
+def mean(values):
+    """Computes the arithmetic mean of the given values."""
+    return float(sum(values)) / max(len(values), 1)
+
 def run_perf_test(files, parameters, optimization_level, n_iterations=1):
     """Compiles the given sequence of files, feeds them the given input in the given mode,
        and then collects their output. This process is repeated n_iterations times. The
-       return value is the average of all outputs."""
-    result = 0.0
+       return value is the average of all outputs, along with the mean total run-time."""
+    test_runtimes = []
+    total_runtimes = []
     for _ in xrange(n_iterations):
-        result += float(
-            run_file_single_output(
-                files, parameters + [0], 'CO',
-                optimization_level)) / float(n_iterations)
-    return result
+        start_time = time.time()
+        test_time = run_file_single_output(
+            files, parameters + [0], 'CO',
+            optimization_level)
+        end_time = time.time()
+        total_time = end_time - start_time
+        test_runtimes.append(test_time)
+        total_runtimes.append(total_time)
+    return mean(test_runtimes), mean(total_runtimes)
+
+def get_expectation_checks(expected_values):
+    """Converts the given sequence of expected values to a sequence of functions which tell
+       if an input is allowed. Every function is accompanied by an expected value."""
+    def get_single_expectation_checks(expectation):
+        """Gets an expectation checker for a single expected value."""
+        if isinstance(expectation, set):
+            # We expect to receive a number of outputs equal to the size of the set, but their
+            # order does not matter.
+            for _ in xrange(len(expectation)):
+                yield (lambda val: val in expectation), expectation
+        elif expectation is None:
+            # Skip output value
+            yield (lambda _: True), expectation
+        else:
+            yield (lambda val: val == expectation), expectation
+
+    for expectation in expected_values:
+        for checker in get_single_expectation_checks(expectation):
+            yield checker
+
+def run_correctness_test(files, parameters, expected, optimization_level):
+    """Compiles the given sequence of files, feeds them the given input in the given mode,
+       and then compares the output with the expected output. The return value is the total
+       run-time of the test."""
+    checks = get_expectation_checks(expected)
+    def handle_output(output):
+        """Checks the given output against the expected output."""
+        try:
+            check, expectation = next(checks)
+        except StopIteration:
+            return False
+
+        print("Got %s, expect %s" % (output, expectation))
+        assert check(output)
+
+        return True
+
+    start_time = time.time()
+    try:
+        run_file(files, parameters, 'CO', handle_output, optimization_level)
+    except ModelverseTerminated:
+        return
+    end_time = time.time()
+    return end_time - start_time
 
 def format_output(output):
     """Formats the output of `run_file_to_completion` as a string."""
@@ -277,20 +345,56 @@ def define_perf_tests(target_class, test_function):
     for optimization_level in ALL_OPTIMIZATION_LEVELS:
         define_perf_test(target_class, test_function, optimization_level)
 
+def get_model_constructor(code):
+    # First change multiple spaces to a tab
+    code_fragments = code.split("\n")
+    code_fragments = [i for i in code_fragments if i.strip() != ""]
+    code_fragments = [i.replace("    ", "\t") for i in code_fragments]
+    initial_tabs = min([len(i) - len(i.lstrip("\t")) for i in code_fragments])
+    code_fragments = [i[initial_tabs:] for i in code_fragments]
+    code = "\n".join(code_fragments)
+
+    with open("__model.mvc", "w") as f:
+        f.write(code)
+        f.flush()
+
+    constructors = do_compile("__model.mvc", "interface/HUTN/grammars/modelling.g", "M") + ["exit"]
+
+    return constructors
+
 DEFAULT_PERF_FILE_NAME = 'perf_data.txt'
 
-def write_perf_to_file(test_name, optimization_level, result, file_name=DEFAULT_PERF_FILE_NAME):
+TOTAL_TIME_QUANTITY = 'total-runtime'
+TEST_TIME_QUANTITY = 'test-runtime'
+
+def write_perf_entry_to_stream(
+        test_name, optimization_level, quantity,
+        result, output_stream):
+    """Writes a performance measurement entry to the given stream."""
+    output_stream.write('%s:%s:%s:%f\n' % (test_name, optimization_level, quantity, result))
+
+def write_perf_to_file(
+        test_name, optimization_level, runtimes, file_name=DEFAULT_PERF_FILE_NAME):
     """Writes performance data to a file."""
+    test_runtime, total_runtime = runtimes
+    with open(file_name, "a") as perf_file:
+        write_perf_entry_to_stream(
+            test_name, optimization_level, TEST_TIME_QUANTITY, test_runtime, perf_file)
+        write_perf_entry_to_stream(
+            test_name, optimization_level, TOTAL_TIME_QUANTITY, total_runtime, perf_file)
+
+def write_total_runtime_to_file(
+        test_name, optimization_level, total_runtime, file_name=DEFAULT_PERF_FILE_NAME):
+    """Writes a total runtime entry to a file."""
     with open(file_name, "a") as perf_file:
-        perf_file.write('%s:%s:%f\n' % (test_name, optimization_level, result))
+        write_perf_entry_to_stream(
+            test_name, optimization_level, TOTAL_TIME_QUANTITY, total_runtime, perf_file)
 
 def parse_perf_data(file_name):
     """Parses the performance data in the given file."""
-    results = {}
+    results = defaultdict(lambda: defaultdict(list))
     with open(file_name, 'r') as perf_file:
         for line in perf_file.readlines():
-            test_name, optimization_level, result = line.strip().split(':')
-            if optimization_level not in results:
-                results[optimization_level] = []
-            results[optimization_level].append((test_name, result))
+            test_name, optimization_level, quantity, result = line.strip().split(':')
+            results[quantity][optimization_level].append((test_name, result))
     return sorted(results.items(), key=operator.itemgetter(1))