Explorar o código

Turn one of the test_mvc tests into a performance test

jonathanvdc %!s(int64=8) %!d(string=hai) anos
pai
achega
db25261e7c
Modificáronse 3 ficheiros con 488 adicións e 48 borrados
  1. 3 1
      performance/perf2tex.py
  2. 334 0
      performance/test_mvc.py
  3. 151 47
      performance/utils.py

+ 3 - 1
performance/perf2tex.py

@@ -99,4 +99,6 @@ def create_latex_chart(perf_data):
     return assemble_latex_chart(opt_levels, color_defs, test_names, data)
     return assemble_latex_chart(opt_levels, color_defs, test_names, data)
 
 
 if __name__ == '__main__':
 if __name__ == '__main__':
-    print(create_latex_chart(utils.parse_perf_data(utils.DEFAULT_PERF_FILE_NAME)))
+    print(
+        create_latex_chart(
+            utils.parse_perf_data(utils.DEFAULT_PERF_FILE_NAME)[utils.TOTAL_TIME_QUANTITY]))

+ 334 - 0
performance/test_mvc.py

@@ -0,0 +1,334 @@
+import unittest
+import utils
+
+all_files = [
+    "core/mini_modify.alc",
+    "core/core_formalism.mvc",
+    "core/core_algorithm.alc",
+    "primitives.alc",
+    "object_operations.alc",
+    "conformance_scd.alc",
+    "library.alc",
+    "transform.alc",
+    "model_management.alc",
+    "ramify.alc",
+    "metamodels.alc",
+    "random.alc",
+    "constructors.alc",
+    "modelling.alc",
+    "compilation_manager.alc",
+]
+
+class TestMvC(unittest.TestCase):
+    def transform_add_MT_pn_simulate_larger(self, optimization_level):
+        def step_and_print():
+            return [
+                # transformation_execute (pn_step)
+                "Which transformation do you want to execute?",
+                "Which model to bind for source element PetriNets_Runtime",
+                "Which model to create for target element PetriNets_Runtime",
+                "Transformation executed with result: True",
+                "Ready for command...",
+                # transformation_execute (pn_runtime_to_design)
+                "Which transformation do you want to execute?",
+                "Which model to bind for source element PetriNets_Runtime",
+                "Which model to create for target element PetriNets",
+                "Transformation executed with result: True",
+                "Ready for command...",
+                # transformation_execute (pn_print)
+                "Which transformation do you want to execute?",
+                "Which model to bind for source element PetriNets",
+                None,
+                None,
+                None,
+                "Transformation executed with result: True",
+                "Ready for command...",
+            ]
+
+        utils.write_total_runtime_to_file(
+            'test_mvc', optimization_level,
+            utils.run_correctness_test(all_files,
+            ["root", "root", "root",
+                "model_add",
+                    "SimpleClassDiagrams",
+                    "PetriNets",
+                    ] + utils.get_model_constructor(open("integration/code/pn_design.mvc", "r").read()) + [
+                "model_add",
+                    "SimpleClassDiagrams",
+                    "PetriNets_Runtime",
+                    ] + utils.get_model_constructor(open("integration/code/pn_runtime.mvc", "r").read()) + [
+                "model_add",
+                    "PetriNets",
+                    "my_pn",
+                    ] + utils.get_model_constructor(open("integration/code/pn_design_model_larger.mvc", "r").read()) + [
+                "model_list",
+                "transformation_add_MT_language",
+                    "PetriNets_Runtime",
+                    "PetriNets",
+                    "",
+                    "PetriNets_RAM",
+                "model_list",
+                "model_modify",
+                    "__merged_PetriNets_RAM",
+                        "instantiate",
+                            "Association",
+                            "D2R_PlaceLink",
+                            "PetriNets/Place",
+                            "PetriNets_Runtime/Place",
+                        "instantiate",
+                            "Association",
+                            "D2R_TransitionLink",
+                            "PetriNets/Transition",
+                            "PetriNets_Runtime/Transition",
+                        "instantiate",
+                            "Association",
+                            "R2D_PlaceLink",
+                            "PetriNets_Runtime/Place",
+                            "PetriNets/Place",
+                        "instantiate",
+                            "Association",
+                            "R2D_TransitionLink",
+                            "PetriNets_Runtime/Transition",
+                            "PetriNets/Transition",
+                        "exit",
+                "transformation_RAMify",
+                    "__merged_PetriNets_RAM",
+                    "PetriNets_RAM",
+                "transformation_add_MT",
+                    "PetriNets_RAM",
+                    "PetriNets",
+                    "",
+                    "PetriNets_Runtime",
+                    "",
+                    "pn_design_to_runtime",
+                    ] + utils.get_model_constructor(open("integration/code/pn_design_to_runtime.mvc", "r").read()) + [
+                "transformation_add_MT",
+                    "PetriNets_RAM",
+                    "PetriNets_Runtime",
+                    "",
+                    "PetriNets",
+                    "",
+                    "pn_runtime_to_design",
+                    ] + utils.get_model_constructor(open("integration/code/pn_runtime_to_design.mvc", "r").read()) + [
+                "transformation_add_MT",
+                    "PetriNets_RAM",
+                    "PetriNets_Runtime",
+                    "",
+                    "PetriNets_Runtime",
+                    "",
+                    "pn_step",
+                    ] + utils.get_model_constructor(open("integration/code/pn_simulate.mvc", "r").read()) + [
+                "transformation_add_MT",
+                    "PetriNets_RAM",
+                    "PetriNets",
+                    "",
+                    "",
+                    "pn_print",
+                    ] + utils.get_model_constructor(open("integration/code/pn_print.mvc", "r").read()) + [
+                "model_list",
+                "transformation_list",
+                "transformation_execute",
+                "pn_print",
+                "my_pn",
+                "transformation_execute",
+                "pn_design_to_runtime",
+                "my_pn",
+                "my_pn_runtime", ] + [
+                    "transformation_execute",
+                    "pn_step",
+                    "my_pn_runtime",
+                    "my_pn_runtime",
+                    "transformation_execute",
+                    "pn_runtime_to_design",
+                    "my_pn_runtime",
+                    "my_pn",
+                    "transformation_execute",
+                    "pn_print",
+                    "my_pn",
+                    ] * 10 + [
+            ],
+            [   # bootup phase
+                "Desired username for admin user?",
+                "Desired password for admin user?",
+                "Please repeat the password",
+                "Passwords match!",
+                "Welcome to the Model Management Interface v2.0!",
+                "Use the 'help' command for a list of possible commands",
+                "Ready for command...",
+                # model_add
+                "Creating new model!",
+                "Model type?",
+                "Model name?",
+                "Waiting for model constructors...",
+                "Model upload success!",
+                "Ready for command...",
+                # model_add
+                "Creating new model!",
+                "Model type?",
+                "Model name?",
+                "Waiting for model constructors...",
+                "Model upload success!",
+                "Ready for command...",
+                # model_add
+                "Creating new model!",
+                "Model type?",
+                "Model name?",
+                "Waiting for model constructors...",
+                "Model upload success!",
+                "Ready for command...",
+                # model_list
+                set(["  SimpleClassDiagrams : SimpleClassDiagrams",
+                     "  CoreFormalism : SimpleClassDiagrams",
+                     "  PetriNets : SimpleClassDiagrams",
+                     "  my_pn : PetriNets",
+                     "  PetriNets_Runtime : SimpleClassDiagrams",
+                     "  core : CoreFormalism"]),
+                "Ready for command...",
+                # transformation_add_MT_language
+                "Formalisms to include (terminate with empty string)?",
+                "Name of the RAMified transformation metamodel?",
+                "Ready for command...",
+                # model_list
+                set(["  SimpleClassDiagrams : SimpleClassDiagrams",
+                     "  CoreFormalism : SimpleClassDiagrams",
+                     "  PetriNets_Runtime : SimpleClassDiagrams",
+                     "  PetriNets : SimpleClassDiagrams",
+                     "  __merged_PetriNets_RAM : SimpleClassDiagrams",
+                     "  PetriNets_RAM : SimpleClassDiagrams",
+                     "  my_pn : PetriNets",
+                     "  core : CoreFormalism"]),
+                "Ready for command...",
+                # model_modify
+                "Which model do you want to modify?",
+                "Model loaded, ready for commands!",
+                "Use 'help' command for a list of possible commands",
+                "Please give your command.",
+                # instantiate 1
+                "Type to instantiate?",
+                "Name of new element?",
+                "Source name?",
+                "Destination name?",
+                "Instantiation successful!",
+                "Please give your command.",
+                # instantiate 2
+                "Type to instantiate?",
+                "Name of new element?",
+                "Source name?",
+                "Destination name?",
+                "Instantiation successful!",
+                "Please give your command.",
+                # instantiate 3
+                "Type to instantiate?",
+                "Name of new element?",
+                "Source name?",
+                "Destination name?",
+                "Instantiation successful!",
+                "Please give your command.",
+                # instantiate 4
+                "Type to instantiate?",
+                "Name of new element?",
+                "Source name?",
+                "Destination name?",
+                "Instantiation successful!",
+                "Please give your command.",
+                "Ready for command...",
+                # transformation_RAMify
+                "Which metamodel do you want to RAMify?",
+                "Where do you want to store the RAMified metamodel?",
+                "Ready for command...",
+                # transformation_add_MT
+                "RAMified metamodel to use?",
+                "Supported metamodels:",
+                set(["  PetriNets",
+                     "  PetriNets_Runtime",
+                    ]),
+                "",
+                "Which ones do you want to use as source (empty string to finish)?",
+                "Model added as source",
+                "Which ones do you want to use as target (empty string to finish)?",
+                "Model added as target",
+                "Name of new transformation?",
+                "Waiting for model constructors...",
+                "Ready for command...",
+                # transformation_add_MT
+                "RAMified metamodel to use?",
+                "Supported metamodels:",
+                set(["  PetriNets",
+                     "  PetriNets_Runtime",
+                    ]),
+                "",
+                "Which ones do you want to use as source (empty string to finish)?",
+                "Model added as source",
+                "Which ones do you want to use as target (empty string to finish)?",
+                "Model added as target",
+                "Name of new transformation?",
+                "Waiting for model constructors...",
+                "Ready for command...",
+                # transformation_add_MT
+                "RAMified metamodel to use?",
+                "Supported metamodels:",
+                set(["  PetriNets",
+                     "  PetriNets_Runtime",
+                    ]),
+                "",
+                "Which ones do you want to use as source (empty string to finish)?",
+                "Model added as source",
+                "Which ones do you want to use as target (empty string to finish)?",
+                "Model added as target",
+                "Name of new transformation?",
+                "Waiting for model constructors...",
+                "Ready for command...",
+                # transformation_add_MT
+                "RAMified metamodel to use?",
+                "Supported metamodels:",
+                set(["  PetriNets",
+                     "  PetriNets_Runtime",
+                    ]),
+                "",
+                "Which ones do you want to use as source (empty string to finish)?",
+                "Model added as source",
+                "Which ones do you want to use as target (empty string to finish)?",
+                "Name of new transformation?",
+                "Waiting for model constructors...",
+                "Ready for command...",
+                # model_list
+                set(["  SimpleClassDiagrams : SimpleClassDiagrams",
+                     "  CoreFormalism : SimpleClassDiagrams",
+                     "  PetriNets_Runtime : SimpleClassDiagrams",
+                     "  PetriNets : SimpleClassDiagrams",
+                     "  pn_print : PetriNets_RAM",
+                     "  pn_design_to_runtime : PetriNets_RAM",
+                     "  pn_runtime_to_design : PetriNets_RAM",
+                     "  pn_step : PetriNets_RAM",
+                     "  __merged_PetriNets_RAM : SimpleClassDiagrams",
+                     "  PetriNets_RAM : SimpleClassDiagrams",
+                     "  my_pn : PetriNets",
+                     "  core : CoreFormalism"]),
+                "Ready for command...",
+                # transformation_list
+                set(["[ModelTransformation] pn_print : PetriNets_RAM",
+                     "[ModelTransformation] pn_design_to_runtime : PetriNets_RAM",
+                     "[ModelTransformation] pn_runtime_to_design : PetriNets_RAM",
+                     "[ModelTransformation] pn_step : PetriNets_RAM"]),
+                "Ready for command...",
+                # transformation_execute (pn_print)
+                "Which transformation do you want to execute?",
+                "Which model to bind for source element PetriNets",
+                set(['"lock_available" --> 1',
+                     '"critical_section_1" --> 0',
+                     '"critical_section_2" --> 0',
+                    ]),
+                "Transformation executed with result: True",
+                "Ready for command...",
+                # transformation_execute (pn_design_to_runtime)
+                "Which transformation do you want to execute?",
+                "Which model to bind for source element PetriNets",
+                "Which model to create for target element PetriNets_Runtime",
+                "Transformation executed with result: True",
+                "Ready for command...",
+                ] + \
+                    step_and_print() * 10 +
+                [],
+            optimization_level))
+
+utils.define_perf_tests(TestMvC, TestMvC.transform_add_MT_pn_simulate_larger)

+ 151 - 47
performance/utils.py

@@ -12,6 +12,8 @@ import signal
 import random
 import random
 import operator
 import operator
 
 
+from collections import defaultdict
+
 sys.path.append("interface/HUTN")
 sys.path.append("interface/HUTN")
 sys.path.append("scripts")
 sys.path.append("scripts")
 from hutn_compiler.compiler import main as do_compile
 from hutn_compiler.compiler import main as do_compile
@@ -135,6 +137,49 @@ def compile_file(address, mod_filename, filename, mode, proc):
         except UnboundLocalError:
         except UnboundLocalError:
             pass
             pass
 
 
+def compile_files(address, process, files, mode):
+    """Compiles the given files in the given mode."""
+    threads = []
+    mod_files = []
+    for filename in files:
+        if os.path.isfile(filename):
+            mod_filename = filename
+        elif os.path.isfile("%s/%s" % (get_code_folder_name(), filename)):
+            mod_filename = "%s/%s" % (get_code_folder_name(), filename)
+        elif os.path.isfile("%s/%s" % (BOOTSTRAP_FOLDER_NAME, filename)):
+            mod_filename = "%s/%s" % (BOOTSTRAP_FOLDER_NAME, filename)
+        else:
+            raise Exception("File not found: %s" % filename)
+        mod_files.append(mod_filename)
+
+    to_compile = to_recompile(address, mod_files)
+
+    for mod_filename in to_compile:
+        if mod_filename.endswith(".mvc"):
+            model_mode = "MO"
+            mod_files.remove(mod_filename)
+        else:
+            model_mode = mode
+        if PARALLEL_PUSH:
+            import threading
+            threads.append(
+                threading.Thread(
+                    target=compile_file,
+                    args=[address, mod_filename, mod_filename, model_mode, process]))
+            threads[-1].start()
+        else:
+            compile_file(address, mod_filename, mod_filename, model_mode, process)
+
+    if PARALLEL_PUSH:
+        for t in threads:
+            t.join()
+
+    if mode[-1] == "O":
+        # Fire up the linker
+        val = execute("link_and_load", [address, USERNAME] + mod_files, wait=True)
+        if val != 0:
+            raise Exception("Linking error")
+
 def run_file(files, parameters, mode, handle_output, optimization_level=None):
 def run_file(files, parameters, mode, handle_output, optimization_level=None):
     """Compiles the given sequence of files, feeds them the given input in the given mode,
     """Compiles the given sequence of files, feeds them the given input in the given mode,
        and handles their output."""
        and handles their output."""
@@ -151,39 +196,8 @@ def run_file(files, parameters, mode, handle_output, optimization_level=None):
             modelverse_args.append('--kernel=%s' % optimization_level)
             modelverse_args.append('--kernel=%s' % optimization_level)
         proc = execute("run_local_modelverse", modelverse_args, wait=False)
         proc = execute("run_local_modelverse", modelverse_args, wait=False)
 
 
-        threads = []
-        mod_files = []
-        for filename in files:
-            if os.path.isfile("%s/%s" % (get_code_folder_name(), filename)):
-                mod_filename = "%s/%s" % (get_code_folder_name(), filename)
-            elif os.path.isfile("%s/%s" % (BOOTSTRAP_FOLDER_NAME, filename)):
-                mod_filename = "%s/%s" % (BOOTSTRAP_FOLDER_NAME, filename)
-            else:
-                raise Exception("File not found: %s" % filename)
-            mod_files.append(mod_filename)
-
-        to_compile = to_recompile(address, mod_files)
-
-        for mod_filename in to_compile:
-            if PARALLEL_PUSH:
-                import threading
-                threads.append(
-                    threading.Thread(
-                        target=compile_file,
-                        args=[address, mod_filename, mod_filename, mode, proc]))
-                threads[-1].start()
-            else:
-                compile_file(address, mod_filename, mod_filename, mode, proc)
-
-        if PARALLEL_PUSH:
-            for t in threads:
-                t.join()
-
-        if mode[-1] == "O":
-            # Fire up the linker
-            val = execute("link_and_load", [address, USERNAME] + mod_files, wait=True)
-            if val != 0:
-                raise Exception("Linking error")
+        # Compile, push and link the source code files.
+        compile_files(address, proc, files, mode)
 
 
         # Send the request ...
         # Send the request ...
         set_input_data(address, parameters)
         set_input_data(address, parameters)
@@ -248,17 +262,71 @@ def run_file_single_output(files, parameters, mode, optimization_level=None):
        and then collects and returns a single output."""
        and then collects and returns a single output."""
     return run_file_fixed_output_count(files, parameters, mode, 1, optimization_level)[0]
     return run_file_fixed_output_count(files, parameters, mode, 1, optimization_level)[0]
 
 
+def mean(values):
+    """Computes the arithmetic mean of the given values."""
+    return float(sum(values)) / max(len(values), 1)
+
 def run_perf_test(files, parameters, optimization_level, n_iterations=1):
 def run_perf_test(files, parameters, optimization_level, n_iterations=1):
     """Compiles the given sequence of files, feeds them the given input in the given mode,
     """Compiles the given sequence of files, feeds them the given input in the given mode,
        and then collects their output. This process is repeated n_iterations times. The
        and then collects their output. This process is repeated n_iterations times. The
-       return value is the average of all outputs."""
-    result = 0.0
+       return value is the average of all outputs, along with the mean total run-time."""
+    test_runtimes = []
+    total_runtimes = []
     for _ in xrange(n_iterations):
     for _ in xrange(n_iterations):
-        result += float(
-            run_file_single_output(
-                files, parameters + [0], 'CO',
-                optimization_level)) / float(n_iterations)
-    return result
+        start_time = time.time()
+        test_time = run_file_single_output(
+            files, parameters + [0], 'CO',
+            optimization_level)
+        end_time = time.time()
+        total_time = end_time - start_time
+        test_runtimes.append(test_time)
+        total_runtimes.append(total_time)
+    return mean(test_runtimes), mean(total_runtimes)
+
+def get_expectation_checks(expected_values):
+    """Converts the given sequence of expected values to a sequence of functions which tell
+       if an input is allowed. Every function is accompanied by an expected value."""
+    def get_single_expectation_checks(expectation):
+        """Gets an expectation checker for a single expected value."""
+        if isinstance(expectation, set):
+            # We expect to receive a number of outputs equal to the size of the set, but their
+            # order does not matter.
+            for _ in xrange(len(expectation)):
+                yield (lambda val: val in expectation), expectation
+        elif expectation is None:
+            # Skip output value
+            yield (lambda _: True), expectation
+        else:
+            yield (lambda val: val == expectation), expectation
+
+    for expectation in expected_values:
+        for checker in get_single_expectation_checks(expectation):
+            yield checker
+
+def run_correctness_test(files, parameters, expected, optimization_level):
+    """Compiles the given sequence of files, feeds them the given input in the given mode,
+       and then compares the output with the expected output. The return value is the total
+       run-time of the test."""
+    checks = get_expectation_checks(expected)
+    def handle_output(output):
+        """Checks the given output against the expected output."""
+        try:
+            check, expectation = next(checks)
+        except StopIteration:
+            return False
+
+        print("Got %s, expect %s" % (output, expectation))
+        assert check(output)
+
+        return True
+
+    start_time = time.time()
+    try:
+        run_file(files, parameters, 'CO', handle_output, optimization_level)
+    except ModelverseTerminated:
+        return
+    end_time = time.time()
+    return end_time - start_time
 
 
 def format_output(output):
 def format_output(output):
     """Formats the output of `run_file_to_completion` as a string."""
     """Formats the output of `run_file_to_completion` as a string."""
@@ -277,20 +345,56 @@ def define_perf_tests(target_class, test_function):
     for optimization_level in ALL_OPTIMIZATION_LEVELS:
     for optimization_level in ALL_OPTIMIZATION_LEVELS:
         define_perf_test(target_class, test_function, optimization_level)
         define_perf_test(target_class, test_function, optimization_level)
 
 
+def get_model_constructor(code):
+    # First change multiple spaces to a tab
+    code_fragments = code.split("\n")
+    code_fragments = [i for i in code_fragments if i.strip() != ""]
+    code_fragments = [i.replace("    ", "\t") for i in code_fragments]
+    initial_tabs = min([len(i) - len(i.lstrip("\t")) for i in code_fragments])
+    code_fragments = [i[initial_tabs:] for i in code_fragments]
+    code = "\n".join(code_fragments)
+
+    with open("__model.mvc", "w") as f:
+        f.write(code)
+        f.flush()
+
+    constructors = do_compile("__model.mvc", "interface/HUTN/grammars/modelling.g", "M") + ["exit"]
+
+    return constructors
+
 DEFAULT_PERF_FILE_NAME = 'perf_data.txt'
 DEFAULT_PERF_FILE_NAME = 'perf_data.txt'
 
 
-def write_perf_to_file(test_name, optimization_level, result, file_name=DEFAULT_PERF_FILE_NAME):
+TOTAL_TIME_QUANTITY = 'total-runtime'
+TEST_TIME_QUANTITY = 'test-runtime'
+
+def write_perf_entry_to_stream(
+        test_name, optimization_level, quantity,
+        result, output_stream):
+    """Writes a performance measurement entry to the given stream."""
+    output_stream.write('%s:%s:%s:%f\n' % (test_name, optimization_level, quantity, result))
+
+def write_perf_to_file(
+        test_name, optimization_level, runtimes, file_name=DEFAULT_PERF_FILE_NAME):
     """Writes performance data to a file."""
     """Writes performance data to a file."""
+    test_runtime, total_runtime = runtimes
+    with open(file_name, "a") as perf_file:
+        write_perf_entry_to_stream(
+            test_name, optimization_level, TEST_TIME_QUANTITY, test_runtime, perf_file)
+        write_perf_entry_to_stream(
+            test_name, optimization_level, TOTAL_TIME_QUANTITY, total_runtime, perf_file)
+
+def write_total_runtime_to_file(
+        test_name, optimization_level, total_runtime, file_name=DEFAULT_PERF_FILE_NAME):
+    """Writes a total runtime entry to a file."""
     with open(file_name, "a") as perf_file:
     with open(file_name, "a") as perf_file:
-        perf_file.write('%s:%s:%f\n' % (test_name, optimization_level, result))
+        write_perf_entry_to_stream(
+            test_name, optimization_level, TOTAL_TIME_QUANTITY, total_runtime, perf_file)
 
 
 def parse_perf_data(file_name):
 def parse_perf_data(file_name):
     """Parses the performance data in the given file."""
     """Parses the performance data in the given file."""
-    results = {}
+    results = defaultdict(lambda: defaultdict(list))
     with open(file_name, 'r') as perf_file:
     with open(file_name, 'r') as perf_file:
         for line in perf_file.readlines():
         for line in perf_file.readlines():
-            test_name, optimization_level, result = line.strip().split(':')
-            if optimization_level not in results:
-                results[optimization_level] = []
-            results[optimization_level].append((test_name, result))
+            test_name, optimization_level, quantity, result = line.strip().split(':')
+            results[quantity][optimization_level].append((test_name, result))
     return sorted(results.items(), key=operator.itemgetter(1))
     return sorted(results.items(), key=operator.itemgetter(1))