Works for aDEVS, Cadmium v2, and xDEVS

.gitignore

@@ -1,5 +1,4 @@
 .idea/
-.log
 
 cmake-build-debug/
 build/
@@ -10,3 +9,6 @@ bin/
 __pycache__
 
 .python-version
+
+*.csv
+*.log

Dockerfile

@@ -1,28 +1,28 @@
-FROM ubuntu:18.04
+FROM ubuntu:22.04
 
 RUN apt-get update && apt-get install -y \
   build-essential \
   cmake \
+  curl \
   git \
   openjdk-11-jdk-headless \
   libboost-all-dev \
-  python3.6 \
-  python3-pip
-
+  python3.8 \
+  python3-pip \
 
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+RUN rustup update
 RUN pip3 install setuptools flask
 
 COPY .git/ /root/devstone_comparative/.git/
 COPY devstone/ /root/devstone_comparative/devstone/
+COPY simulators/ /root/devstone_comparative/simulators/
 COPY web_service/ /root/devstone_comparative/web_service/
 COPY events_devstone.txt clean.sh setup.sh devstone_comparative.py .gitmodules /root/devstone_comparative/
 
-RUN pip3 install setuptools flask
-
 WORKDIR /root/devstone_comparative
 RUN ./clean.sh
 RUN ./setup.sh
 
 RUN python3 web_service/manage.py -i -a test
 ENTRYPOINT python3 web_service/web_service.py
-

devstone_comparative.py

@@ -1,3 +1,4 @@
+from __future__ import annotations
 import argparse
 import csv
 import re
@@ -9,9 +10,7 @@ import time
 DEFAULT_PARAMS = ((300, 10, 0, 0), (10, 300, 0, 0), (300, 300, 0, 0))
 DEFAULT_MODEL_TYPES = ("LI", "HI", "HO", "HOmod")
 DEFAULT_MAX_TIME = 1e10
-DEFAULT_NUM_REPS = 10
-
-RE_SIM_TIMES = r"Model creation time: ?([0-9.e-]+) ?.*Engine set ?up time: ?([0-9.e-]+) ?.*Simulation time: ?([0-9.e-]+)"
+DEFAULT_NUM_REPS = 30
 
 COMMANDS = {
     "adevs": "devstone/adevs/bin/devstone {model_type} {width} {depth} {int_cycles} {ext_cycles}",
@@ -34,20 +33,32 @@ COMMANDS = {
     },
     "xdevs": {
         "c": "simulators/xdevs.c/examples/devstone/devstone -w {width} -d {depth} -b {model_type} -m 1",
-        "cpp": "simulators/xdevs-c++/src/xdevs/examples/DevStone/DevStone -w {width} -d {depth} -b {model_type} -m 1",
+        "cpp": "simulators/xdevs.cpp/src/xdevs/examples/DevStone/DevStone -w {width} -d {depth} -b {model_type} -m 1",
         # "go": ,  # TODO add this
         "java": {
-            "sequential": "java -cp simulators/xdevs.java/target/xdevs-2.0.1-jar-with-dependencies.jar xdevs.lib.performance.DevStoneSimulation --model={model_type} --width={width} --depth={depth} --delay-distribution=Constant-{int_cycles} --coordinator=Coordinator",
-            "parallel": "java -cp simulators/xdevs.java/target/xdevs-2.0.1-jar-with-dependencies.jar xdevs.lib.performance.DevStoneSimulation --model={model_type} --width={width} --depth={depth} --delay-distribution=Constant-{int_cycles} --coordinator=CoordinatorParallel",
+            "sequential": "java -cp simulators/xdevs.java/target/xdevs-2.0.3-jar-with-dependencies.jar xdevs.lib.performance.DevStoneSimulation --model={model_type} --width={width} --depth={depth} --delay-distribution=Constant-{int_cycles} --coordinator=Coordinator",
+            "parallel": "java -cp simulators/xdevs.java/target/xdevs-2.0.3-jar-with-dependencies.jar xdevs.lib.performance.DevStoneSimulation --model={model_type} --width={width} --depth={depth} --delay-distribution=Constant-{int_cycles} --coordinator=CoordinatorParallel",
         },
-        "py": "python3 simulators/xdevs.py/xdevs/examples/devstone/main.py -m {model_type} -d {depth} -w {width} -i {int_cycles} -e {ext_cycles}",  # TODO
-        "rs": "cargo run --release --manifest-path simulators/xdevs.rs/Cargo.toml {model_type} {width} {depth} {int_cycles} {ext_cycles}",
+        "py": "python3 simulators/xdevs.py/xdevs/examples/devstone/devstone.py {model_type} {width} {depth} {int_cycles} {ext_cycles}",
+        "rs": "simulators/xdevs.rs/target/release/xdevs {model_type} {width} {depth} {int_cycles} {ext_cycles}",
     },
 }
 
+DEFAULT_RE = r"model creation time(?:[ \(s\)]*): ?([0-9.e-]+)([ nuµms]*).*(?:engine|simulator) (?:set ?up|creation) time(?:[ \(s\)]*): ?([0-9.e-]+)([ nuµms]*).*simulation time(?:[ \(s\)]*): ?([0-9.e-]+)([ nuµms]*)"
+
+CUSTOM_RE = {}
+
+
+def serialize_simengines(res: list[str], prefix: str, flavors: str | dict):
+    if isinstance(flavors, str):
+        res.append(prefix)
+    else:
+        for key, val in flavors.items():
+            new_prefix = f'{prefix}-{key}' if prefix else key
+            serialize_simengines(res, new_prefix, val)
+
 def parse_args():
     parser = argparse.ArgumentParser(description='Script to compare DEVStone implementations with different engines')
-
     parser.add_argument('-m', '--model-types', help='DEVStone model type (LI, HI, HO, HOmod)')
     parser.add_argument('-w', '--width', type=int, help='Width of each coupled model.')
     parser.add_argument('-d', '--depth', type=int, help='Number of recursive levels of the model.')
@@ -64,55 +75,104 @@ def parse_args():
     if args.model_types:
         args.model_types = [x.strip() for x in args.model_types.split(",")]
     else:
-        args.model_types = DEFAULT_MODEL_TYPES
+        args.model_types = list(DEFAULT_MODEL_TYPES)
 
+    params = []
     if args.params:
-        args.params = [x.strip().split("-") for x in args.params.split(",")]
+        params = [x.strip().split("-") for x in args.params.split(",")]
+        # args.params = [x.strip().split("-") for x in args.params.split(",")]
     elif args.depth and args.width:
         int_cycles = args.int_cycles or 0
         ext_cycles = args.ext_cycles or 0
-        args.params = ((args.depth, args.width, int_cycles, ext_cycles),)
-
+        params = [(args.depth, args.width, int_cycles, ext_cycles)]
+        # args.params = ((args.depth, args.width, int_cycles, ext_cycles),)
     else:
-        args.params = DEFAULT_PARAMS
+        params = list(DEFAULT_PARAMS)
+    args.params = []
+    for param in params:
+        if len(param) == 5:
+            args.params.append(param)
+        elif len(param) == 4:
+            for model in args.model_types:
+                args.params.append((model, *param))
+        else:
+            raise ValueError(f'invalid number of params ({param})')
 
     if args.include_engines:
-        args.include_engines = args.include_engines.split(",")
-        for engine in args.include_engines:
-            if engine not in engines:
-                raise RuntimeError("%s is not in the engines list" % engine)
+        args.include_engines = args.include_engines.split(',')
+        # TODO if an engine has more than one flavor, serialize here (e.g., xdevs -> all the xdevs stuff)
     else:
-        args.include_engines = engines.keys()
-
+        args.include_engines = []
+        serialize_simengines(args.include_engines, '', COMMANDS)
     if args.exclude_engines:
-        excluded = args.exclude_engines.split(",")
-        args.include_engines = [x for x in args.include_engines if x not in excluded]
+        args.include_engines = [x for x in args.include_engines if x not in args.exclude_engines.split(",")]
+
+    args.commands = {}
+    args.regex = {}
+    for eng in args.include_engines:
+        cmd = COMMANDS
+        regex = CUSTOM_RE
+        for s in eng.split('-'):
+            cmd = cmd.get(s, dict())
+            if not isinstance(regex, str):
+                regex = regex.get(s, DEFAULT_RE)
+        if not cmd:
+            raise RuntimeError(f'unknown simulation engine {eng}')
+        if isinstance(cmd, dict):
+            # TODO instead of this, serialize all the flavors of the engine
+            raise RuntimeError(f'There are more than one flavor for the engine {eng}')
+        else:
+            args.commands[eng] = cmd
+            args.regex[eng] = regex
 
     if not args.num_rep:
         args.num_rep = DEFAULT_NUM_REPS
 
     if not args.out_file:
-        args.out_file = "devstone_%de_%dp_%d.csv" % (len(engines), len(args.params), int(time.time()))
+        args.out_file = "devstone_%de_%dp_%d.csv" % (len(args.include_engines), len(args.params), int(time.time()))
 
     return args
 
-def execute_cmd(cmd, csv_writer):
+
+def execute_cmd(cmd, regex, csv_writer):
     # Execute simulation
     try:
-        result = subprocess.run(engine_cmd_f.split(), stdout=subprocess.PIPE)
+        result = subprocess.run(cmd.split(), stdout=subprocess.PIPE)
     except Exception as e:
-        print("%s: Error executing simulation." % engine)
+        print(f"{engine}: Error executing simulation. ({str(e)})")
         return
 
     # Read data from output
-    found = re.search(RE_SIM_TIMES, str(result.stdout))
-
+    res_out = result.stdout.decode('UTF-8').strip()
+    if not res_out:
+        res_out = str(result.stderr).strip()
+    res_out.replace("\n", " ")
+    found = re.search(regex, res_out, flags=re.IGNORECASE | re.DOTALL)
     if not found:
-        print("%s: Simulation execution times could not be extracted." % engine)
+        print(f"{engine}: Simulation execution times could not be extracted.")
         print(result.stdout)
         return
-
-    model_time, engine_time, sim_time = tuple(map(float, found.groups()))
+    groups = found.groups()
+    if len(groups) == 3:
+        model_time, engine_time, sim_time = tuple(map(float, found.groups()))
+    elif len(groups) == 6:
+        times = list(map(float, (groups[0], groups[2], groups[4])))
+        units = (groups[1], groups[3], groups[5])
+        for i in range(3):
+            if units[i]:
+                unit = units[i].strip()
+                if unit.startswith('s'):
+                    pass
+                elif unit.startswith('ms'):
+                    times[i] *= 1e-3
+                elif unit.startswith('us') or unit.startswith('µs'):
+                    times[i] *= 1e-6
+                elif unit.startswith('ns'):
+                    times[i] *= 1e-9
+                else:
+                    raise Exception(f"unknown time units: {unit}")
+        times = tuple(times)
+        model_time, engine_time, sim_time = times
     total_time = sum((model_time, engine_time, sim_time))
 
     # Write results into output file
@@ -126,36 +186,21 @@ if __name__ == "__main__":
 
     args = parse_args()
 
-    if not engines:
+    if not args.include_engines:
         raise RuntimeError("No engines were selected.")
 
     with open(args.out_file, "w") as csv_file:
         csv_writer = csv.writer(csv_file, delimiter=';')
         csv_writer.writerow(("engine", "iter", "model", "depth", "width", "int_delay", "ext_delay", "model_time", "runner_time", "sim_time", "total_time"))
 
-        for engine in args.include_engines:
-            engine_cmd = engines[engine]
-
-            if len(args.params[0]) == 4:
-                    for model_type in args.model_types:
-                        for depth, width, int_cycles, ext_cycles in args.params:
-                            engine_cmd_f = engine_cmd.format(model_type=model_type, depth=depth, width=width, int_cycles=int_cycles, ext_cycles=ext_cycles)
-                            for i_exec in range(args.num_rep):
-
-                                if not engine_cmd_f:
-                                    continue
-
-                                print(engine_cmd_f)
-                                execute_cmd(engine_cmd_f, csv_writer)
-
-            elif len(args.params[0]) == 5:
-                for model_type, depth, width, int_cycles, ext_cycles in args.params:
-                    engine_cmd_f = engine_cmd.format(model_type=model_type, depth=depth, width=width, int_cycles=int_cycles,
-                                                     ext_cycles=ext_cycles)
-                    for i_exec in range(args.num_rep):
-
-                        if not engine_cmd_f:
-                            continue
-
-                        print(engine_cmd_f)
-                        execute_cmd(engine_cmd_f, csv_writer)
+        for engine, engine_cmd in args.commands.items():
+            engine_regex = args.regex[engine]
+            for params in args.params:
+                model_type, depth, width, int_cycles, ext_cycles = params
+                engine_cmd_f = engine_cmd.format(model_type=model_type, depth=depth, width=width,
+                                                 int_cycles=int_cycles, ext_cycles=ext_cycles)
+                if not engine_cmd_f:
+                    continue
+                for i_exec in range(args.num_rep):
+                    print(f'({i_exec}) {engine_cmd_f}')
+                    execute_cmd(engine_cmd_f, engine_regex, csv_writer)

setup.sh

@@ -20,7 +20,7 @@ git checkout 4a5b9cf7b6498e88e760dfba177a761240999955
 # TODO xDEVS Go
 # Set up xDEVS Java repository
 cd ../xdevs.java
-git checkout 9482cf31a873b63f529aa328e913ade9e8edad55
+git checkout 94b7d15ab3e79220e8145f9576e69948db4b4fb6
 # Set up xDEVS Python repository
 cd ../xdevs.py
 git checkout b53a6f170350af8296af43d1b2334e173e95990c

simulators/xdevs.java

@@ -1 +1 @@
-Subproject commit 9482cf31a873b63f529aa328e913ade9e8edad55
+Subproject commit 94b7d15ab3e79220e8145f9576e69948db4b4fb6