PyPDEVS-and-JADEVS-DEVStone-benchmarks/benchmarks/mean.py

import argparse
from pathlib import Path
import numpy as np
import pandas as pd

BASE_DIR = Path(__file__).resolve().parent

ENGINES = {
    "agents": "Results/Agents/Time/{model}_{type}.csv",
    "pypdevs-minimal": "Results/PyPDEVS/Time/minimal/{model}_{type}.csv",
    "pypdevs-classic": "Results/PyPDEVS/Time/classic/{model}_{type}.csv",
    "pypdevs-parallel": "Results/PyPDEVS/Time/parallel/{model}_{type}.csv",
    "pypdevs-dynamic": "Results/PyPDEVS/Time/dynamic/{model}_{type}.csv",
    "pypdevs-dynamic-classic": "Results/PyPDEVS/Time/dynamic-classic/{model}_{type}.csv",
}

# Configured dimensions per model scenario
MODEL_DIMENSIONS = {
    "LI": {"depth": 200, "width": 200},
    "HI": {"depth": 200, "width": 200},
    "HO": {"depth": 200, "width": 200},
    "HOmod": {"depth": 40, "width": 40},
}

MODELS = ["LI", "HI", "HO", "HOmod"]
TRANSLATIONS = ["baseline", "scaled", "accessed"]


def format_to_latex(mean, std):
    if pd.isna(mean) or pd.isna(std):
        return "-"
    if std == 0:
        return f"${mean:.3f} \\pm 0.000$"

    std_magnitude = int(np.floor(np.log10(abs(std))))
    decimal_places = -std_magnitude

    if decimal_places >= 0:
        return f"${mean:.{decimal_places}f} \\pm {std:.{decimal_places}f}$"
    else:
        return f"${round(mean, decimal_places):.0f} \\pm {round(std, decimal_places):.0f}$"


def get_metrics(engine, model, data_type, custom_depth=None, custom_width=None):
    """Dynamically locates file, removes configuration-specific outliers using the IQR method

    from compare.py, and computes the final mean/std.
    """
    template = ENGINES.get(engine)
    if not template:
        return None, None

    file_path = BASE_DIR / template.format(model=model, type=data_type)
    if not file_path.exists():
        return None, None

    depth = (
        custom_depth
        if custom_depth is not None
        else MODEL_DIMENSIONS[model]["depth"]
    )
    width = (
        custom_width
        if custom_width is not None
        else MODEL_DIMENSIONS[model]["width"]
    )

    try:
        df = pd.read_csv(file_path)

        # 1. Compute total_time exactly like clean_data(df) in compare.py
        df["total_time"] = (
            df["simulation_time"] + df["creation_time"] + df["setup_time"]
        )

        # 2. Identify outlier thresholds per unique simulation grouping parameter context
        group_params = [
            c for c in ["depth", "width", "number"] if c in df.columns
        ]

        if group_params:
            q1 = df.groupby(group_params)["total_time"].transform(
                lambda x: x.quantile(0.25)
            )
            q3 = df.groupby(group_params)["total_time"].transform(
                lambda x: x.quantile(0.75)
            )

            iqr = q3 - q1
            lower = q1 - 1.5 * iqr
            upper = q3 + 1.5 * iqr

            # 3. Filter out rows matching outlier bounds
            df = df[(df["total_time"] >= lower) & (df["total_time"] <= upper)]

        # 4. Extract specific dimension rows for the requested table metric cell
        filtered_df = df[(df["depth"] == depth) & (df["width"] == width)]
        if filtered_df.empty:
            return None, None

        combined_time = filtered_df["total_time"]
        return combined_time.mean(), combined_time.std()
    except Exception:
        return None, None

def get_dual_metrics(engine, model, data_type, custom_depth=None, custom_width=None):
    """
    Locates file, removes outliers using the IQR method based on total_time,
    and returns both the Total (sum) and Sim-only metrics.
    """
    template = ENGINES.get(engine)
    if not template:
        return (None, None), (None, None)

    file_path = BASE_DIR / template.format(model=model, type=data_type)
    if not file_path.exists():
        return (None, None), (None, None)

    depth = custom_depth if custom_depth is not None else MODEL_DIMENSIONS[model]["depth"]
    width = custom_width if custom_width is not None else MODEL_DIMENSIONS[model]["width"]

    try:
        df = pd.read_csv(file_path)

        df["total_time"] = df["simulation_time"] + df["creation_time"] + df["setup_time"]

        group_params = [c for c in ["depth", "width", "number"] if c in df.columns]

        if group_params:
            q1 = df.groupby(group_params)["total_time"].transform(lambda x: x.quantile(0.25))
            q3 = df.groupby(group_params)["total_time"].transform(lambda x: x.quantile(0.75))

            iqr = q3 - q1
            lower = q1 - 1.5 * iqr
            upper = q3 + 1.5 * iqr

            df = df[(df["total_time"] >= lower) & (df["total_time"] <= upper)]

        filtered_df = df[(df["depth"] == depth) & (df["width"] == width)]
        if filtered_df.empty:
            return (None, None), (None, None)

        tot_mean, tot_std = filtered_df["total_time"].mean(), filtered_df["total_time"].std()
        sim_mean, sim_std = filtered_df["simulation_time"].mean(), filtered_df["simulation_time"].std()

        return (tot_mean, tot_std), (sim_mean, sim_std)
    except Exception:
        return (None, None), (None, None)

def handle_single(args):
    """Computes a single measurement point using manual overrides or rules if omitted."""
    depth = args.depth if args.depth is not None else MODEL_DIMENSIONS[args.model]["depth"]
    width = args.width if args.width is not None else MODEL_DIMENSIONS[args.model]["width"]

    mean_val, std_val = get_metrics(
        args.engine, args.model, args.type, custom_depth=depth, custom_width=width
    )
    if mean_val is None:
        print(f"❌ Data matching filters (Depth={depth}, Width={width}) could not be resolved.")
        return

    print(f"--- Matrix Point Verification ---")
    print(f"Target: {args.engine} | {args.model}_{args.type}.csv")
    print(f"Filter: Depth={depth} | Width={width}")
    print(f"Raw Mean: {mean_val:.6f} | Raw Std: {std_val:.6f}")
    print(f"LaTeX Output: {format_to_latex(mean_val, std_val)}")


def handle_comparison(args):
    """Generates the engine cross-comparison performance matrix table with fixed dimensions."""
    print("Generating Engine Performance Matrix Table...\n")

    table_data = {}
    for engine in ENGINES.keys():
        row_cells = []
        for model in MODELS:
            mean_val, std_val = get_metrics(engine, model, "baseline")
            row_cells.append(format_to_latex(mean_val, std_val))
        table_data[engine] = row_cells

    labels = {
        "pypdevs-minimal": "PyPDEVS (m)",
        "pypdevs-classic": "PyPDEVS (c)",
        "pypdevs-parallel": "PyPDEVS (p)",
        "pypdevs-dynamic": "PyPDEVS (d)",
        "pypdevs-dynamic-classic": "PyPDEVS (dc)",
        "agents": "Agents.jl",
    }

    latex_table = (
        "\\begin{table}[htbp]\n"
        "    \\centering\n"
        "    \\begin{tabular}{lcccc}\n"
        "        \\toprule\n"
        "        \\textbf{Engine} & {\\textbf{LI (200x200)}} & {\\textbf{HI (200x200)}} & {\\textbf{HO (200x200)}} & {\\textbf{HOmod (40x40)}} \\\\\n"
        "        \\midrule\n"
    )
    for eng in ENGINES.keys():
        print(eng)
        latex_table += f"        {labels[eng]} & {' & '.join(table_data[eng])} \\\\\n"

    latex_table += (
        "        \\bottomrule\n"
        "    \\end{tabular}\n"
        "    \\caption{Performance comparison of different DEVS simulation engines in seconds.}\n"
        "    \\label{tab:devstone-performance}\n"
        "\\end{table}"
    )
    print(latex_table)


def handle_devstone_comparison(args):
    """Generates the dual-metric comparison table (Total vs. Sim)."""
    print("Generating Dual-Metric Engine Comparison Table...\n")

    labels = {
        "pypdevs-minimal": "PyPDEVS (m)",
        "pypdevs-classic": "PyPDEVS (c)",
        "pypdevs-parallel": "PyPDEVS (p)",
        "pypdevs-dynamic": "PyPDEVS (d)",
        "pypdevs-dynamic-classic": "PyPDEVS (dc)",
        "agents": "Agents.jl",
    }

    latex_table = (
        "\\begin{table}[htbp]\n"
        "    \\centering\n"
        "    \\begin{tabular}{llcccc}\n"
        "        \\toprule\n"
        "        \\textbf{Engine} & \\textbf{Phase} & {\\textbf{LI (200x200)}} & {\\textbf{HI (200x200)}} & {\\textbf{HO (200x200)}} & {\\textbf{HOmod (40x40)}} \\\\\n"
        "        \\midrule\n"
    )

    for eng in ENGINES.keys():
        tot_cells = []
        sim_cells = []
        for model in MODELS:
            tot_metrics, sim_metrics = get_dual_metrics(eng, model, "baseline")

            if tot_metrics[0] is not None:
                tot_cells.append(format_to_latex(tot_metrics[0], tot_metrics[1]))
                sim_cells.append(format_to_latex(sim_metrics[0], sim_metrics[1]))
            else:
                tot_cells.append("-")
                sim_cells.append("-")

        latex_table += f"        \\multirow{{2}}{{*}}{{{labels[eng]}}} & Total & {' & '.join(tot_cells)} \\\\\n"
        latex_table += f"         & Sim. Only & {' & '.join(sim_cells)} \\\\\n"
        latex_table += "        \\midrule\n"

    latex_table = latex_table.rstrip("\\midrule\n") + "\\bottomrule\n"

    latex_table += (
        "    \\end{tabular}\n"
        "    \\caption{Comparison of total execution time versus isolated simulation execution time in seconds.}\n"
        "    \\label{tab:devstone-dual-comparison}\n"
        "\\end{table}"
    )
    print(latex_table)


def handle_translation(args):
    """Generates two separate cross-functional Translation strategy tables with fixed dimensions."""
    print("Generating Comparative Translation Function Tables...\n")

    engine_targets = ["pypdevs-parallel", "agents"]
    engine_labels = {
        "pypdevs-parallel": "PyPDEVS Parallel",
        "agents": "Agents.jl",
    }

    labels = {
        "baseline": "Identity (Baseline)",
        "scaled": "Scaled             ",
        "accessed": "Accessed           ",
    }

    for engine in engine_targets:
        print(f"% --- TRANSLATION TABLE FOR {engine_labels[engine].upper()} ---")
        latex_table = (
            "\\begin{table}[htbp]\n"
            "    \\centering\n"
            "    \\begin{tabular}{lcccc}\n"
            "        \\toprule\n"
            "        \\textbf{Translation Function} & {\\textbf{LI (200x200)}} & {\\textbf{HI (200x200)}} & {\\textbf{HO (200x200)}} & {\\textbf{HOmod (40x40)}} \\\\\n"
            "        \\midrule\n"
        )
        for tx in TRANSLATIONS:
            row_cells = []
            for model in MODELS:
                mean_val, std_val = get_metrics(engine, model, tx)
                row_cells.append(format_to_latex(mean_val, std_val))
            latex_table += f"        {labels[tx]} & {' & '.join(row_cells)} \\\\\n"

        latex_table += (
            "        \\bottomrule\n"
            "    \\end{tabular}\n"
            f"    \\caption{{Performance comparison of translation functions under {engine_labels[engine]} in seconds.}}\n"
            f"    \\label{{tab:translation-performance-{engine.replace('-', '_')}}}\n"
            "\\end{table}\n"
        )
        print(latex_table)

def main():
    parser = argparse.ArgumentParser(
        description="DEVS Benchmarking Performance Statistics Aggregator CLI"
    )
    subparsers = parser.add_subparsers(dest="command", required=True, help="Available actions")

    p_single = subparsers.add_parser("single", help="Calculate point result for individual matrix logs.")
    p_single.add_argument("-e", "--engine", choices=["agents", "pypdevs-min", "pypdevs-parallel"], required=True)
    p_single.add_argument("-m", "--model", choices=MODELS, required=True)
    p_single.add_argument("-t", "--type", choices=TRANSLATIONS, default="baseline")
    p_single.add_argument("-d", "--depth", type=int, default=None, help="Force override depth constraint")
    p_single.add_argument("-w", "--width", type=int, default=None, help="Force override width constraint")

    subparsers.add_parser("comparison", help="Generate Engine Comparison Table (Fixed sizes automatically applied).")
    subparsers.add_parser("devstone-comparison", help="Generate Engine Comparison Table showing Total and Sim times separately.")
    subparsers.add_parser("translation", help="Generate Translation Strategy Tables (Fixed sizes automatically applied).")

    args = parser.parse_args()

    if args.command == "single":
        handle_single(args)
    elif args.command == "comparison":
        handle_comparison(args)
    elif args.command == "devstone-comparison":
        handle_devstone_comparison(args)
    elif args.command == "translation":
        handle_translation(args)


if __name__ == "__main__":
    main()