Add capturing of mqtt traces of real and sim at runtime. Logs to file.

.gitignore

@@ -179,3 +179,4 @@ cython_debug/
 /.idea/misc.xml
 /.idea/modules.xml
 /.idea/vcs.xml
+.idea/webResources.xml

flowchart.md

@@ -1,269 +0,0 @@
-```mermaid
----
-title: FischertechnikFactory
----
-flowchart LR
-	subgraph Generator
-		Generator_out("out")
-		Generator_mqtt_in("mqtt_in")
-		Generator_mqtt_in ~~~ Generator_out
-	end
-	Generator_out --> DSI_inp
-	subgraph DSI
-		DSI_inp("inp")
-		DSI_out("out")
-		DSI_mqtt_out("mqtt_out")
-		DSI_mqtt_in("mqtt_in")
-		DSI_vgr_in("vgr_in")
-		DSI_inp ~~~ DSI_out
-		DSI_mqtt_in ~~~ DSI_mqtt_out
-	end
-	DSI_out --> VacuumGripper_dsi_in
-	DSI_mqtt_out --> MQTTControlUnit_mqtt_in
-	subgraph ReaderStation
-		ReaderStation_nfc_out("nfc_out")
-		ReaderStation_color_in("color_in")
-		ReaderStation_color_out("color_out")
-		ReaderStation_mqtt_out("mqtt_out")
-		ReaderStation_nfc_in("nfc_in")
-		ReaderStation_mqtt_in("mqtt_in")
-		ReaderStation_nfc_in ~~~ ReaderStation_nfc_out
-		ReaderStation_color_in ~~~ ReaderStation_color_out
-		ReaderStation_mqtt_in ~~~ ReaderStation_mqtt_out
-	end
-	ReaderStation_nfc_out --> VacuumGripper_nfc_in
-	ReaderStation_color_out --> VacuumGripper_color_sensor_in
-	ReaderStation_mqtt_out --> MQTTControlUnit_mqtt_in
-	subgraph VacuumGripper
-		VacuumGripper_sld_white_out("sld_white_out")
-		VacuumGripper_sld_white_in("sld_white_in")
-		VacuumGripper_mqtt_in("mqtt_in")
-		VacuumGripper_dsi_out("dsi_out")
-		VacuumGripper_ct_out("ct_out")
-		VacuumGripper_sld_red_in("sld_red_in")
-		VacuumGripper_mqtt_out("mqtt_out")
-		VacuumGripper_nfc_out("nfc_out")
-		VacuumGripper_color_sensor_in("color_sensor_in")
-		VacuumGripper_sld_blue_in("sld_blue_in")
-		VacuumGripper_dso_out("dso_out")
-		VacuumGripper_nfc_in("nfc_in")
-		VacuumGripper_color_sensor_out("color_sensor_out")
-		VacuumGripper_mpo_out("mpo_out")
-		VacuumGripper_ct_in("ct_in")
-		VacuumGripper_dsi_in("dsi_in")
-		VacuumGripper_sld_red_out("sld_red_out")
-		VacuumGripper_sld_blue_out("sld_blue_out")
-		VacuumGripper_dsi_in ~~~ VacuumGripper_dsi_out
-		VacuumGripper_ct_in ~~~ VacuumGripper_ct_out
-		VacuumGripper_nfc_in ~~~ VacuumGripper_nfc_out
-		VacuumGripper_color_sensor_in ~~~ VacuumGripper_color_sensor_out
-		VacuumGripper_sld_red_in ~~~ VacuumGripper_mpo_out
-		VacuumGripper_sld_blue_in ~~~ VacuumGripper_sld_red_out
-		VacuumGripper_sld_white_in ~~~ VacuumGripper_sld_blue_out
-		VacuumGripper_mqtt_in ~~~ VacuumGripper_sld_white_out
-	end
-	VacuumGripper_dsi_out --> DSI_vgr_in
-	VacuumGripper_ct_out --> Transporter_right_in
-	VacuumGripper_nfc_out --> ReaderStation_nfc_in
-	VacuumGripper_color_sensor_out --> ReaderStation_color_in
-	VacuumGripper_mpo_out --> MPO_vgr_in
-	VacuumGripper_sld_red_out --> SLD_red_in
-	VacuumGripper_sld_blue_out --> SLD_blue_in
-	VacuumGripper_sld_white_out --> SLD_white_in
-	VacuumGripper_dso_out --> DSO_inp
-	VacuumGripper_mqtt_out --> MQTTControlUnit_mqtt_in
-	subgraph Transporter
-		Transporter_left_in("left_in")
-		Transporter_right_out("right_out")
-		Transporter_mqtt_in("mqtt_in")
-		Transporter_mqtt_out("mqtt_out")
-		Transporter_right_in("right_in")
-		Transporter_left_out("left_out")
-		Transporter_left_in ~~~ Transporter_right_out
-		Transporter_right_in ~~~ Transporter_left_out
-		Transporter_mqtt_in ~~~ Transporter_mqtt_out
-	end
-	Transporter_right_out --> VacuumGripper_ct_in
-	Transporter_left_out --> HighBayWarehouse_inp
-	Transporter_mqtt_out --> MQTTControlUnit_mqtt_in
-	subgraph HighBayWarehouse
-		HighBayWarehouse_mqtt_out("mqtt_out")
-		HighBayWarehouse_mqtt_in("mqtt_in")
-		HighBayWarehouse_out("out")
-		HighBayWarehouse_inp("inp")
-		HighBayWarehouse_inventory_out("inventory_out")
-		HighBayWarehouse_inp ~~~ HighBayWarehouse_out
-		HighBayWarehouse_mqtt_in ~~~ HighBayWarehouse_mqtt_out
-	end
-	HighBayWarehouse_out --> Transporter_left_in
-	HighBayWarehouse_mqtt_out --> MQTTControlUnit_mqtt_in
-	HighBayWarehouse_inventory_out --> InventoryPublisher_inp
-	subgraph InventoryPublisher
-		InventoryPublisher_inp("inp")
-		InventoryPublisher_mqtt_out("mqtt_out")
-		InventoryPublisher_inp ~~~ InventoryPublisher_mqtt_out
-	end
-	InventoryPublisher_mqtt_out --> MQTTControlUnit_mqtt_in
-	subgraph MPO
-		MPO_vgr_in("vgr_in")
-		MPO_mqtt_out("mqtt_out")
-		MPO_conveyor_out("conveyor_out")
-		MPO_mqtt_in("mqtt_in")
-		MPO_vgr_in ~~~ MPO_mqtt_out
-		MPO_mqtt_in ~~~ MPO_conveyor_out
-		subgraph MPO_oven
-			MPO_oven_mqtt_out("mqtt_out")
-			MPO_oven_vgr_in("vgr_in")
-			MPO_oven_gripper_out("gripper_out")
-			MPO_oven_gripper_in("gripper_in")
-			MPO_oven_mqtt_in("mqtt_in")
-			MPO_oven_vgr_in ~~~ MPO_oven_gripper_out
-			MPO_oven_gripper_in ~~~ MPO_oven_mqtt_out
-		end
-		MPO_oven_gripper_out --> MPO_gripper_oven_in
-		MPO_oven_mqtt_out --> MPO_mqtt_out
-		subgraph MPO_gripper
-			MPO_gripper_mqtt_out("mqtt_out")
-			MPO_gripper_mqtt_in("mqtt_in")
-			MPO_gripper_oven_out("oven_out")
-			MPO_gripper_table_out("table_out")
-			MPO_gripper_table_in("table_in")
-			MPO_gripper_oven_in("oven_in")
-			MPO_gripper_oven_in ~~~ MPO_gripper_oven_out
-			MPO_gripper_table_in ~~~ MPO_gripper_table_out
-			MPO_gripper_mqtt_in ~~~ MPO_gripper_mqtt_out
-		end
-		MPO_gripper_oven_out --> MPO_oven_gripper_in
-		MPO_gripper_table_out --> MPO_saw_gripper_in
-		MPO_gripper_mqtt_out --> MPO_mqtt_out
-		subgraph MPO_saw
-			MPO_saw_mqtt_in("mqtt_in")
-			MPO_saw_conveyor_out("conveyor_out")
-			MPO_saw_gripper_out("gripper_out")
-			MPO_saw_mqtt_out("mqtt_out")
-			MPO_saw_gripper_in("gripper_in")
-			MPO_saw_gripper_in ~~~ MPO_saw_gripper_out
-			MPO_saw_mqtt_in ~~~ MPO_saw_conveyor_out
-		end
-		MPO_saw_gripper_out --> MPO_gripper_table_in
-		MPO_saw_conveyor_out --> MPO_conveyor_out
-		MPO_saw_mqtt_out --> MPO_mqtt_out
-	end
-	MPO_mqtt_out --> MQTTControlUnit_mqtt_in
-	MPO_conveyor_out --> Conveyor_inp
-	MPO_vgr_in --> MPO_oven_vgr_in
-	MPO_mqtt_in --> MPO_oven_mqtt_in
-	MPO_mqtt_in --> MPO_gripper_mqtt_in
-	MPO_mqtt_in --> MPO_saw_mqtt_in
-	subgraph Conveyor
-		Conveyor_out("out")
-		Conveyor_mqtt_in("mqtt_in")
-		Conveyor_mqtt_out("mqtt_out")
-		Conveyor_inp("inp")
-		Conveyor_inp ~~~ Conveyor_out
-		Conveyor_mqtt_in ~~~ Conveyor_mqtt_out
-	end
-	Conveyor_out --> SLD_inp
-	Conveyor_mqtt_out --> MQTTControlUnit_mqtt_in
-	subgraph SLD
-		SLD_blue_in("blue_in")
-		SLD_inp("inp")
-		SLD_red_in("red_in")
-		SLD_mqtt_in("mqtt_in")
-		SLD_white_out("white_out")
-		SLD_blue_out("blue_out")
-		SLD_white_in("white_in")
-		SLD_mqtt_out("mqtt_out")
-		SLD_red_out("red_out")
-		SLD_inp ~~~ SLD_white_out
-		SLD_mqtt_in ~~~ SLD_red_out
-		SLD_white_in ~~~ SLD_blue_out
-		SLD_red_in ~~~ SLD_mqtt_out
-		subgraph SLD_conveyor
-			SLD_conveyor_mqtt_in("mqtt_in")
-			SLD_conveyor_out_white("out_white")
-			SLD_conveyor_inp("inp")
-			SLD_conveyor_out_blue("out_blue")
-			SLD_conveyor_mqtt_out("mqtt_out")
-			SLD_conveyor_out_red("out_red")
-			SLD_conveyor_inp ~~~ SLD_conveyor_out_white
-			SLD_conveyor_mqtt_in ~~~ SLD_conveyor_out_red
-		end
-		SLD_conveyor_out_white --> SLD_white_bay_sld_in
-		SLD_conveyor_out_red --> SLD_red_bay_sld_in
-		SLD_conveyor_out_blue --> SLD_blue_bay_sld_in
-		SLD_conveyor_mqtt_out --> SLD_mqtt_out
-		subgraph SLD_white_bay
-			SLD_white_bay_vgr_in("vgr_in")
-			SLD_white_bay_mqtt_out("mqtt_out")
-			SLD_white_bay_sld_in("sld_in")
-			SLD_white_bay_vgr_out("vgr_out")
-			SLD_white_bay_mqtt_in("mqtt_in")
-			SLD_white_bay_sld_in ~~~ SLD_white_bay_vgr_out
-			SLD_white_bay_vgr_in ~~~ SLD_white_bay_mqtt_out
-		end
-		SLD_white_bay_vgr_out --> SLD_white_out
-		SLD_white_bay_mqtt_out --> SLD_mqtt_out
-		subgraph SLD_red_bay
-			SLD_red_bay_vgr_out("vgr_out")
-			SLD_red_bay_mqtt_in("mqtt_in")
-			SLD_red_bay_sld_in("sld_in")
-			SLD_red_bay_vgr_in("vgr_in")
-			SLD_red_bay_mqtt_out("mqtt_out")
-			SLD_red_bay_sld_in ~~~ SLD_red_bay_vgr_out
-			SLD_red_bay_vgr_in ~~~ SLD_red_bay_mqtt_out
-		end
-		SLD_red_bay_vgr_out --> SLD_red_out
-		SLD_red_bay_mqtt_out --> SLD_mqtt_out
-		subgraph SLD_blue_bay
-			SLD_blue_bay_vgr_in("vgr_in")
-			SLD_blue_bay_mqtt_in("mqtt_in")
-			SLD_blue_bay_mqtt_out("mqtt_out")
-			SLD_blue_bay_vgr_out("vgr_out")
-			SLD_blue_bay_sld_in("sld_in")
-			SLD_blue_bay_sld_in ~~~ SLD_blue_bay_vgr_out
-			SLD_blue_bay_vgr_in ~~~ SLD_blue_bay_mqtt_out
-		end
-		SLD_blue_bay_vgr_out --> SLD_blue_out
-		SLD_blue_bay_mqtt_out --> SLD_mqtt_out
-	end
-	SLD_white_out --> VacuumGripper_sld_white_in
-	SLD_red_out --> VacuumGripper_sld_red_in
-	SLD_blue_out --> VacuumGripper_sld_blue_in
-	SLD_mqtt_out --> MQTTControlUnit_mqtt_in
-	SLD_inp --> SLD_conveyor_inp
-	SLD_mqtt_in --> SLD_conveyor_mqtt_in
-	SLD_mqtt_in --> SLD_white_bay_mqtt_in
-	SLD_mqtt_in --> SLD_red_bay_mqtt_in
-	SLD_mqtt_in --> SLD_blue_bay_mqtt_in
-	SLD_white_in --> SLD_white_bay_vgr_in
-	SLD_red_in --> SLD_red_bay_vgr_in
-	SLD_blue_in --> SLD_blue_bay_vgr_in
-	subgraph DSO
-		DSO_mqtt_out("mqtt_out")
-		DSO_mqtt_in("mqtt_in")
-		DSO_inp("inp")
-		DSO_out("out")
-		DSO_inp ~~~ DSO_out
-		DSO_mqtt_in ~~~ DSO_mqtt_out
-	end
-	DSO_mqtt_out --> MQTTControlUnit_mqtt_in
-	subgraph MQTTControlUnit
-		MQTTControlUnit_mqtt_in("mqtt_in")
-		MQTTControlUnit_mqtt_out("mqtt_out")
-		MQTTControlUnit_REALTIME_OBSERVED("REALTIME_OBSERVED")
-		MQTTControlUnit_REALTIME_INTERRUPT("REALTIME_INTERRUPT")
-		MQTTControlUnit_mqtt_in ~~~ MQTTControlUnit_mqtt_out
-		MQTTControlUnit_REALTIME_INTERRUPT ~~~ MQTTControlUnit_REALTIME_OBSERVED
-	end
-	MQTTControlUnit_mqtt_out --> Generator_mqtt_in
-	MQTTControlUnit_mqtt_out --> DSI_mqtt_in
-	MQTTControlUnit_mqtt_out --> ReaderStation_mqtt_in
-	MQTTControlUnit_mqtt_out --> VacuumGripper_mqtt_in
-	MQTTControlUnit_mqtt_out --> Transporter_mqtt_in
-	MQTTControlUnit_mqtt_out --> HighBayWarehouse_mqtt_in
-	MQTTControlUnit_mqtt_out --> MPO_mqtt_in
-	MQTTControlUnit_mqtt_out --> SLD_mqtt_in
-	MQTTControlUnit_mqtt_out --> DSO_mqtt_in
-	MQTTControlUnit_REALTIME_OBSERVED --> FischertechnikFactory_REALTIME_OBSERVED
-```

simulator/realtime_simulation.py

@@ -9,6 +9,7 @@ from pypdevs.simulator import Simulator
 from devs_models.fischertechnik_factory import FischertechnikFactory
 from utils.flowchart_generator import FlowchartGenerator
 from config import load_config  # Local config.py using dotenv
+from datetime import datetime
 
 load_dotenv()
 CFG = load_config()
@@ -23,23 +24,65 @@ class MQTTSimulationBridge:
         self.sim_client.on_message = self._on_sim
         self.real_client.on_message = self._on_real
 
-        # vars to keep track of real factory state, and notifications to the simulation
+        self.trace_real = [] # (time_wall, topic, payload)
+        self.trace_sim = []
+
+        # vars to keep track of the real factory state, and notifications to the simulation
         self.is_new_wp_notified: bool = False # if there is new input, will be set to true to prevent double notification
         self.is_dso_busy: bool = False # whether there is a workpiece in the DSO (output tray)
 
+    def _log(self, origin, msg):
+        """ Logs the received mqtt messages """
+        t = time.time()
+        if origin == "real":
+            self.trace_real.append((t, msg.topic, msg.payload.decode()))
+        else:
+            self.trace_sim.append((t, msg.topic, msg.payload.decode()))
+    
+    def log_trace_to_jsonl(self, filename_prefix="logs/trace_log"):
+        """
+        Outputs the trace_real and trace_sim data to a JSON Lines (.jsonl) file.
+        Each entry is written as a single JSON object on a new line.
+        """
+        for item in [[self.trace_real, 'real'], [self.trace_sim, 'sim']]:
+            trace = item[0]
+            filename = f"{filename_prefix}_{item[1]}.jsonl"
+            with open(filename, 'w') as f:
+                for entry in trace:
+                    time_wall, topic, payload = entry
+                    timestamp_str = datetime.fromtimestamp(float(time_wall)).isoformat()
+                    #timestamp_str = time_wall
+                    try:
+                        payload = json.loads(payload)
+                    except json.JSONDecodeError:
+                        logger.warning(f"Failed to decode payload: {payload}")
+                        continue
+                    log_entry = {
+                        "timestamp_wall": timestamp_str,
+                        "topic": topic,
+                        "payload": payload
+                    }
+                    json_line = json.dumps(log_entry)
+                    f.write(json_line + "\n")
+
     def _on_sim(self, _cli, _u, msg):
         json_string = json.dumps({
             "topic": msg.topic,
             "payload": msg.payload.decode(),
             "origin": "sim"
         })
+        topic_blacklist = ["o/broadcast", "simulation/", "visualization/"]
+        if all(bad_topic not in msg.topic for bad_topic in topic_blacklist):
+            self._log("sim", msg)
 
-        if self.sim:
+        if self.sim and msg.topic.startswith("simulation/"):
+            print(self.sim.server.getTime())
             logger.info(f"Received SIM MQTT message, topic: {msg.topic}")
             self.sim.realtime_interrupt(f"REALTIME_INTERRUPT {json_string}") # Interrupts can only send strings
 
     def _on_real(self, _cli, _u, msg):
         data = json.loads(msg.payload.decode())
+        self._log("real", msg)
         
         # --- Messages directly passed through to the simulation ---
         if msg.topic == "f/o/order": # orders placed in the real world should be passed to the simulation
@@ -96,9 +139,11 @@ class MQTTSimulationBridge:
         self.sim_client.connect(CFG.MQTT_SIM.HOST, CFG.MQTT_SIM.PORT)
         self.real_client.connect(CFG.MQTT_REAL.HOST, CFG.MQTT_REAL.PORT)
 
-        self.sim_client.subscribe("simulation/#")
-        self.real_client.subscribe("f/i/#") # TODO: check if I actually may narrow this down
-        self.real_client.subscribe("f/o/#") # orders
+        # self.sim_client.subscribe("simulation/#")
+        # self.real_client.subscribe("f/i/#")
+        # self.real_client.subscribe("f/o/#") # orders
+        self.sim_client.subscribe("#") # subscribe to all
+        self.real_client.subscribe("#") # subscribe to all
         self.sim_client.loop_start()
         self.real_client.loop_start()
 
@@ -148,7 +193,10 @@ class MQTTSimulationBridge:
             logger.info(self.sim.model.get_inventory())
             logger.info("Collector contents:")
             logger.info(self.sim.model.collector.get_contents())
-
+            # Log the traces to JSONL files
+            logger.info("Logging traces to JSONL files...")
+            self.log_trace_to_jsonl()
+            logger.info("Traces logged successfully.")
 
 if __name__ == "__main__":
     bridge = MQTTSimulationBridge()