modelverse/integration/utils.py

import unittest
import sys
import os

import sys
import time
import json
import urllib
import urllib2
import subprocess
import signal
import random

sys.path.append("interface/HUTN")
sys.path.append("scripts")
from hutn_compiler.compiler import main as do_compile
from check_objects import to_recompile

taskname = "test_task"
parallel_push = True
INIT_TIMEOUT = 30
TIMEOUT = 2000

try:
    import pytest
    slow = pytest.mark.skipif(
        not pytest.config.getoption("--runslow"),
            reason="need --runslow option to run"
    )
except:
    slow = lambda i:i

ports = set()

def getFreePort():
    """Gets a unique new port."""
    while 1:
        port = random.randint(10000, 20000)
        # Check if this port is in the set of ports.
        if port not in ports:
            # We have found a unique port. Add it to the set and return.
            ports.add(port)
            return port

def execute(scriptname, parameters=[], wait=False):
    if os.name not in ["nt", "posix"]:
        # Stop now, as we would have no clue on how to kill its subtree
        raise Exception("Unknown OS version: " + str(os.name))

    command = [sys.executable, "-u", "scripts/%s.py" % scriptname] + parameters

    if wait:
        return subprocess.call(command, shell=False)
    else:
        return subprocess.Popen(command, shell=False, stdout=subprocess.PIPE)

def kill(process):
    if os.name == "nt":
        subprocess.call(["taskkill", "/F", "/T", "/PID", "%i" % process.pid])
    elif os.name == "posix":
        # Kill parents
        subprocess.call(["pkill", "-P", "%i" % process.pid])
        # Kill self
        subprocess.call(["kill", "%i" % process.pid])

def flush_data(address, data):
    if data:
        urllib2.urlopen(urllib2.Request(address, urllib.urlencode({"op": "set_input", "data": json.dumps(data), "taskname": taskname})), timeout=INIT_TIMEOUT).read()
    return []

def compile_file(address, mod_filename, filename, mode, proc):
    # Load in the file required
    try:
        timeout_val = 240
        import random
        taskname = str(random.random())
        while 1:
            proc2 = execute("compile", [address, mod_filename, taskname, filename, mode], wait=False)

            if proc.returncode is not None:
                # Modelverse has already terminated, which isn't a good sign!
                raise Exception("Modelverse died!")

            while proc2.returncode is None:
                time.sleep(0.01)
                proc2.poll()
                timeout_val -= 0.01
                if timeout_val < 0:
                    kill(proc2)
                    print("Compilation timeout expired!")
                    return False

            if proc2.returncode != 2:
                break

        # Make sure everything stopped correctly
        assert proc2.returncode == 0
        if proc2.returncode != 0:
            return False
    except:
        raise
    finally:
        try:
            kill(proc2)
        except UnboundLocalError:
            pass

def start_mvc():
    address = "http://127.0.0.1:%s" % getFreePort()
    proc = execute("run_MvC_server", [address], wait=False)
    # TODO Return only when everything is fine! (i.e., parse the output of proc and wait until it contains "MvC is ready"
    while 1:
        l = proc.stdout.readline()
        if "MvC is ready" in l:
            return proc, address

def run_file(files, parameters, expected, wait=False):
    # Resolve file
    import os.path

    if wait is True:
        expected = None

    time.sleep(0.01)
    port = getFreePort()
    address = "http://127.0.0.1:%i" % port
    try:
        # Run Modelverse server
        proc = execute("run_local_modelverse", [str(port)], wait=False)

        threads = []
        mod_files = []
        for filename in files:
            if os.path.isfile(filename):
                mod_filename = filename
            elif os.path.isfile("integration/code/%s" % filename):
                mod_filename = "integration/code/%s" % filename
            elif os.path.isfile("bootstrap/%s" % filename):
                mod_filename = "bootstrap/%s" % filename
            else:
                raise Exception("File not found: %s" % filename)
            mod_files.append(mod_filename)

        to_compile = to_recompile(address, mod_files)

        for mod_filename in to_compile:
            if mod_filename.endswith(".mvc"):
                model_mode = "MO"
                mod_files.remove(mod_filename)
            else:
                model_mode = "PO"
            if parallel_push:
                import threading
                threads.append(threading.Thread(target=compile_file, args=[address, mod_filename, mod_filename, model_mode, proc]))
                threads[-1].start()
            else:
                compile_file(address, mod_filename, mod_filename, model_mode, proc)

        if parallel_push:
            for t in threads:
                t.join()

        # Fire up the linker
        val = execute("link_and_load", [address, taskname] + mod_files, wait=True)
        if val != 0:
            raise Exception("Linking error")

        # Send the request ...
        flush_data(address, parameters)
        
        # ... and wait for replies
        if expected is None:
            while 1:
                val = urllib2.urlopen(urllib2.Request(address, urllib.urlencode({"op": "get_output", "taskname": taskname})), timeout=TIMEOUT).read()
                val = json.loads(val)
                print(val)
        for e in expected:
            c = len(e) if isinstance(e, set) else 1
            if isinstance(e, set):
                # Copy set before we start popping from it, as it might be reused elsewhere
                e = set(e)
            for _ in range(c):
                val = urllib2.urlopen(urllib2.Request(address, urllib.urlencode({"op": "get_output", "taskname": taskname})), timeout=TIMEOUT).read()
                val = json.loads(val)

                if proc.returncode is not None:
                    # Modelverse has already terminated, which isn't a good sign!
                    raise Exception("Modelverse died!")

                print("Got %s, expect %s" % (val, e))
                if isinstance(e, set):
                    assert val in e
                    if val not in e:
                        return False
                    e.remove(val)
                elif e is None:
                    # Skip output value
                    pass
                else:
                    assert val == e
                    if val != e:
                        return False

        # All passed!
        return True
    except:
        raise
    finally:
        try:
            kill(proc)
        except UnboundLocalError:
            pass

def run_barebone(parameters, expected, interface="0", timeout=False, wait=False, link=None, inputs=[]):
    port = getFreePort()
    address = "http://127.0.0.1:%i" % port
    try:
        # Run Modelverse server
        proc = execute("run_local_modelverse", [str(port)], wait=False)

        # Create task and set interface
        timeout_val = INIT_TIMEOUT
        start = time.time()
        while 1:
            proc.poll()
            if proc.returncode is not None:
                # Modelverse has already terminated, which isn't a good sign!
                return False

            try:
                urllib2.urlopen(urllib2.Request(address, urllib.urlencode({"op": "set_input", "value": '"%s"' % taskname, "taskname": "task_manager"})), timeout=INIT_TIMEOUT).read()
                if interface is not None:
                    urllib2.urlopen(urllib2.Request(address, urllib.urlencode({"op": "set_input", "value": interface, "taskname": taskname})), timeout=INIT_TIMEOUT).read()
                break
            except:
                time.sleep(0.01)

                if time.time() - start > timeout_val:
                    raise

        # Send the request
        print("Sending data: " + str(parameters))
        flush_data(address, parameters)

        # Now do linking and loading
        if link is not None:
            # Execute linker
            timeout_val = INIT_TIMEOUT
            proc2 = execute("link_and_load", [address, taskname] + link, wait=False)

            while proc2.returncode is None:
                time.sleep(0.01)
                proc2.poll()
                timeout_val -= 0.01
                if timeout_val < 0:
                    kill(proc2)
                    print("Linking timeout expired!")
                    return False
                if proc.returncode is not None:
                    # Modelverse has already terminated, which isn't a good sign!
                    return False

        for inp in inputs:
            urllib2.urlopen(urllib2.Request(address, urllib.urlencode({"op": "set_input", "value": inp, "taskname": taskname})), timeout=INIT_TIMEOUT).read()
            proc.poll()
            if proc.returncode is not None:
                # Modelverse has already terminated, which isn't a good sign!
                return False

        counter = 0
        for e in expected:
            c = len(e) if isinstance(e, set) else 1
            for _ in range(c):
                try:
                    proc.poll()
                    if proc.returncode is not None:
                        # Modelverse has already terminated, which isn't a good sign!
                        return False

                    val = urllib2.urlopen(urllib2.Request(address, urllib.urlencode({"op": "get_output", "taskname": taskname})), timeout=TIMEOUT if not timeout else INIT_TIMEOUT).read()
                    val = json.loads(val)
                except:
                    if timeout:
                        return True
                    else:
                        raise

                #print("Got %s, expect %s" % (val, e))
                if isinstance(e, set):
                    assert val in e
                    if val not in e:
                        return False
                    e.remove(val)
                elif e is None:
                    # Skip this input
                    pass
                else:
                    assert val == e
                    if val != e:
                        return False

        # All passed!
        return not timeout
    finally:
        kill(proc)

def get_constructor(code):
    code_fragments = code.split("\n")
    code_fragments = [i for i in code_fragments if i.strip() != ""]
    code_fragments = [i.replace("    ", "\t") for i in code_fragments]
    initial_tabs = min([len(i) - len(i.lstrip("\t")) for i in code_fragments])
    code_fragments = [i[initial_tabs:] for i in code_fragments]
    code_fragments.append("")
    code = "\n".join(code_fragments)

    with open("__constraint.alc", "w") as f:
        f.write(code)
        f.flush()

    constructors = do_compile("__constraint.alc", "interface/HUTN/grammars/actionlanguage.g", "CS")

    return constructors

def get_model_constructor(code):
    # First change multiple spaces to a tab
    code_fragments = code.split("\n")
    code_fragments = [i for i in code_fragments if i.strip() != ""]
    code_fragments = [i.replace("    ", "\t") for i in code_fragments]
    initial_tabs = min([len(i) - len(i.lstrip("\t")) for i in code_fragments])
    code_fragments = [i[initial_tabs:] for i in code_fragments]
    code_fragments.append("")
    code = "\n".join(code_fragments)

    with open("__model.mvc", "w") as f:
        f.write(code)
        f.flush()

    return get_model_constructor_2("__model.mvc")

def get_model_constructor_2(f):
    return do_compile(f, "interface/HUTN/grammars/modelling.g", "M") + ["exit"]