123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344 |
- import unittest
- import sys
- import os
- import sys
- import time
- import json
- import urllib
- import urllib2
- import subprocess
- import signal
- import random
- sys.path.append("interface/HUTN")
- sys.path.append("scripts")
- from hutn_compiler.compiler import main as do_compile
- from check_objects import to_recompile
- taskname = "test_task"
- parallel_push = True
- INIT_TIMEOUT = 30
- TIMEOUT = 2000
- try:
- import pytest
- slow = pytest.mark.skipif(
- not pytest.config.getoption("--runslow"),
- reason="need --runslow option to run"
- )
- except:
- slow = lambda i:i
- ports = set()
- def getFreePort():
- """Gets a unique new port."""
- while 1:
- port = random.randint(10000, 20000)
- # Check if this port is in the set of ports.
- if port not in ports:
- # We have found a unique port. Add it to the set and return.
- ports.add(port)
- return port
- def execute(scriptname, parameters=[], wait=False):
- if os.name not in ["nt", "posix"]:
- # Stop now, as we would have no clue on how to kill its subtree
- raise Exception("Unknown OS version: " + str(os.name))
- command = [sys.executable, "-u", "scripts/%s.py" % scriptname] + parameters
- if wait:
- return subprocess.call(command, shell=False)
- else:
- return subprocess.Popen(command, shell=False, stdout=subprocess.PIPE)
- def kill(process):
- if os.name == "nt":
- subprocess.call(["taskkill", "/F", "/T", "/PID", "%i" % process.pid])
- elif os.name == "posix":
- # Kill parents
- subprocess.call(["pkill", "-P", "%i" % process.pid])
- # Kill self
- subprocess.call(["kill", "%i" % process.pid])
- def flush_data(address, data):
- if data:
- urllib2.urlopen(urllib2.Request(address, urllib.urlencode({"op": "set_input", "data": json.dumps(data), "taskname": taskname})), timeout=INIT_TIMEOUT).read()
- return []
- def compile_file(address, mod_filename, filename, mode, proc):
- # Load in the file required
- try:
- timeout_val = 240
- import random
- taskname = str(random.random())
- while 1:
- proc2 = execute("compile", [address, mod_filename, taskname, filename, mode], wait=False)
- if proc.returncode is not None:
- # Modelverse has already terminated, which isn't a good sign!
- raise Exception("Modelverse died!")
- while proc2.returncode is None:
- time.sleep(0.01)
- proc2.poll()
- timeout_val -= 0.01
- if timeout_val < 0:
- kill(proc2)
- print("Compilation timeout expired!")
- return False
- if proc2.returncode != 2:
- break
- # Make sure everything stopped correctly
- assert proc2.returncode == 0
- if proc2.returncode != 0:
- return False
- except:
- raise
- finally:
- try:
- kill(proc2)
- except UnboundLocalError:
- pass
- def start_mvc():
- port = getFreePort()
- address = "http://127.0.0.1:%s" % port
- proc = execute("run_local_modelverse", [str(port)], wait=False)
- return proc, address
- # TODO Return only when everything is fine! (i.e., parse the output of proc and wait until it contains "MvC is ready"
- while 1:
- l = proc.stdout.readline()
- if "MvC is ready" in l:
- return proc, address
- def run_file(files, parameters, expected, wait=False):
- # Resolve file
- import os.path
- if wait is True:
- expected = None
- time.sleep(0.01)
- port = getFreePort()
- address = "http://127.0.0.1:%i" % port
- try:
- # Run Modelverse server
- proc = execute("run_local_modelverse", [str(port)], wait=False)
- threads = []
- mod_files = []
- for filename in files:
- if os.path.isfile(filename):
- mod_filename = filename
- elif os.path.isfile("integration/code/%s" % filename):
- mod_filename = "integration/code/%s" % filename
- elif os.path.isfile("bootstrap/%s" % filename):
- mod_filename = "bootstrap/%s" % filename
- else:
- raise Exception("File not found: %s" % filename)
- mod_files.append(mod_filename)
- to_compile = to_recompile(address, mod_files)
- for mod_filename in to_compile:
- if mod_filename.endswith(".mvc"):
- model_mode = "MO"
- mod_files.remove(mod_filename)
- else:
- model_mode = "PO"
- if parallel_push:
- import threading
- threads.append(threading.Thread(target=compile_file, args=[address, mod_filename, mod_filename, model_mode, proc]))
- threads[-1].start()
- else:
- compile_file(address, mod_filename, mod_filename, model_mode, proc)
- if parallel_push:
- for t in threads:
- t.join()
- # Fire up the linker
- val = execute("link_and_load", [address, taskname] + mod_files, wait=True)
- if val != 0:
- raise Exception("Linking error")
- # Send the request ...
- flush_data(address, parameters)
-
- # ... and wait for replies
- if expected is None:
- while 1:
- val = urllib2.urlopen(urllib2.Request(address, urllib.urlencode({"op": "get_output", "taskname": taskname})), timeout=TIMEOUT).read()
- val = json.loads(val)
- print(val)
- for e in expected:
- c = len(e) if isinstance(e, set) else 1
- if isinstance(e, set):
- # Copy set before we start popping from it, as it might be reused elsewhere
- e = set(e)
- for _ in range(c):
- val = urllib2.urlopen(urllib2.Request(address, urllib.urlencode({"op": "get_output", "taskname": taskname})), timeout=TIMEOUT).read()
- val = json.loads(val)
- if proc.returncode is not None:
- # Modelverse has already terminated, which isn't a good sign!
- raise Exception("Modelverse died!")
- print("Got %s, expect %s" % (val, e))
- if isinstance(e, set):
- assert val in e
- if val not in e:
- return False
- e.remove(val)
- elif e is None:
- # Skip output value
- pass
- else:
- assert val == e
- if val != e:
- return False
- # All passed!
- return True
- except:
- raise
- finally:
- try:
- kill(proc)
- except UnboundLocalError:
- pass
- def run_barebone(parameters, expected, interface="0", timeout=False, wait=False, link=None, inputs=[]):
- port = getFreePort()
- address = "http://127.0.0.1:%i" % port
- try:
- # Run Modelverse server
- proc = execute("run_local_modelverse", [str(port)], wait=False)
- # Create task and set interface
- timeout_val = INIT_TIMEOUT
- start = time.time()
- while 1:
- proc.poll()
- if proc.returncode is not None:
- # Modelverse has already terminated, which isn't a good sign!
- return False
- try:
- urllib2.urlopen(urllib2.Request(address, urllib.urlencode({"op": "set_input", "value": '"%s"' % taskname, "taskname": "task_manager"})), timeout=INIT_TIMEOUT).read()
- if interface is not None:
- urllib2.urlopen(urllib2.Request(address, urllib.urlencode({"op": "set_input", "value": interface, "taskname": taskname})), timeout=INIT_TIMEOUT).read()
- break
- except:
- time.sleep(0.01)
- if time.time() - start > timeout_val:
- raise
- # Send the request
- print("Sending data: " + str(parameters))
- flush_data(address, parameters)
- # Now do linking and loading
- if link is not None:
- # Execute linker
- timeout_val = INIT_TIMEOUT
- proc2 = execute("link_and_load", [address, taskname] + link, wait=False)
- while proc2.returncode is None:
- time.sleep(0.01)
- proc2.poll()
- timeout_val -= 0.01
- if timeout_val < 0:
- kill(proc2)
- print("Linking timeout expired!")
- return False
- if proc.returncode is not None:
- # Modelverse has already terminated, which isn't a good sign!
- return False
- for inp in inputs:
- urllib2.urlopen(urllib2.Request(address, urllib.urlencode({"op": "set_input", "value": inp, "taskname": taskname})), timeout=INIT_TIMEOUT).read()
- proc.poll()
- if proc.returncode is not None:
- # Modelverse has already terminated, which isn't a good sign!
- return False
- counter = 0
- for e in expected:
- c = len(e) if isinstance(e, set) else 1
- for _ in range(c):
- try:
- proc.poll()
- if proc.returncode is not None:
- # Modelverse has already terminated, which isn't a good sign!
- return False
- val = urllib2.urlopen(urllib2.Request(address, urllib.urlencode({"op": "get_output", "taskname": taskname})), timeout=TIMEOUT if not timeout else INIT_TIMEOUT).read()
- val = json.loads(val)
- except:
- if timeout:
- return True
- else:
- raise
- #print("Got %s, expect %s" % (val, e))
- if isinstance(e, set):
- assert val in e
- if val not in e:
- return False
- e.remove(val)
- elif e is None:
- # Skip this input
- pass
- else:
- assert val == e
- if val != e:
- return False
- # All passed!
- return not timeout
- finally:
- kill(proc)
- def get_constructor(code):
- code_fragments = code.split("\n")
- code_fragments = [i for i in code_fragments if i.strip() != ""]
- code_fragments = [i.replace(" ", "\t") for i in code_fragments]
- initial_tabs = min([len(i) - len(i.lstrip("\t")) for i in code_fragments])
- code_fragments = [i[initial_tabs:] for i in code_fragments]
- code_fragments.append("")
- code = "\n".join(code_fragments)
- with open("__constraint.alc", "w") as f:
- f.write(code)
- f.flush()
- constructors = do_compile("__constraint.alc", "interface/HUTN/grammars/actionlanguage.g", "CS")
- return constructors
- def get_model_constructor(code):
- # First change multiple spaces to a tab
- code_fragments = code.split("\n")
- code_fragments = [i for i in code_fragments if i.strip() != ""]
- code_fragments = [i.replace(" ", "\t") for i in code_fragments]
- initial_tabs = min([len(i) - len(i.lstrip("\t")) for i in code_fragments])
- code_fragments = [i[initial_tabs:] for i in code_fragments]
- code_fragments.append("")
- code = "\n".join(code_fragments)
- with open("__model.mvc", "w") as f:
- f.write(code)
- f.flush()
- return get_model_constructor_2("__model.mvc")
- def get_model_constructor_2(f):
- return do_compile(f, "interface/HUTN/grammars/modelling.g", "M") + ["exit"]
|