utils.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320
  1. import unittest
  2. import sys
  3. import os
  4. import sys
  5. import time
  6. import json
  7. import urllib
  8. import urllib2
  9. import subprocess
  10. import signal
  11. import random
  12. import pytest
  13. sys.path.append("interface/HUTN")
  14. sys.path.append("scripts")
  15. from hutn_compiler.compiler import main as do_compile
  16. from check_objects import to_recompile
  17. taskname = "test_task"
  18. parallel_push = True
  19. slow = pytest.mark.skipif(
  20. not pytest.config.getoption("--runslow"),
  21. reason="need --runslow option to run"
  22. )
  23. ports = set()
  24. def getFreePort():
  25. """Gets a unique new port."""
  26. while 1:
  27. port = random.randint(10000, 20000)
  28. # Check if this port is in the set of ports.
  29. if port not in ports:
  30. # We have found a unique port. Add it to the set and return.
  31. ports.add(port)
  32. return port
  33. def execute(scriptname, parameters=[], wait=False):
  34. if os.name not in ["nt", "posix"]:
  35. # Stop now, as we would have no clue on how to kill its subtree
  36. raise Exception("Unknown OS version: " + str(os.name))
  37. command = [sys.executable, "scripts/%s.py" % scriptname] + parameters
  38. if wait:
  39. return subprocess.call(command, shell=False)
  40. else:
  41. return subprocess.Popen(command, shell=False)
  42. def kill(process):
  43. if os.name == "nt":
  44. subprocess.call(["taskkill", "/F", "/T", "/PID", "%i" % process.pid])
  45. elif os.name == "posix":
  46. subprocess.call(["pkill", "-P", "%i" % process.pid])
  47. def flush_data(address, data):
  48. if data:
  49. urllib2.urlopen(urllib2.Request(address, urllib.urlencode({"op": "set_input", "data": json.dumps(data), "taskname": taskname})), timeout=10).read()
  50. return []
  51. def compile_file(address, mod_filename, filename, mode, proc):
  52. # Load in the file required
  53. try:
  54. timeout_val = 240
  55. import random
  56. taskname = str(random.random())
  57. while 1:
  58. proc2 = execute("compile", [address, mod_filename, taskname, filename, mode], wait=False)
  59. if proc.returncode is not None:
  60. # Modelverse has already terminated, which isn't a good sign!
  61. raise Exception("Modelverse died!")
  62. while proc2.returncode is None:
  63. time.sleep(0.01)
  64. proc2.poll()
  65. timeout_val -= 0.01
  66. if timeout_val < 0:
  67. kill(proc2)
  68. print("Compilation timeout expired!")
  69. return False
  70. if proc2.returncode != 2:
  71. break
  72. # Make sure everything stopped correctly
  73. assert proc2.returncode == 0
  74. if proc2.returncode != 0:
  75. return False
  76. except:
  77. raise
  78. finally:
  79. try:
  80. kill(proc2)
  81. except UnboundLocalError:
  82. pass
  83. def run_file(files, parameters, expected, mode, wait=False):
  84. # Resolve file
  85. import os.path
  86. if wait is True:
  87. expected = None
  88. time.sleep(0.01)
  89. port = getFreePort()
  90. address = "http://127.0.0.1:%i" % port
  91. try:
  92. # Run Modelverse server
  93. proc = execute("run_local_modelverse", [str(port)], wait=False)
  94. threads = []
  95. mod_files = []
  96. for filename in files:
  97. if os.path.isfile(filename):
  98. mod_filename = filename
  99. elif os.path.isfile("integration/code/%s" % filename):
  100. mod_filename = "integration/code/%s" % filename
  101. elif os.path.isfile("bootstrap/%s" % filename):
  102. mod_filename = "bootstrap/%s" % filename
  103. else:
  104. raise Exception("File not found: %s" % filename)
  105. mod_files.append(mod_filename)
  106. to_compile = to_recompile(address, mod_files)
  107. for mod_filename in to_compile:
  108. if mod_filename.endswith(".mvc"):
  109. model_mode = "MO"
  110. mod_files.remove(mod_filename)
  111. else:
  112. model_mode = mode
  113. if parallel_push:
  114. import threading
  115. threads.append(threading.Thread(target=compile_file, args=[address, mod_filename, mod_filename, model_mode, proc]))
  116. threads[-1].start()
  117. else:
  118. compile_file(address, mod_filename, mod_filename, model_mode, proc)
  119. if parallel_push:
  120. for t in threads:
  121. t.join()
  122. if mode[-1] == "O":
  123. # Fire up the linker
  124. val = execute("link_and_load", [address, taskname] + mod_files, wait=True)
  125. if val != 0:
  126. raise Exception("Linking error")
  127. # Send the request ...
  128. flush_data(address, parameters)
  129. # ... and wait for replies
  130. if expected is None:
  131. while 1:
  132. val = urllib2.urlopen(urllib2.Request(address, urllib.urlencode({"op": "get_output", "taskname": taskname})), timeout=240).read()
  133. val = json.loads(val)
  134. print(val)
  135. for e in expected:
  136. c = len(e) if isinstance(e, set) else 1
  137. for _ in range(c):
  138. val = urllib2.urlopen(urllib2.Request(address, urllib.urlencode({"op": "get_output", "taskname": taskname})), timeout=240).read()
  139. val = json.loads(val)
  140. if proc.returncode is not None:
  141. # Modelverse has already terminated, which isn't a good sign!
  142. raise Exception("Modelverse died!")
  143. print("Got %s, expect %s" % (val, e))
  144. if isinstance(e, set):
  145. assert val in e
  146. if val not in e:
  147. return False
  148. elif e is None:
  149. # Skip output value
  150. pass
  151. else:
  152. assert val == e
  153. if val != e:
  154. return False
  155. # All passed!
  156. return True
  157. except:
  158. raise
  159. finally:
  160. try:
  161. kill(proc)
  162. except UnboundLocalError:
  163. pass
  164. def run_barebone(parameters, expected, interface="0", timeout=False, wait=False, link=None, inputs=[]):
  165. port = getFreePort()
  166. address = "http://127.0.0.1:%i" % port
  167. try:
  168. # Run Modelverse server
  169. proc = execute("run_local_modelverse", [str(port)], wait=False)
  170. # Create task and set interface
  171. timeout_val = 15
  172. start = time.time()
  173. while 1:
  174. proc.poll()
  175. if proc.returncode is not None:
  176. # Modelverse has already terminated, which isn't a good sign!
  177. return False
  178. try:
  179. urllib2.urlopen(urllib2.Request(address, urllib.urlencode({"op": "set_input", "element_type": "V", "value": '"%s"' % taskname, "taskname": "task_manager"})), timeout=1).read()
  180. if interface is not None:
  181. urllib2.urlopen(urllib2.Request(address, urllib.urlencode({"op": "set_input", "element_type": "V", "value": interface, "taskname": taskname})), timeout=1).read()
  182. break
  183. except:
  184. time.sleep(0.01)
  185. if time.time() - start > timeout_val:
  186. raise
  187. # Send the request
  188. flush_data(address, parameters)
  189. # Now do linking and loading
  190. if link is not None:
  191. # Execute linker
  192. timeout_val = 10
  193. proc2 = execute("link_and_load", [address, taskname] + link, wait=False)
  194. while proc2.returncode is None:
  195. time.sleep(0.01)
  196. proc2.poll()
  197. timeout_val -= 0.01
  198. if timeout_val < 0:
  199. kill(proc2)
  200. print("Linking timeout expired!")
  201. return False
  202. if proc.returncode is not None:
  203. # Modelverse has already terminated, which isn't a good sign!
  204. return False
  205. for inp in inputs:
  206. urllib2.urlopen(urllib2.Request(address, urllib.urlencode({"op": "set_input", "element_type": "V", "value": inp, "taskname": taskname})), timeout=1).read()
  207. proc.poll()
  208. if proc.returncode is not None:
  209. # Modelverse has already terminated, which isn't a good sign!
  210. return False
  211. counter = 0
  212. for e in expected:
  213. print("Expect " + str(e))
  214. c = len(e) if isinstance(e, set) else 1
  215. for _ in range(c):
  216. try:
  217. proc.poll()
  218. if proc.returncode is not None:
  219. # Modelverse has already terminated, which isn't a good sign!
  220. return False
  221. val = urllib2.urlopen(urllib2.Request(address, urllib.urlencode({"op": "get_output", "taskname": taskname})), timeout=240 if not timeout else 20).read()
  222. val = json.loads(val)
  223. except:
  224. if timeout:
  225. return True
  226. else:
  227. raise
  228. print("Got %s, expect %s" % (val, e))
  229. if isinstance(e, set):
  230. assert val in e
  231. if val not in e:
  232. return False
  233. elif e is None:
  234. # Skip this input
  235. pass
  236. else:
  237. assert val == e
  238. if val != e:
  239. return False
  240. # All passed!
  241. return not timeout
  242. finally:
  243. kill(proc)
  244. def get_constructor(code):
  245. code_fragments = code.split("\n")
  246. code_fragments = [i for i in code_fragments if i.strip() != ""]
  247. code_fragments = [i.replace(" ", "\t") for i in code_fragments]
  248. initial_tabs = min([len(i) - len(i.lstrip("\t")) for i in code_fragments])
  249. code_fragments = [i[initial_tabs:] for i in code_fragments]
  250. code_fragments.append("")
  251. code = "\n".join(code_fragments)
  252. with open("__constraint.alc", "w") as f:
  253. f.write(code)
  254. f.flush()
  255. constructors = do_compile("__constraint.alc", "interface/HUTN/grammars/actionlanguage.g", "CS")
  256. return constructors
  257. def get_model_constructor(code):
  258. # First change multiple spaces to a tab
  259. code_fragments = code.split("\n")
  260. code_fragments = [i for i in code_fragments if i.strip() != ""]
  261. code_fragments = [i.replace(" ", "\t") for i in code_fragments]
  262. initial_tabs = min([len(i) - len(i.lstrip("\t")) for i in code_fragments])
  263. code_fragments = [i[initial_tabs:] for i in code_fragments]
  264. code_fragments.append("")
  265. code = "\n".join(code_fragments)
  266. with open("__model.mvc", "w") as f:
  267. f.write(code)
  268. f.flush()
  269. constructors = do_compile("__model.mvc", "interface/HUTN/grammars/modelling.g", "M") + ["exit"]
  270. return constructors