utils.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322
  1. import unittest
  2. import sys
  3. import os
  4. import sys
  5. import time
  6. import json
  7. import urllib
  8. import urllib2
  9. import subprocess
  10. import signal
  11. import random
  12. sys.path.append("interface/HUTN")
  13. sys.path.append("scripts")
  14. from hutn_compiler.compiler import main as do_compile
  15. from check_objects import to_recompile
  16. username = "test_user"
  17. parallel_push = True
  18. ports = []
  19. def getFreePort():
  20. while 1:
  21. port = random.randint(10000, 20000)
  22. ports.append(port)
  23. exists = False
  24. for p in ports:
  25. if p == port:
  26. if not exists:
  27. # We have hopefully found our own
  28. exists = True
  29. else:
  30. # We seem to be the second entry, so chose another one
  31. ports.remove(port)
  32. break
  33. else:
  34. # Didn't find a duplicate
  35. return port
  36. def execute(scriptname, parameters=[], wait=False):
  37. if os.name not in ["nt", "posix"]:
  38. # Stop now, as we would have no clue on how to kill its subtree
  39. raise Exception("Unknown OS version: " + str(os.name))
  40. command = [sys.executable, "scripts/%s.py" % scriptname] + parameters
  41. if wait:
  42. return subprocess.call(command, shell=False)
  43. else:
  44. return subprocess.Popen(command, shell=False)
  45. def kill(process):
  46. if os.name == "nt":
  47. subprocess.call(["taskkill", "/F", "/T", "/PID", "%i" % process.pid])
  48. elif os.name == "posix":
  49. subprocess.call(["pkill", "-P", "%i" % process.pid])
  50. def flush_data(address, data):
  51. if data:
  52. urllib2.urlopen(urllib2.Request(address, urllib.urlencode({"op": "set_input", "data": json.dumps(data), "username": username})), timeout=10).read()
  53. return []
  54. def compile_file(address, mod_filename, filename, mode, proc):
  55. # Load in the file required
  56. try:
  57. timeout_val = 240
  58. import random
  59. username = str(random.random())
  60. while 1:
  61. proc2 = execute("compile", [address, mod_filename, username, filename, mode], wait=False)
  62. if proc.returncode is not None:
  63. # Modelverse has already terminated, which isn't a good sign!
  64. raise Exception("Modelverse died!")
  65. while proc2.returncode is None:
  66. time.sleep(0.01)
  67. proc2.poll()
  68. timeout_val -= 0.01
  69. if timeout_val < 0:
  70. kill(proc2)
  71. print("Compilation timeout expired!")
  72. return False
  73. if proc2.returncode != 2:
  74. break
  75. # Make sure everything stopped correctly
  76. assert proc2.returncode == 0
  77. if proc2.returncode != 0:
  78. return False
  79. except:
  80. raise
  81. finally:
  82. try:
  83. kill(proc2)
  84. except UnboundLocalError:
  85. pass
  86. def run_file(files, parameters, expected, mode):
  87. # Resolve file
  88. import os.path
  89. time.sleep(0.01)
  90. port = getFreePort()
  91. address = "http://127.0.0.1:%i" % port
  92. try:
  93. # Run Modelverse server
  94. proc = execute("run_local_modelverse", [str(port)], wait=False)
  95. threads = []
  96. mod_files = []
  97. for filename in files:
  98. if os.path.isfile(filename):
  99. mod_filename = filename
  100. elif os.path.isfile("integration/code/%s" % filename):
  101. mod_filename = "integration/code/%s" % filename
  102. elif os.path.isfile("bootstrap/%s" % filename):
  103. mod_filename = "bootstrap/%s" % filename
  104. else:
  105. raise Exception("File not found: %s" % filename)
  106. mod_files.append(mod_filename)
  107. to_compile = to_recompile(address, mod_files)
  108. for mod_filename in to_compile:
  109. if mod_filename.endswith(".mvc"):
  110. model_mode = "MO"
  111. mod_files.remove(mod_filename)
  112. else:
  113. model_mode = mode
  114. if parallel_push:
  115. import threading
  116. threads.append(threading.Thread(target=compile_file, args=[address, mod_filename, mod_filename, model_mode, proc]))
  117. threads[-1].start()
  118. else:
  119. compile_file(address, mod_filename, mod_filename, model_mode, proc)
  120. if parallel_push:
  121. for t in threads:
  122. t.join()
  123. if mode[-1] == "O":
  124. # Fire up the linker
  125. val = execute("link_and_load", [address, username] + mod_files, wait=True)
  126. if val != 0:
  127. raise Exception("Linking error")
  128. # Send the request ...
  129. flush_data(address, parameters)
  130. # ... and wait for replies
  131. if expected is None:
  132. while 1:
  133. val = urllib2.urlopen(urllib2.Request(address, urllib.urlencode({"op": "get_output", "username": username})), timeout=240).read()
  134. val = json.loads(val)
  135. print(val)
  136. for e in expected:
  137. c = len(e) if isinstance(e, set) else 1
  138. for _ in range(c):
  139. val = urllib2.urlopen(urllib2.Request(address, urllib.urlencode({"op": "get_output", "username": username})), timeout=240).read()
  140. val = json.loads(val)
  141. if proc.returncode is not None:
  142. # Modelverse has already terminated, which isn't a good sign!
  143. raise Exception("Modelverse died!")
  144. print("Got %s, expect %s" % (val, e))
  145. if isinstance(e, set):
  146. assert val in e
  147. if val not in e:
  148. return False
  149. else:
  150. assert val == e
  151. if val != e:
  152. return False
  153. # All passed!
  154. return True
  155. except:
  156. raise
  157. finally:
  158. try:
  159. kill(proc)
  160. except UnboundLocalError:
  161. pass
  162. def run_barebone(parameters, expected, interface="0", timeout=False, wait=False, link=None, inputs=[]):
  163. port = getFreePort()
  164. address = "http://127.0.0.1:%i" % port
  165. try:
  166. # Run Modelverse server
  167. proc = execute("run_local_modelverse", [str(port)], wait=False)
  168. # Create user and set interface
  169. timeout_val = 15
  170. start = time.time()
  171. while 1:
  172. proc.poll()
  173. if proc.returncode is not None:
  174. # Modelverse has already terminated, which isn't a good sign!
  175. return False
  176. try:
  177. urllib2.urlopen(urllib2.Request(address, urllib.urlencode({"op": "set_input", "element_type": "V", "value": '"%s"' % username, "username": "user_manager"})), timeout=1).read()
  178. if interface is not None:
  179. urllib2.urlopen(urllib2.Request(address, urllib.urlencode({"op": "set_input", "element_type": "V", "value": interface, "username": username})), timeout=1).read()
  180. break
  181. except:
  182. time.sleep(0.01)
  183. if time.time() - start > timeout_val:
  184. raise
  185. # Send the request
  186. flush_data(address, parameters)
  187. # Now do linking and loading
  188. if link is not None:
  189. # Execute linker
  190. timeout_val = 10
  191. proc2 = execute("link_and_load", [address, username] + link, wait=False)
  192. while proc2.returncode is None:
  193. time.sleep(0.01)
  194. proc2.poll()
  195. timeout_val -= 0.01
  196. if timeout_val < 0:
  197. kill(proc2)
  198. print("Linking timeout expired!")
  199. return False
  200. if proc.returncode is not None:
  201. # Modelverse has already terminated, which isn't a good sign!
  202. return False
  203. for inp in inputs:
  204. urllib2.urlopen(urllib2.Request(address, urllib.urlencode({"op": "set_input", "element_type": "V", "value": inp, "username": username})), timeout=1).read()
  205. proc.poll()
  206. if proc.returncode is not None:
  207. # Modelverse has already terminated, which isn't a good sign!
  208. return False
  209. counter = 0
  210. for e in expected:
  211. print("Expect " + str(e))
  212. c = len(e) if isinstance(e, set) else 1
  213. for _ in range(c):
  214. try:
  215. proc.poll()
  216. if proc.returncode is not None:
  217. # Modelverse has already terminated, which isn't a good sign!
  218. return False
  219. val = urllib2.urlopen(urllib2.Request(address, urllib.urlencode({"op": "get_output", "username": username})), timeout=240 if not timeout else 20).read()
  220. val = json.loads(val)
  221. except:
  222. if timeout:
  223. return True
  224. else:
  225. raise
  226. print("Got %s, expect %s" % (val, e))
  227. if isinstance(e, set):
  228. assert val in e
  229. if val not in e:
  230. return False
  231. else:
  232. assert val == e
  233. if val != e:
  234. return False
  235. # All passed!
  236. return not timeout
  237. finally:
  238. kill(proc)
  239. def get_constructor(code):
  240. with open("__constraint.alc", "w") as f:
  241. f.write(code)
  242. f.flush()
  243. constructors = do_compile("__constraint.alc", "interface/HUTN/grammars/actionlanguage.g", "CS")
  244. return constructors
  245. def get_model_constructor(code):
  246. # First change multiple spaces to a tab
  247. code_fragments = code.split("\n")
  248. code_fragments = [i for i in code_fragments if i.strip() != ""]
  249. code_fragments = [i.replace(" ", "\t") for i in code_fragments]
  250. initial_tabs = min([len(i) - len(i.lstrip("\t")) for i in code_fragments])
  251. code_fragments = [i[initial_tabs:] for i in code_fragments]
  252. code = "\n".join(code_fragments)
  253. with open("__model.mvc", "w") as f:
  254. f.write(code)
  255. f.flush()
  256. constructors = do_compile("__model.mvc", "interface/HUTN/grammars/modelling.g", "M") + ["exit"]
  257. return constructors
  258. def get_raw_model_constructor(code):
  259. # First change multiple spaces to a tab
  260. code_fragments = code.split("\n")
  261. code_fragments = [i for i in code_fragments if i.strip() != ""]
  262. code_fragments = [i.replace(" ", "\t") for i in code_fragments]
  263. initial_tabs = min([len(i) - len(i.lstrip("\t")) for i in code_fragments])
  264. code_fragments = [i[initial_tabs:] for i in code_fragments]
  265. code = "\n".join(code_fragments)
  266. with open("__model.mvc", "w") as f:
  267. f.write(code)
  268. f.flush()
  269. constructors = do_compile("__model.mvc", "interface/HUTN/grammars/modelling_raw.g", "MR") + ["exit"]
  270. return constructors