utils.py 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308
  1. import unittest
  2. import sys
  3. import os
  4. import sys
  5. import time
  6. import json
  7. import urllib
  8. import urllib2
  9. import subprocess
  10. import signal
  11. import random
  12. sys.path.append("interface/HUTN")
  13. sys.path.append("scripts")
  14. from hutn_compiler.compiler import main as do_compile
  15. from check_objects import to_recompile
  16. username = "test_user"
  17. parallel_push = True
  18. ports = []
  19. def getFreePort():
  20. while 1:
  21. port = random.randint(10000, 20000)
  22. ports.append(port)
  23. exists = False
  24. for p in ports:
  25. if p == port:
  26. if not exists:
  27. # We have hopefully found our own
  28. exists = True
  29. else:
  30. # We seem to be the second entry, so chose another one
  31. ports.remove(port)
  32. break
  33. else:
  34. # Didn't find a duplicate
  35. return port
  36. def execute(scriptname, parameters=[], wait=False):
  37. if os.name not in ["nt", "posix"]:
  38. # Stop now, as we would have no clue on how to kill its subtree
  39. raise Exception("Unknown OS version: " + str(os.name))
  40. command = [sys.executable, "scripts/%s.py" % scriptname] + parameters
  41. if wait:
  42. return subprocess.call(command, shell=False)
  43. else:
  44. return subprocess.Popen(command, shell=False)
  45. def kill(process):
  46. if os.name == "nt":
  47. subprocess.call(["taskkill", "/F", "/T", "/PID", "%i" % process.pid])
  48. elif os.name == "posix":
  49. subprocess.call(["pkill", "-P", "%i" % process.pid])
  50. def flush_data(address, data):
  51. if data:
  52. urllib2.urlopen(urllib2.Request(address, urllib.urlencode({"op": "set_input", "data": json.dumps(data), "username": username})), timeout=10).read()
  53. return []
  54. def compile_file(address, mod_filename, filename, mode, proc):
  55. # Load in the file required
  56. try:
  57. timeout_val = 240
  58. import random
  59. username = str(random.random())
  60. while 1:
  61. proc2 = execute("compile", [address, mod_filename, username, filename, mode], wait=False)
  62. if proc.returncode is not None:
  63. # Modelverse has already terminated, which isn't a good sign!
  64. raise Exception("Modelverse died!")
  65. while proc2.returncode is None:
  66. time.sleep(0.01)
  67. proc2.poll()
  68. timeout_val -= 0.01
  69. if timeout_val < 0:
  70. kill(proc2)
  71. print("Compilation timeout expired!")
  72. return False
  73. if proc2.returncode != 2:
  74. break
  75. # Make sure everything stopped correctly
  76. assert proc2.returncode == 0
  77. if proc2.returncode != 0:
  78. return False
  79. except:
  80. raise
  81. finally:
  82. try:
  83. kill(proc2)
  84. except UnboundLocalError:
  85. pass
  86. def run_file(files, parameters, expected, mode, wait=False):
  87. # Resolve file
  88. import os.path
  89. if wait is True:
  90. expected = None
  91. time.sleep(0.01)
  92. port = getFreePort()
  93. address = "http://127.0.0.1:%i" % port
  94. try:
  95. # Run Modelverse server
  96. proc = execute("run_local_modelverse", [str(port)], wait=False)
  97. threads = []
  98. mod_files = []
  99. for filename in files:
  100. if os.path.isfile(filename):
  101. mod_filename = filename
  102. elif os.path.isfile("integration/code/%s" % filename):
  103. mod_filename = "integration/code/%s" % filename
  104. elif os.path.isfile("bootstrap/%s" % filename):
  105. mod_filename = "bootstrap/%s" % filename
  106. else:
  107. raise Exception("File not found: %s" % filename)
  108. mod_files.append(mod_filename)
  109. to_compile = to_recompile(address, mod_files)
  110. for mod_filename in to_compile:
  111. if mod_filename.endswith(".mvc"):
  112. model_mode = "MO"
  113. mod_files.remove(mod_filename)
  114. else:
  115. model_mode = mode
  116. if parallel_push:
  117. import threading
  118. threads.append(threading.Thread(target=compile_file, args=[address, mod_filename, mod_filename, model_mode, proc]))
  119. threads[-1].start()
  120. else:
  121. compile_file(address, mod_filename, mod_filename, model_mode, proc)
  122. if parallel_push:
  123. for t in threads:
  124. t.join()
  125. if mode[-1] == "O":
  126. # Fire up the linker
  127. val = execute("link_and_load", [address, username] + mod_files, wait=True)
  128. if val != 0:
  129. raise Exception("Linking error")
  130. # Send the request ...
  131. flush_data(address, parameters)
  132. # ... and wait for replies
  133. if expected is None:
  134. while 1:
  135. val = urllib2.urlopen(urllib2.Request(address, urllib.urlencode({"op": "get_output", "username": username})), timeout=240).read()
  136. val = json.loads(val)
  137. print(val)
  138. for e in expected:
  139. c = len(e) if isinstance(e, set) else 1
  140. for _ in range(c):
  141. val = urllib2.urlopen(urllib2.Request(address, urllib.urlencode({"op": "get_output", "username": username})), timeout=240).read()
  142. val = json.loads(val)
  143. if proc.returncode is not None:
  144. # Modelverse has already terminated, which isn't a good sign!
  145. raise Exception("Modelverse died!")
  146. print("Got %s, expect %s" % (val, e))
  147. if isinstance(e, set):
  148. assert val in e
  149. if val not in e:
  150. return False
  151. else:
  152. assert val == e
  153. if val != e:
  154. return False
  155. # All passed!
  156. return True
  157. except:
  158. raise
  159. finally:
  160. try:
  161. kill(proc)
  162. except UnboundLocalError:
  163. pass
  164. def run_barebone(parameters, expected, interface="0", timeout=False, wait=False, link=None, inputs=[]):
  165. port = getFreePort()
  166. address = "http://127.0.0.1:%i" % port
  167. try:
  168. # Run Modelverse server
  169. proc = execute("run_local_modelverse", [str(port)], wait=False)
  170. # Create user and set interface
  171. timeout_val = 15
  172. start = time.time()
  173. while 1:
  174. proc.poll()
  175. if proc.returncode is not None:
  176. # Modelverse has already terminated, which isn't a good sign!
  177. return False
  178. try:
  179. urllib2.urlopen(urllib2.Request(address, urllib.urlencode({"op": "set_input", "element_type": "V", "value": '"%s"' % username, "username": "user_manager"})), timeout=1).read()
  180. if interface is not None:
  181. urllib2.urlopen(urllib2.Request(address, urllib.urlencode({"op": "set_input", "element_type": "V", "value": interface, "username": username})), timeout=1).read()
  182. break
  183. except:
  184. time.sleep(0.01)
  185. if time.time() - start > timeout_val:
  186. raise
  187. # Send the request
  188. flush_data(address, parameters)
  189. # Now do linking and loading
  190. if link is not None:
  191. # Execute linker
  192. timeout_val = 10
  193. proc2 = execute("link_and_load", [address, username] + link, wait=False)
  194. while proc2.returncode is None:
  195. time.sleep(0.01)
  196. proc2.poll()
  197. timeout_val -= 0.01
  198. if timeout_val < 0:
  199. kill(proc2)
  200. print("Linking timeout expired!")
  201. return False
  202. if proc.returncode is not None:
  203. # Modelverse has already terminated, which isn't a good sign!
  204. return False
  205. for inp in inputs:
  206. urllib2.urlopen(urllib2.Request(address, urllib.urlencode({"op": "set_input", "element_type": "V", "value": inp, "username": username})), timeout=1).read()
  207. proc.poll()
  208. if proc.returncode is not None:
  209. # Modelverse has already terminated, which isn't a good sign!
  210. return False
  211. counter = 0
  212. for e in expected:
  213. print("Expect " + str(e))
  214. c = len(e) if isinstance(e, set) else 1
  215. for _ in range(c):
  216. try:
  217. proc.poll()
  218. if proc.returncode is not None:
  219. # Modelverse has already terminated, which isn't a good sign!
  220. return False
  221. val = urllib2.urlopen(urllib2.Request(address, urllib.urlencode({"op": "get_output", "username": username})), timeout=240 if not timeout else 20).read()
  222. val = json.loads(val)
  223. except:
  224. if timeout:
  225. return True
  226. else:
  227. raise
  228. print("Got %s, expect %s" % (val, e))
  229. if isinstance(e, set):
  230. assert val in e
  231. if val not in e:
  232. return False
  233. else:
  234. assert val == e
  235. if val != e:
  236. return False
  237. # All passed!
  238. return not timeout
  239. finally:
  240. kill(proc)
  241. def get_constructor(code):
  242. with open("__constraint.alc", "w") as f:
  243. f.write(code)
  244. f.flush()
  245. constructors = do_compile("__constraint.alc", "interface/HUTN/grammars/actionlanguage.g", "CS")
  246. return constructors
  247. def get_model_constructor(code):
  248. # First change multiple spaces to a tab
  249. code_fragments = code.split("\n")
  250. code_fragments = [i for i in code_fragments if i.strip() != ""]
  251. code_fragments = [i.replace(" ", "\t") for i in code_fragments]
  252. initial_tabs = min([len(i) - len(i.lstrip("\t")) for i in code_fragments])
  253. code_fragments = [i[initial_tabs:] for i in code_fragments]
  254. code = "\n".join(code_fragments)
  255. with open("__model.mvc", "w") as f:
  256. f.write(code)
  257. f.flush()
  258. constructors = do_compile("__model.mvc", "interface/HUTN/grammars/modelling.g", "M") + ["exit"]
  259. return constructors