utils.py 6.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205
  1. import unittest
  2. import sys
  3. import os
  4. import sys
  5. import time
  6. import json
  7. import urllib
  8. import urllib2
  9. import subprocess
  10. import signal
  11. import random
  12. sys.path.append("interface/HUTN")
  13. sys.path.append("scripts")
  14. from hutn_compiler.compiler import main as do_compile
  15. from check_objects import to_recompile
  16. USERNAME = "test_user"
  17. PARALLEL_PUSH = True
  18. BOOTSTRAP_FOLDER_NAME = "bootstrap"
  19. CURRENT_FOLDER_NAME = "performance"
  20. PORTS = set()
  21. class ModelverseTerminated(Exception):
  22. """An exception that tells the user that the Modelverse has terminated."""
  23. pass
  24. def get_code_folder_name():
  25. """Gets the name of the code folder."""
  26. return '%s/code' % CURRENT_FOLDER_NAME
  27. def get_free_port():
  28. """Gets a unique new port."""
  29. while 1:
  30. port = random.randint(10000, 20000)
  31. # Check if this port is in the set of ports.
  32. if port not in PORTS:
  33. # We have found a unique port. Add it to the set and return.
  34. PORTS.add(port)
  35. return port
  36. def execute(scriptname, parameters=None, wait=False):
  37. """Runs a script."""
  38. if os.name not in ["nt", "posix"]:
  39. # Stop now, as we would have no clue on how to kill its subtree
  40. raise Exception("Unknown OS version: " + str(os.name))
  41. command = [sys.executable, "scripts/%s.py" % scriptname] + (
  42. [] if parameters is None else parameters)
  43. if wait:
  44. return subprocess.call(command, shell=False)
  45. else:
  46. return subprocess.Popen(command, shell=False)
  47. def kill(process):
  48. """Kills the given process."""
  49. if os.name == "nt":
  50. subprocess.call(["taskkill", "/F", "/T", "/PID", "%i" % process.pid])
  51. elif os.name == "posix":
  52. subprocess.call(["pkill", "-P", "%i" % process.pid])
  53. def set_input_data(address, data):
  54. """Sets the Modelverse program's input data."""
  55. if data is not None:
  56. urllib2.urlopen(
  57. urllib2.Request(
  58. address,
  59. urllib.urlencode(
  60. {"op": "set_input", "data": json.dumps(data), "username": USERNAME})),
  61. timeout=10).read()
  62. else:
  63. return []
  64. def compile_file(address, mod_filename, filename, mode, proc):
  65. """Compiles the given file."""
  66. # Load in the file required
  67. try:
  68. timeout_val = 240
  69. username = str(random.random())
  70. while 1:
  71. proc2 = execute(
  72. "compile", [address, mod_filename, username, filename, mode], wait=False)
  73. if proc.returncode is not None:
  74. # Modelverse has already terminated, which isn't a good sign!
  75. raise Exception("Modelverse died!")
  76. while proc2.returncode is None:
  77. time.sleep(0.01)
  78. proc2.poll()
  79. timeout_val -= 0.01
  80. if timeout_val < 0:
  81. kill(proc2)
  82. print("Compilation timeout expired!")
  83. return False
  84. if proc2.returncode != 2:
  85. break
  86. # Make sure everything stopped correctly
  87. assert proc2.returncode == 0
  88. if proc2.returncode != 0:
  89. return False
  90. except:
  91. raise
  92. finally:
  93. try:
  94. kill(proc2)
  95. except UnboundLocalError:
  96. pass
  97. def run_file(files, parameters, mode, handle_output):
  98. """Compiles the given sequence of files, feeds them the given input in the given mode,
  99. and handles their output."""
  100. # Resolve file
  101. import os.path
  102. time.sleep(0.01)
  103. port = get_free_port()
  104. address = "http://127.0.0.1:%i" % port
  105. try:
  106. # Run Modelverse server
  107. proc = execute("run_local_modelverse", [str(port)], wait=False)
  108. threads = []
  109. mod_files = []
  110. for filename in files:
  111. if os.path.isfile("%s/%s" % (get_code_folder_name(), filename)):
  112. mod_filename = "%s/%s" % (get_code_folder_name(), filename)
  113. elif os.path.isfile("%s/%s" % (BOOTSTRAP_FOLDER_NAME, filename)):
  114. mod_filename = "%s/%s" % (BOOTSTRAP_FOLDER_NAME, filename)
  115. else:
  116. raise Exception("File not found: %s" % filename)
  117. mod_files.append(mod_filename)
  118. to_compile = to_recompile(address, mod_files)
  119. for mod_filename in to_compile:
  120. if PARALLEL_PUSH:
  121. import threading
  122. threads.append(
  123. threading.Thread(
  124. target=compile_file,
  125. args=[address, mod_filename, mod_filename, mode, proc]))
  126. threads[-1].start()
  127. else:
  128. compile_file(address, mod_filename, mod_filename, mode, proc)
  129. if PARALLEL_PUSH:
  130. for t in threads:
  131. t.join()
  132. if mode[-1] == "O":
  133. # Fire up the linker
  134. val = execute("link_and_load", [address, USERNAME] + mod_files, wait=True)
  135. if val != 0:
  136. raise Exception("Linking error")
  137. # Send the request ...
  138. set_input_data(address, parameters)
  139. # ... and wait for replies
  140. while 1:
  141. val = urllib2.urlopen(
  142. urllib2.Request(
  143. address,
  144. urllib.urlencode({"op": "get_output", "username": USERNAME})),
  145. timeout=240).read()
  146. val = json.loads(val)
  147. if proc.returncode is not None:
  148. # Modelverse has terminated. This may or may not be what we want.
  149. raise ModelverseTerminated()
  150. if not handle_output(val):
  151. return
  152. # All passed!
  153. return
  154. except:
  155. raise
  156. finally:
  157. try:
  158. kill(proc)
  159. except UnboundLocalError:
  160. pass
  161. def run_file_to_completion(files, parameters, mode):
  162. """Compiles the given sequence of files, feeds them the given input in the given mode,
  163. then collects and returns their output."""
  164. results = []
  165. def handle_output(output):
  166. """Appends the given output to the list of results."""
  167. results.append(output)
  168. return True
  169. try:
  170. run_file(files, parameters, mode, handle_output)
  171. except ModelverseTerminated:
  172. return results