utils.py 7.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243
  1. import unittest
  2. import sys
  3. import os
  4. import sys
  5. import time
  6. import json
  7. import urllib
  8. import urllib2
  9. import subprocess
  10. import signal
  11. import random
  12. sys.path.append("interface/HUTN")
  13. sys.path.append("scripts")
  14. from hutn_compiler.compiler import main as do_compile
  15. from check_objects import to_recompile
  16. USERNAME = "test_user"
  17. PARALLEL_PUSH = True
  18. BOOTSTRAP_FOLDER_NAME = "bootstrap"
  19. CURRENT_FOLDER_NAME = "performance"
  20. PORTS = set()
  21. OPTIMIZATION_LEVEL_INTERPRETER = "interpreter"
  22. OPTIMIZATION_LEVEL_BASELINE_JIT = "baseline-jit"
  23. class ModelverseTerminated(Exception):
  24. """An exception that tells the user that the Modelverse has terminated."""
  25. pass
  26. def get_code_folder_name():
  27. """Gets the name of the code folder."""
  28. return '%s/code' % CURRENT_FOLDER_NAME
  29. def get_free_port():
  30. """Gets a unique new port."""
  31. while 1:
  32. port = random.randint(10000, 20000)
  33. # Check if this port is in the set of ports.
  34. if port not in PORTS:
  35. # We have found a unique port. Add it to the set and return.
  36. PORTS.add(port)
  37. return port
  38. def execute(scriptname, parameters=None, wait=False):
  39. """Runs a script."""
  40. if os.name not in ["nt", "posix"]:
  41. # Stop now, as we would have no clue on how to kill its subtree
  42. raise Exception("Unknown OS version: " + str(os.name))
  43. command = [sys.executable, "scripts/%s.py" % scriptname] + (
  44. [] if parameters is None else parameters)
  45. if wait:
  46. return subprocess.call(command, shell=False)
  47. else:
  48. return subprocess.Popen(command, shell=False)
  49. def kill(process):
  50. """Kills the given process."""
  51. if os.name == "nt":
  52. subprocess.call(["taskkill", "/F", "/T", "/PID", "%i" % process.pid])
  53. elif os.name == "posix":
  54. subprocess.call(["pkill", "-P", "%i" % process.pid])
  55. def set_input_data(address, data):
  56. """Sets the Modelverse program's input data."""
  57. if data is not None:
  58. urllib2.urlopen(
  59. urllib2.Request(
  60. address,
  61. urllib.urlencode(
  62. {"op": "set_input", "data": json.dumps(data), "username": USERNAME})),
  63. timeout=10).read()
  64. else:
  65. return []
  66. def compile_file(address, mod_filename, filename, mode, proc):
  67. """Compiles the given file."""
  68. # Load in the file required
  69. try:
  70. timeout_val = 240
  71. username = str(random.random())
  72. while 1:
  73. proc2 = execute(
  74. "compile", [address, mod_filename, username, filename, mode], wait=False)
  75. if proc.returncode is not None:
  76. # Modelverse has already terminated, which isn't a good sign!
  77. raise Exception("Modelverse died!")
  78. while proc2.returncode is None:
  79. time.sleep(0.01)
  80. proc2.poll()
  81. timeout_val -= 0.01
  82. if timeout_val < 0:
  83. kill(proc2)
  84. print("Compilation timeout expired!")
  85. return False
  86. if proc2.returncode != 2:
  87. break
  88. # Make sure everything stopped correctly
  89. assert proc2.returncode == 0
  90. if proc2.returncode != 0:
  91. return False
  92. except:
  93. raise
  94. finally:
  95. try:
  96. kill(proc2)
  97. except UnboundLocalError:
  98. pass
  99. def run_file(files, parameters, mode, handle_output):
  100. """Compiles the given sequence of files, feeds them the given input in the given mode,
  101. and handles their output."""
  102. # Resolve file
  103. import os.path
  104. time.sleep(0.01)
  105. port = get_free_port()
  106. address = "http://127.0.0.1:%i" % port
  107. try:
  108. # Run Modelverse server
  109. proc = execute("run_local_modelverse", [str(port)], wait=False)
  110. threads = []
  111. mod_files = []
  112. for filename in files:
  113. if os.path.isfile("%s/%s" % (get_code_folder_name(), filename)):
  114. mod_filename = "%s/%s" % (get_code_folder_name(), filename)
  115. elif os.path.isfile("%s/%s" % (BOOTSTRAP_FOLDER_NAME, filename)):
  116. mod_filename = "%s/%s" % (BOOTSTRAP_FOLDER_NAME, filename)
  117. else:
  118. raise Exception("File not found: %s" % filename)
  119. mod_files.append(mod_filename)
  120. to_compile = to_recompile(address, mod_files)
  121. for mod_filename in to_compile:
  122. if PARALLEL_PUSH:
  123. import threading
  124. threads.append(
  125. threading.Thread(
  126. target=compile_file,
  127. args=[address, mod_filename, mod_filename, mode, proc]))
  128. threads[-1].start()
  129. else:
  130. compile_file(address, mod_filename, mod_filename, mode, proc)
  131. if PARALLEL_PUSH:
  132. for t in threads:
  133. t.join()
  134. if mode[-1] == "O":
  135. # Fire up the linker
  136. val = execute("link_and_load", [address, USERNAME] + mod_files, wait=True)
  137. if val != 0:
  138. raise Exception("Linking error")
  139. # Send the request ...
  140. set_input_data(address, parameters)
  141. # ... and wait for replies
  142. while 1:
  143. val = urllib2.urlopen(
  144. urllib2.Request(
  145. address,
  146. urllib.urlencode({"op": "get_output", "username": USERNAME})),
  147. timeout=240).read()
  148. val = json.loads(val)
  149. if proc.returncode is not None:
  150. # Modelverse has terminated. This may or may not be what we want.
  151. raise ModelverseTerminated()
  152. if not handle_output(val):
  153. return
  154. # All passed!
  155. return
  156. except:
  157. raise
  158. finally:
  159. try:
  160. kill(proc)
  161. except UnboundLocalError:
  162. pass
  163. def run_file_to_completion(files, parameters, mode):
  164. """Compiles the given sequence of files, feeds them the given input in the given mode,
  165. and then collects and returns output."""
  166. results = []
  167. def handle_output(output):
  168. """Appends the given output to the list of results."""
  169. results.append(output)
  170. return True
  171. try:
  172. run_file(files, parameters, mode, handle_output)
  173. except ModelverseTerminated:
  174. return results
  175. def run_file_fixed_output_count(files, parameters, mode, output_count):
  176. """Compiles the given sequence of files, feeds them the given input in the given mode,
  177. and then collects and returns a fixed number of outputs."""
  178. results = []
  179. def handle_output(output):
  180. """Appends the given output to the list of results."""
  181. if len(results) < output_count:
  182. results.append(output)
  183. return True
  184. else:
  185. return False
  186. run_file(files, parameters, mode, handle_output)
  187. return results
  188. def run_file_single_output(files, parameters, mode):
  189. """Compiles the given sequence of files, feeds them the given input in the given mode,
  190. and then collects and returns a single output."""
  191. return run_file_fixed_output_count(files, parameters, mode, 1)[0]
  192. def run_perf_test(files, parameters, optimization_level, n_iterations=1):
  193. """Compiles the given sequence of files, feeds them the given input in the given mode,
  194. and then collects their output. This process is repeated n_iterations times. The
  195. return value is the average of all outputs."""
  196. result = 0.0
  197. for _ in xrange(n_iterations):
  198. result += float(
  199. run_file_single_output(
  200. files, [optimization_level] + parameters + [0], 'CO')) / float(n_iterations)
  201. return result
  202. def format_output(output):
  203. """Formats the output of `run_file_to_completion` as a string."""
  204. return '\n'.join(output)