utils.py 9.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287
  1. import unittest
  2. import sys
  3. import os
  4. import sys
  5. import time
  6. import json
  7. import urllib
  8. import urllib2
  9. import subprocess
  10. import signal
  11. import random
  12. import operator
  13. sys.path.append("interface/HUTN")
  14. sys.path.append("scripts")
  15. from hutn_compiler.compiler import main as do_compile
  16. from check_objects import to_recompile
  17. USERNAME = "test_user"
  18. PARALLEL_PUSH = True
  19. BOOTSTRAP_FOLDER_NAME = "bootstrap"
  20. CURRENT_FOLDER_NAME = "performance"
  21. PORTS = set()
  22. OPTIMIZATION_LEVEL_LEGACY_INTERPRETER = "legacy-interpreter"
  23. OPTIMIZATION_LEVEL_INTERPRETER = "interpreter"
  24. OPTIMIZATION_LEVEL_BASELINE_JIT = "baseline-jit"
  25. OPTIMIZATION_LEVEL_BASELINE_JIT_NO_THUNKS = "baseline-jit,no-thunks"
  26. ALL_OPTIMIZATION_LEVELS = [
  27. OPTIMIZATION_LEVEL_LEGACY_INTERPRETER,
  28. OPTIMIZATION_LEVEL_INTERPRETER,
  29. OPTIMIZATION_LEVEL_BASELINE_JIT,
  30. OPTIMIZATION_LEVEL_BASELINE_JIT_NO_THUNKS
  31. ]
  32. class ModelverseTerminated(Exception):
  33. """An exception that tells the user that the Modelverse has terminated."""
  34. pass
  35. def get_code_folder_name():
  36. """Gets the name of the code folder."""
  37. return '%s/code' % CURRENT_FOLDER_NAME
  38. def get_free_port():
  39. """Gets a unique new port."""
  40. while 1:
  41. port = random.randint(10000, 20000)
  42. # Check if this port is in the set of ports.
  43. if port not in PORTS:
  44. # We have found a unique port. Add it to the set and return.
  45. PORTS.add(port)
  46. return port
  47. def execute(scriptname, parameters=None, wait=False):
  48. """Runs a script."""
  49. if os.name not in ["nt", "posix"]:
  50. # Stop now, as we would have no clue on how to kill its subtree
  51. raise Exception("Unknown OS version: " + str(os.name))
  52. command = [sys.executable, "scripts/%s.py" % scriptname] + (
  53. [] if parameters is None else parameters)
  54. if wait:
  55. return subprocess.call(command, shell=False)
  56. else:
  57. return subprocess.Popen(command, shell=False)
  58. def kill(process):
  59. """Kills the given process."""
  60. if os.name == "nt":
  61. subprocess.call(["taskkill", "/F", "/T", "/PID", "%i" % process.pid])
  62. elif os.name == "posix":
  63. subprocess.call(["pkill", "-P", "%i" % process.pid])
  64. def set_input_data(address, data):
  65. """Sets the Modelverse program's input data."""
  66. if data is not None:
  67. urllib2.urlopen(
  68. urllib2.Request(
  69. address,
  70. urllib.urlencode(
  71. {"op": "set_input", "data": json.dumps(data), "username": USERNAME})),
  72. timeout=10).read()
  73. else:
  74. return []
  75. def compile_file(address, mod_filename, filename, mode, proc):
  76. """Compiles the given file."""
  77. # Load in the file required
  78. try:
  79. timeout_val = 240
  80. username = str(random.random())
  81. while 1:
  82. proc2 = execute(
  83. "compile", [address, mod_filename, username, filename, mode], wait=False)
  84. if proc.returncode is not None:
  85. # Modelverse has already terminated, which isn't a good sign!
  86. raise Exception("Modelverse died!")
  87. while proc2.returncode is None:
  88. time.sleep(0.01)
  89. proc2.poll()
  90. timeout_val -= 0.01
  91. if timeout_val < 0:
  92. kill(proc2)
  93. print("Compilation timeout expired!")
  94. return False
  95. if proc2.returncode != 2:
  96. break
  97. # Make sure everything stopped correctly
  98. assert proc2.returncode == 0
  99. if proc2.returncode != 0:
  100. return False
  101. except:
  102. raise
  103. finally:
  104. try:
  105. kill(proc2)
  106. except UnboundLocalError:
  107. pass
  108. def run_file(files, parameters, mode, handle_output, optimization_level=None):
  109. """Compiles the given sequence of files, feeds them the given input in the given mode,
  110. and handles their output."""
  111. # Resolve file
  112. import os.path
  113. time.sleep(0.01)
  114. port = get_free_port()
  115. address = "http://127.0.0.1:%i" % port
  116. try:
  117. # Run Modelverse server
  118. modelverse_args = [str(port)]
  119. if optimization_level is not None:
  120. modelverse_args.append('--kernel=%s' % optimization_level)
  121. proc = execute("run_local_modelverse", modelverse_args, wait=False)
  122. threads = []
  123. mod_files = []
  124. for filename in files:
  125. if os.path.isfile("%s/%s" % (get_code_folder_name(), filename)):
  126. mod_filename = "%s/%s" % (get_code_folder_name(), filename)
  127. elif os.path.isfile("%s/%s" % (BOOTSTRAP_FOLDER_NAME, filename)):
  128. mod_filename = "%s/%s" % (BOOTSTRAP_FOLDER_NAME, filename)
  129. else:
  130. raise Exception("File not found: %s" % filename)
  131. mod_files.append(mod_filename)
  132. to_compile = to_recompile(address, mod_files)
  133. for mod_filename in to_compile:
  134. if PARALLEL_PUSH:
  135. import threading
  136. threads.append(
  137. threading.Thread(
  138. target=compile_file,
  139. args=[address, mod_filename, mod_filename, mode, proc]))
  140. threads[-1].start()
  141. else:
  142. compile_file(address, mod_filename, mod_filename, mode, proc)
  143. if PARALLEL_PUSH:
  144. for t in threads:
  145. t.join()
  146. if mode[-1] == "O":
  147. # Fire up the linker
  148. val = execute("link_and_load", [address, USERNAME] + mod_files, wait=True)
  149. if val != 0:
  150. raise Exception("Linking error")
  151. # Send the request ...
  152. set_input_data(address, parameters)
  153. # ... and wait for replies
  154. while 1:
  155. val = urllib2.urlopen(
  156. urllib2.Request(
  157. address,
  158. urllib.urlencode({"op": "get_output", "username": USERNAME})),
  159. timeout=240).read()
  160. val = json.loads(val)
  161. if proc.returncode is not None:
  162. # Modelverse has terminated. This may or may not be what we want.
  163. raise ModelverseTerminated()
  164. if not handle_output(val):
  165. return
  166. # All passed!
  167. return
  168. except:
  169. raise
  170. finally:
  171. try:
  172. kill(proc)
  173. except UnboundLocalError:
  174. pass
  175. def run_file_to_completion(files, parameters, mode):
  176. """Compiles the given sequence of files, feeds them the given input in the given mode,
  177. and then collects and returns output."""
  178. results = []
  179. def handle_output(output):
  180. """Appends the given output to the list of results."""
  181. results.append(output)
  182. return True
  183. try:
  184. run_file(files, parameters, mode, handle_output)
  185. except ModelverseTerminated:
  186. return results
  187. def run_file_fixed_output_count(files, parameters, mode, output_count, optimization_level=None):
  188. """Compiles the given sequence of files, feeds them the given input in the given mode,
  189. and then collects and returns a fixed number of outputs."""
  190. results = []
  191. def handle_output(output):
  192. """Appends the given output to the list of results."""
  193. if len(results) < output_count:
  194. results.append(output)
  195. return True
  196. else:
  197. return False
  198. run_file(files, parameters, mode, handle_output, optimization_level)
  199. return results
  200. def run_file_single_output(files, parameters, mode, optimization_level=None):
  201. """Compiles the given sequence of files, feeds them the given input in the given mode,
  202. and then collects and returns a single output."""
  203. return run_file_fixed_output_count(files, parameters, mode, 1, optimization_level)[0]
  204. def run_perf_test(files, parameters, optimization_level, n_iterations=1):
  205. """Compiles the given sequence of files, feeds them the given input in the given mode,
  206. and then collects their output. This process is repeated n_iterations times. The
  207. return value is the average of all outputs."""
  208. result = 0.0
  209. for _ in xrange(n_iterations):
  210. result += float(
  211. run_file_single_output(
  212. files, parameters + [0], 'CO',
  213. optimization_level)) / float(n_iterations)
  214. return result
  215. def format_output(output):
  216. """Formats the output of `run_file_to_completion` as a string."""
  217. return '\n'.join(output)
  218. def define_perf_test(target_class, test_function, optimization_level):
  219. """Defines a performance test in the given class. The performance test calls the given function
  220. at the given optimization level."""
  221. setattr(
  222. target_class,
  223. 'test_%s' % optimization_level.replace('-', '_').lower(),
  224. lambda self: test_function(self, optimization_level))
  225. def define_perf_tests(target_class, test_function):
  226. """Defines performance tests in the given class. Each test calls the given function."""
  227. for optimization_level in ALL_OPTIMIZATION_LEVELS:
  228. define_perf_test(target_class, test_function, optimization_level)
  229. DEFAULT_PERF_FILE_NAME = 'perf_data.txt'
  230. def write_perf_to_file(test_name, optimization_level, result, file_name=DEFAULT_PERF_FILE_NAME):
  231. """Writes performance data to a file."""
  232. with open(file_name, "a") as perf_file:
  233. perf_file.write('%s:%s:%f\n' % (test_name, optimization_level, result))
  234. def parse_perf_data(file_name):
  235. """Parses the performance data in the given file."""
  236. results = {}
  237. with open(file_name, 'r') as perf_file:
  238. for line in perf_file.readlines():
  239. test_name, optimization_level, result = line.strip().split(':')
  240. if optimization_level not in results:
  241. results[optimization_level] = []
  242. results[optimization_level].append((test_name, result))
  243. return sorted(results.items(), key=operator.itemgetter(1))