utils.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404
  1. import unittest
  2. import sys
  3. import os
  4. import sys
  5. import time
  6. import json
  7. import urllib
  8. import urllib2
  9. import subprocess
  10. import signal
  11. import random
  12. import operator
  13. from collections import defaultdict
  14. sys.path.append("interface/HUTN")
  15. sys.path.append("scripts")
  16. from hutn_compiler.compiler import main as do_compile
  17. from check_objects import to_recompile
  18. USERNAME = "test_task"
  19. PARALLEL_PUSH = True
  20. BOOTSTRAP_FOLDER_NAME = "bootstrap"
  21. CURRENT_FOLDER_NAME = "performance"
  22. PORTS = set()
  23. OPTIMIZATION_LEVEL_LEGACY_INTERPRETER = "legacy-interpreter"
  24. OPTIMIZATION_LEVEL_INTERPRETER = "interpreter"
  25. OPTIMIZATION_LEVEL_BYTECODE_INTERPRETER = "bytecode-interpreter"
  26. OPTIMIZATION_LEVEL_BASELINE_JIT = "baseline-jit"
  27. OPTIMIZATION_LEVEL_BASELINE_JIT_NO_THUNKS = "baseline-jit,no-thunks"
  28. OPTIMIZATION_LEVEL_FAST_JIT = "fast-jit"
  29. OPTIMIZATION_LEVEL_FAST_JIT_NO_NOPS = "fast-jit,no-insert-nops"
  30. OPTIMIZATION_LEVEL_ADAPTIVE_JIT_FAVOR_LARGE_FUNCTIONS = "adaptive-jit-favor-large-functions"
  31. OPTIMIZATION_LEVEL_ADAPTIVE_JIT_FAVOR_SMALL_FUNCTIONS = "adaptive-jit-favor-small-functions"
  32. OPTIMIZATION_LEVEL_ADAPTIVE_JIT_FAVOR_LOOPS = "adaptive-jit-favor-loops"
  33. ALL_OPTIMIZATION_LEVELS = [
  34. OPTIMIZATION_LEVEL_LEGACY_INTERPRETER,
  35. OPTIMIZATION_LEVEL_INTERPRETER,
  36. OPTIMIZATION_LEVEL_BYTECODE_INTERPRETER,
  37. OPTIMIZATION_LEVEL_BASELINE_JIT,
  38. OPTIMIZATION_LEVEL_BASELINE_JIT_NO_THUNKS,
  39. OPTIMIZATION_LEVEL_FAST_JIT,
  40. OPTIMIZATION_LEVEL_FAST_JIT_NO_NOPS,
  41. OPTIMIZATION_LEVEL_ADAPTIVE_JIT_FAVOR_LARGE_FUNCTIONS,
  42. OPTIMIZATION_LEVEL_ADAPTIVE_JIT_FAVOR_SMALL_FUNCTIONS,
  43. OPTIMIZATION_LEVEL_ADAPTIVE_JIT_FAVOR_LOOPS
  44. ]
  45. class ModelverseTerminated(Exception):
  46. """An exception that tells the task that the Modelverse has terminated."""
  47. pass
  48. def get_code_folder_name():
  49. """Gets the name of the code folder."""
  50. return '%s/code' % CURRENT_FOLDER_NAME
  51. def get_free_port():
  52. """Gets a unique new port."""
  53. while 1:
  54. port = random.randint(10000, 20000)
  55. # Check if this port is in the set of ports.
  56. if port not in PORTS:
  57. # We have found a unique port. Add it to the set and return.
  58. PORTS.add(port)
  59. return port
  60. def execute(scriptname, parameters=None, wait=False):
  61. """Runs a script."""
  62. if os.name not in ["nt", "posix"]:
  63. # Stop now, as we would have no clue on how to kill its subtree
  64. raise Exception("Unknown OS version: " + str(os.name))
  65. command = [sys.executable, "scripts/%s.py" % scriptname] + (
  66. [] if parameters is None else parameters)
  67. if wait:
  68. return subprocess.call(command, shell=False)
  69. else:
  70. return subprocess.Popen(command, shell=False)
  71. def kill(process):
  72. """Kills the given process."""
  73. if os.name == "nt":
  74. subprocess.call(["taskkill", "/F", "/T", "/PID", "%i" % process.pid])
  75. elif os.name == "posix":
  76. subprocess.call(["pkill", "-P", "%i" % process.pid])
  77. def set_input_data(address, data):
  78. """Sets the Modelverse program's input data."""
  79. if data is not None:
  80. urllib2.urlopen(
  81. urllib2.Request(
  82. address,
  83. urllib.urlencode(
  84. {"op": "set_input", "data": json.dumps(data), "taskname": USERNAME})),
  85. timeout=10).read()
  86. else:
  87. return []
  88. def compile_file(address, mod_filename, filename, mode, proc):
  89. """Compiles the given file."""
  90. # Load in the file required
  91. try:
  92. timeout_val = 240
  93. taskname = str(random.random())
  94. while 1:
  95. proc2 = execute(
  96. "compile", [address, mod_filename, taskname, filename, mode], wait=False)
  97. if proc.returncode is not None:
  98. # Modelverse has already terminated, which isn't a good sign!
  99. raise Exception("Modelverse died!")
  100. while proc2.returncode is None:
  101. time.sleep(0.01)
  102. proc2.poll()
  103. timeout_val -= 0.01
  104. if timeout_val < 0:
  105. kill(proc2)
  106. print("Compilation timeout expired!")
  107. return False
  108. if proc2.returncode != 2:
  109. break
  110. # Make sure everything stopped correctly
  111. assert proc2.returncode == 0
  112. if proc2.returncode != 0:
  113. return False
  114. except:
  115. raise
  116. finally:
  117. try:
  118. kill(proc2)
  119. except UnboundLocalError:
  120. pass
  121. def compile_files(address, process, files, mode):
  122. """Compiles the given files in the given mode."""
  123. threads = []
  124. mod_files = []
  125. for filename in files:
  126. if os.path.isfile(filename):
  127. mod_filename = filename
  128. elif os.path.isfile("%s/%s" % (get_code_folder_name(), filename)):
  129. mod_filename = "%s/%s" % (get_code_folder_name(), filename)
  130. elif os.path.isfile("%s/%s" % (BOOTSTRAP_FOLDER_NAME, filename)):
  131. mod_filename = "%s/%s" % (BOOTSTRAP_FOLDER_NAME, filename)
  132. else:
  133. raise Exception("File not found: %s" % filename)
  134. mod_files.append(mod_filename)
  135. to_compile = to_recompile(address, mod_files)
  136. for mod_filename in to_compile:
  137. if mod_filename.endswith(".mvc"):
  138. model_mode = "MO"
  139. mod_files.remove(mod_filename)
  140. else:
  141. model_mode = mode
  142. if PARALLEL_PUSH:
  143. import threading
  144. threads.append(
  145. threading.Thread(
  146. target=compile_file,
  147. args=[address, mod_filename, mod_filename, model_mode, process]))
  148. threads[-1].start()
  149. else:
  150. compile_file(address, mod_filename, mod_filename, model_mode, process)
  151. if PARALLEL_PUSH:
  152. for t in threads:
  153. t.join()
  154. if mode[-1] == "O":
  155. # Fire up the linker
  156. val = execute("link_and_load", [address, USERNAME] + mod_files, wait=True)
  157. if val != 0:
  158. raise Exception("Linking error")
  159. def run_file(files, parameters, mode, handle_output, optimization_level=None):
  160. """Compiles the given sequence of files, feeds them the given input in the given mode,
  161. and handles their output."""
  162. # Resolve file
  163. import os.path
  164. time.sleep(0.01)
  165. port = get_free_port()
  166. address = "http://127.0.0.1:%i" % port
  167. try:
  168. # Run Modelverse server
  169. modelverse_args = [str(port)]
  170. if optimization_level is not None:
  171. modelverse_args.append('--kernel=%s' % optimization_level)
  172. proc = execute("run_local_modelverse", modelverse_args, wait=False)
  173. # Compile, push and link the source code files.
  174. compile_files(address, proc, files, mode)
  175. # Send the request ...
  176. set_input_data(address, parameters)
  177. # ... and wait for replies
  178. while 1:
  179. val = urllib2.urlopen(
  180. urllib2.Request(
  181. address,
  182. urllib.urlencode({"op": "get_output", "taskname": USERNAME})),
  183. timeout=240).read()
  184. val = json.loads(val)
  185. if proc.returncode is not None:
  186. # Modelverse has terminated. This may or may not be what we want.
  187. raise ModelverseTerminated()
  188. if not handle_output(val):
  189. return
  190. # All passed!
  191. return
  192. except:
  193. raise
  194. finally:
  195. try:
  196. kill(proc)
  197. except UnboundLocalError:
  198. pass
  199. def run_file_to_completion(files, parameters, mode):
  200. """Compiles the given sequence of files, feeds them the given input in the given mode,
  201. and then collects and returns output."""
  202. results = []
  203. def handle_output(output):
  204. """Appends the given output to the list of results."""
  205. results.append(output)
  206. return True
  207. try:
  208. run_file(files, parameters, mode, handle_output)
  209. except ModelverseTerminated:
  210. return results
  211. def run_file_fixed_output_count(files, parameters, mode, output_count, optimization_level=None):
  212. """Compiles the given sequence of files, feeds them the given input in the given mode,
  213. and then collects and returns a fixed number of outputs."""
  214. results = []
  215. def handle_output(output):
  216. """Appends the given output to the list of results."""
  217. results.append(output)
  218. if len(results) < output_count:
  219. return True
  220. else:
  221. return False
  222. run_file(files, parameters, mode, handle_output, optimization_level)
  223. return results
  224. def run_file_single_output(files, parameters, mode, optimization_level=None):
  225. """Compiles the given sequence of files, feeds them the given input in the given mode,
  226. and then collects and returns a single output."""
  227. return run_file_fixed_output_count(files, parameters, mode, 1, optimization_level)[0]
  228. def mean(values):
  229. """Computes the arithmetic mean of the given values."""
  230. return float(sum(values)) / max(len(values), 1)
  231. def run_perf_test(files, parameters, optimization_level, n_iterations=1):
  232. """Compiles the given sequence of files, feeds them the given input in the given mode,
  233. and then collects their output. This process is repeated n_iterations times. The
  234. return value is the average of all outputs, along with the mean total run-time."""
  235. test_runtimes = []
  236. total_runtimes = []
  237. for _ in xrange(n_iterations):
  238. start_time = time.time()
  239. test_time = run_file_single_output(
  240. files, parameters, 'CO',
  241. optimization_level)
  242. end_time = time.time()
  243. total_time = end_time - start_time
  244. test_runtimes.append(test_time)
  245. total_runtimes.append(total_time)
  246. return mean(test_runtimes), mean(total_runtimes)
  247. def get_expectation_checks(expected_values):
  248. """Converts the given sequence of expected values to a sequence of functions which tell
  249. if an input is allowed. Every function is accompanied by an expected value."""
  250. def get_single_expectation_checks(expectation):
  251. """Gets an expectation checker for a single expected value."""
  252. if isinstance(expectation, set):
  253. # We expect to receive a number of outputs equal to the size of the set, but their
  254. # order does not matter.
  255. for _ in xrange(len(expectation)):
  256. yield lambda val: val in expectation
  257. elif expectation is None:
  258. # Skip output value
  259. yield lambda _: True
  260. else:
  261. yield lambda val: val == expectation
  262. for expectation in expected_values:
  263. for checker in get_single_expectation_checks(expectation):
  264. yield checker, expectation
  265. def run_correctness_test(files, parameters, expected, optimization_level):
  266. """Compiles the given sequence of files, feeds them the given input in the given mode,
  267. and then compares the output with the expected output. The return value is the total
  268. run-time of the test."""
  269. checks = iter(list(get_expectation_checks(expected)))
  270. next_check = [next(checks)]
  271. def handle_output(output):
  272. """Checks the given output against the expected output."""
  273. check, expectation = next_check[0]
  274. print("Got %s, expect %s" % (output, expectation))
  275. assert check(output)
  276. try:
  277. next_check[0] = next(checks)
  278. return True
  279. except StopIteration:
  280. return False
  281. start_time = time.time()
  282. try:
  283. run_file(files, parameters, 'CO', handle_output, optimization_level)
  284. except ModelverseTerminated:
  285. return
  286. end_time = time.time()
  287. return end_time - start_time
  288. def format_output(output):
  289. """Formats the output of `run_file_to_completion` as a string."""
  290. return '\n'.join(output)
  291. def define_perf_test(target_class, test_function, optimization_level):
  292. """Defines a performance test in the given class. The performance test calls the given function
  293. at the given optimization level."""
  294. setattr(
  295. target_class,
  296. 'test_%s' % optimization_level.replace('-', '_').lower(),
  297. lambda self: test_function(self, optimization_level))
  298. def define_perf_tests(target_class, test_function):
  299. """Defines performance tests in the given class. Each test calls the given function."""
  300. for optimization_level in ALL_OPTIMIZATION_LEVELS:
  301. define_perf_test(target_class, test_function, optimization_level)
  302. def get_model_constructor(code):
  303. # First change multiple spaces to a tab
  304. code_fragments = code.split("\n")
  305. code_fragments = [i for i in code_fragments if i.strip() != ""]
  306. code_fragments = [i.replace(" ", "\t") for i in code_fragments]
  307. initial_tabs = min([len(i) - len(i.lstrip("\t")) for i in code_fragments])
  308. code_fragments = [i[initial_tabs:] for i in code_fragments]
  309. code = "\n".join(code_fragments)
  310. with open("__model.mvc", "w") as f:
  311. f.write(code)
  312. f.flush()
  313. constructors = do_compile("__model.mvc", "interface/HUTN/grammars/modelling.g", "M") + ["exit"]
  314. return constructors
  315. DEFAULT_PERF_FILE_NAME = 'perf_data.txt'
  316. TOTAL_TIME_QUANTITY = 'total-runtime'
  317. TEST_TIME_QUANTITY = 'test-runtime'
  318. def write_perf_entry_to_stream(
  319. test_name, optimization_level, quantity,
  320. result, output_stream):
  321. """Writes a performance measurement entry to the given stream."""
  322. output_stream.write('%s:%s:%s:%f\n' % (test_name, optimization_level, quantity, result))
  323. def write_perf_to_file(
  324. test_name, optimization_level, runtimes, file_name=DEFAULT_PERF_FILE_NAME):
  325. """Writes performance data to a file."""
  326. test_runtime, total_runtime = runtimes
  327. with open(file_name, "a") as perf_file:
  328. write_perf_entry_to_stream(
  329. test_name, optimization_level, TEST_TIME_QUANTITY, test_runtime, perf_file)
  330. write_perf_entry_to_stream(
  331. test_name, optimization_level, TOTAL_TIME_QUANTITY, total_runtime, perf_file)
  332. def write_total_runtime_to_file(
  333. test_name, optimization_level, total_runtime, file_name=DEFAULT_PERF_FILE_NAME):
  334. """Writes a total runtime entry to a file."""
  335. with open(file_name, "a") as perf_file:
  336. write_perf_entry_to_stream(
  337. test_name, optimization_level, TOTAL_TIME_QUANTITY, total_runtime, perf_file)
  338. def parse_perf_data(file_name):
  339. """Parses the performance data in the given file."""
  340. results = defaultdict(lambda: defaultdict(list))
  341. with open(file_name, 'r') as perf_file:
  342. for line in perf_file.readlines():
  343. test_name, optimization_level, quantity, result = line.strip().split(':')
  344. results[quantity][optimization_level].append((test_name, result))
  345. return sorted(results.items(), key=operator.itemgetter(1))