Update compiler to use the tree for hashing, instead of the file contents

bootstrap/initial_code_task.alc

@@ -1,6 +1,6 @@
 include "bootstrap/primitives.alc"
 include "bootstrap/semi_primitives.alc"
-include "bootstrap/core_algorithm.alc"
+include "core_algorithm.alh"
 
 Void mutable function __main():
 	Element root

bootstrap/semi_primitives.alc

@@ -64,7 +64,6 @@ Element function set_add_node(a : Element, b : Element):
 	return a!
 
 Element function set_pop(a : Element):
-	log("Pop from set!")
 	if (integer_gt(set_len(a), 0)):
 		Element edge
 		Element result
@@ -253,15 +252,14 @@ Element function set_overlap(sa : Element, sb : Element):
 
 	if (set_len(sa) > set_len(sb)):
 		// Pick the smallest set to iterate over, so switch if sa is not the smallest
-		log("Switch order")
 		result = sa
 		sa = sb
 		sb = result
 
 	result = set_create()
 	sa = set_copy(sa)
-	log("SA: " + set_to_string(sa))
-	log("SB: " + set_to_string(sb))
+	//log("SA: " + set_to_string(sa))
+	//log("SB: " + set_to_string(sb))
 
 	// Iterate over each element of sa and only add it to the result if it is also in sb
 	while (set_len(sa) > 0):

interface/HUTN/hutn_compiler/compiler.py

@@ -1,6 +1,7 @@
 import cPickle as pickle
 import os
 import sys
+import time
 
 from grammar_compiler_visitor import GrammarCompilerVisitor
 from hutnparser import Parser, Tree
@@ -17,15 +18,14 @@ def read(filename):
   with open(filename, 'r') as f:
     return f.read()
 
-def md5digest(filename):
+def md5digest(data):
     hasher = hashlib.md5()
-    with open(filename, 'rb') as afile:
-        hasher.update(afile.read())
+    hasher.update(data)
     return hasher.hexdigest()
 
-def fetch_cached(filename, mode=None):
+def fetch_cached(data, mode=None):
     try:
-        md5 = md5digest(filename)
+        md5 = md5digest(data)
         cache_folder = os.path.abspath("%s/../caches/" % (os.path.dirname(os.path.abspath(__file__))))
         if mode is None:
             picklefile = cache_folder + "/%s.pickle" % md5
@@ -36,8 +36,8 @@ def fetch_cached(filename, mode=None):
     except:
         return None
 
-def make_cached(filename, data, mode=None):
-    md5 = md5digest(filename)
+def make_cached(original_data, data, mode=None):
+    md5 = md5digest(original_data)
     cache_folder = os.path.abspath("%s/../caches/" % (os.path.dirname(os.path.abspath(__file__))))
     if mode is None:
         picklefile = cache_folder + "/%s.pickle" % md5
@@ -48,7 +48,7 @@ def make_cached(filename, data, mode=None):
 
 def do_parse(inputfile, grammarfile):
     if grammarfile not in parsers:
-        grammar = fetch_cached(grammarfile)
+        grammar = fetch_cached(read(grammarfile))
         if grammar is None:
             result = parser = Parser(Grammar(), hide_implicit = True).parse(read(grammarfile))
             if result['status'] != Parser.Constants.Success:
@@ -61,12 +61,12 @@ def do_parse(inputfile, grammarfile):
             grammar = Grammar()          
             grammar.rules = structure['rules']
             grammar.tokens = structure['tokens']        
-            make_cached(grammarfile, grammar)
+            make_cached(read(grammarfile), grammar)
         parsers[grammarfile] = grammar
     else:
         grammar = parsers[grammarfile]
 
-    result = fetch_cached(inputfile)
+    result = fetch_cached(read(inputfile))
     if result is None:
         result = Parser(grammar, line_position = True).parse(read(inputfile))
         if result['status'] != Parser.Constants.Success:
@@ -78,7 +78,7 @@ def do_parse(inputfile, grammarfile):
             lines = "".join(lines)
             msg = "%s:%s:%s: %s\nContext:\n%s" % (inputfile, result["line"], result["column"], result["text"], lines)
             raise Exception(msg)
-        make_cached(inputfile, result)
+        make_cached(read(inputfile), result)
 
     return result
 
@@ -105,54 +105,55 @@ def do_compile(inputfile, grammarfile, visitors=[], include_paths = [], mode="")
     global working_file
     working_file = os.path.abspath(inputfile)
 
-    result = fetch_cached(inputfile, mode)
-    if result is None:
-        result = do_parse(inputfile, grammarfile)
-        error = result["status"] != Parser.Constants.Success
-        if error:
-            lines = open(working_file, 'r').readlines()
-            begin_line = max(result["line"] - 3, 0)
-            end_line = max(result["line"] + 3, len(lines))
-            lines = lines[begin_line:end_line]
-            lines = ["%s:   %s" % (begin_line + i + 1, line) for i, line in enumerate(lines)]
-            lines = "".join(lines)
-            msg = "%s:%s:%s: %s\nContext:\n%s" % (inputfile, result["line"], result["column"], result["text"], lines)
-            raise Exception(msg)
-        else:
-            for child in result["tree"].tail:
-                child.inputfile = inputfile
-            included = set()
-            while True:
-                for i, v in enumerate(result["tree"].tail):
-                    if v.head == "include":
-                        # Expand this node
-                        for j in v.tail:
-                            if j.head == "STRVALUE":
-                                f = str(j.tail[0])[1:-1]
-                                if f in included:
-                                    subtree = []
-                                else:
-                                    name = str(j.tail[0])[1:-1]
-                                    subtree = do_parse(find_file(name, include_paths), grammarfile)["tree"].tail
-                                    if subtree is None:
-                                        raise Exception("Parsing error for included file %s" % find_file(name, include_paths))
-
-                                    for t in subtree:
-                                        t.inputfile = name
-                                    included.add(f)
-                                # Found the string value, so break from the inner for ("searching for element")
-                                break
-
-                        # Merge all nodes in
-                        before = result["tree"].tail[:i]
-                        after = result["tree"].tail[i+1:]
-                        result["tree"].tail = before + subtree + after
-                        # Found an include node, but to prevent corruption of the tree, we need to start over again, so break from the outer for loop
-                        break
-                else:
-                    # The outer for finally finished, so there were no includes remaining, thus terminate the infinite while loop
+    result = do_parse(inputfile, grammarfile)
+
+    error = result["status"] != Parser.Constants.Success
+    if error:
+        lines = open(working_file, 'r').readlines()
+        begin_line = max(result["line"] - 3, 0)
+        end_line = max(result["line"] + 3, len(lines))
+        lines = lines[begin_line:end_line]
+        lines = ["%s:   %s" % (begin_line + i + 1, line) for i, line in enumerate(lines)]
+        lines = "".join(lines)
+        msg = "%s:%s:%s: %s\nContext:\n%s" % (inputfile, result["line"], result["column"], result["text"], lines)
+        raise Exception(msg)
+    else:
+        for child in result["tree"].tail:
+            child.inputfile = inputfile
+        included = set()
+        while True:
+            for i, v in enumerate(result["tree"].tail):
+                if v.head == "include":
+                    # Expand this node
+                    for j in v.tail:
+                        if j.head == "STRVALUE":
+                            f = str(j.tail[0])[1:-1]
+                            if f in included:
+                                subtree = []
+                            else:
+                                name = str(j.tail[0])[1:-1]
+                                subtree = do_parse(find_file(name, include_paths), grammarfile)["tree"].tail
+                                if subtree is None:
+                                    raise Exception("Parsing error for included file %s" % find_file(name, include_paths))
+
+                                for t in subtree:
+                                    t.inputfile = name
+                                included.add(f)
+                            # Found the string value, so break from the inner for ("searching for element")
+                            break
+
+                    # Merge all nodes in
+                    before = result["tree"].tail[:i]
+                    after = result["tree"].tail[i+1:]
+                    result["tree"].tail = before + subtree + after
+                    # Found an include node, but to prevent corruption of the tree, we need to start over again, so break from the outer for loop
                     break
-
+            else:
+                # The outer for finally finished, so there were no includes remaining, thus terminate the infinite while loop
+                break
+    tree_data = pickle.dumps(result["tree"], pickle.HIGHEST_PROTOCOL)
+    new_result = fetch_cached(tree_data, mode)
+    if new_result is None:
         result["tree"].fix_tracability(inputfile)
 
         for visitor in visitors:
@@ -160,7 +161,9 @@ def do_compile(inputfile, grammarfile, visitors=[], include_paths = [], mode="")
 
         if visitors:
             result = visitors[-1].dump()
-            make_cached(inputfile, result, mode)
+            make_cached(tree_data, result, mode)
+    else:
+        result = new_result
 
     if visitors:
         return result