Added much faster way of finding cached files

bootstrap/bootstrap.py

@@ -271,7 +271,6 @@ def bootstrap():
                     f.write("Edge _(elem, hash_value)\n", both=False)
                     f.write("Edge _(_, hash)\n", both=False)
 
-                    #TODO add the symbols as well!
                     f.write('Node symbols("symbols")\n', both=False)
                     f.write('Node __symbols()\n', both=False)
                     f.write('Edge _(elem, __symbols)\n', both=False)

interface/HUTN/hutn_compiler/constructors_object_visitor.py

@@ -24,34 +24,15 @@ class ConstructorsObjectVisitor(ConstructorsVisitor):
             md5.update(f.read())
         self.hash_file = md5.hexdigest()
 
-        # Check if file is already compiled (with same hash) in Modelverse
+        # Create user
         urllib2.urlopen(urllib2.Request(self.address, urllib.urlencode({"op": "set_input", "value": '"%s"' % self.username, "username": "user_manager"}))).read()
 
-        def flush_data(data):
-            if data:
-                urllib2.urlopen(urllib2.Request(self.address, urllib.urlencode({"op": "set_input", "data": json.dumps(data), "username": self.username})), timeout=timeout).read()
-            return []
-
-        flush_data([3, "is_defined", self.obj_file])
-
-        v = urllib2.urlopen(urllib2.Request(self.address, urllib.urlencode({"op": "get_output", "username": self.username}))).read()
-        v = json.loads(v)
         simple_filename = self.real_file.rsplit("/")[-1]
-        if v == "None":
-            # Not defined, so recompile
-            print("[COMPILE] %s" % simple_filename)
-        else:
-            # Is defined already, so let's compare hashes
-            if v != self.hash_file:
-                print("[COMPILE] %s" % simple_filename)
-            else:
-                print("[CACHED] %s" % simple_filename)
-                raise CachedException()
+        print("[COMPILE] %s" % simple_filename)
 
     def dump(self):
         v = ConstructorsVisitor.dump(self)
 
-
         def flush_data(data):
             if data:
                 urllib2.urlopen(urllib2.Request(self.address, urllib.urlencode({"op": "set_input", "data": json.dumps(data), "username": self.username})), timeout=timeout).read()

interface/HUTN/hutn_compiler/primitives_object_visitor.py

@@ -17,35 +17,17 @@ class PrimitivesObjectVisitor(PrimitivesVisitor):
 
         self.object_symbols = {}
 
-        def flush_data(data):
-            if data:
-                urllib2.urlopen(urllib2.Request(self.address, urllib.urlencode({"op": "set_input", "data": json.dumps(data), "username": self.username}))).read()
-            return []
-
         with open(self.real_file, 'r') as f:
             import hashlib
             md5 = hashlib.md5()
             md5.update(f.read())
         self.hash_file = md5.hexdigest()
 
-        # Check if file is already compiled (with same hash) in Modelverse
-        urllib2.urlopen(urllib2.Request(self.address, urllib.urlencode({"op": "set_input", "element_type": "V", "value": '"%s"' % self.username, "username": "user_manager"}))).read()
-
-        flush_data([3, "is_defined", self.obj_file])
+        # Create user
+        urllib2.urlopen(urllib2.Request(self.address, urllib.urlencode({"op": "set_input", "value": '"%s"' % self.username, "username": "user_manager"}))).read()
 
-        v = urllib2.urlopen(urllib2.Request(self.address, urllib.urlencode({"op": "get_output", "username": self.username}))).read()
-        v = json.loads(v)
         simple_filename = self.real_file.rsplit("/")[-1]
-        if v == None:
-            # Not defined, so recompile
-            print("[COMPILE] %s" % simple_filename)
-        else:
-            # Is defined already, so let's compare hashes
-            if v != self.hash_file:
-                print("[COMPILE] %s" % simple_filename)
-            else:
-                print("[CACHED] %s" % simple_filename)
-                raise CachedException()
+        print("[COMPILE] %s" % simple_filename)
 
     def dump(self):
         v = PrimitivesVisitor.dump(self)

scripts/check_objects.py

@@ -0,0 +1,42 @@
+import urllib
+import urllib2
+import json
+import random
+import hashlib
+
+def to_recompile(address, files):
+    username = str(random.random())
+    files = sorted(files)
+    rebuild = []
+
+    def flush_data(data):
+        urllib2.urlopen(urllib2.Request(address, urllib.urlencode({"op": "set_input", "data": json.dumps(data), "username": username}))).read()
+
+    # Create new user
+    urllib2.urlopen(urllib2.Request(address, urllib.urlencode({"op": "set_input", "value": '"%s"' % username, "username": "user_manager"}))).read()
+
+    data = []
+    for f in files:
+        data.extend([3, "is_defined", f])
+    flush_data(data)
+
+    md5_values = {}
+    for f in files:
+        md5 = hashlib.md5()
+        md5.update(open(f, 'r').read())
+        md5_values[f] = md5.hexdigest()
+
+    for f in files:
+        v = urllib2.urlopen(urllib2.Request(address, urllib.urlencode({"op": "get_output", "username": username}))).read()
+        mv_md5 = json.loads(v)
+        if md5_values[f] == mv_md5:
+            # Identical, so don't rebuild
+            pass
+            print("[CACHE] %s" % f)
+        else:
+            # Different, so rebuild
+            rebuild.append(f)
+
+    # Remove user again
+    urllib2.urlopen(urllib2.Request(address, urllib.urlencode({"op": "set_input", "value": '"__%s"' % username, "username": "user_manager"}))).read()
+    return rebuild

scripts/make_all.py

@@ -1,5 +1,6 @@
 import sys
 
+from check_objects import to_recompile
 from compile import do_compile
 from link_and_load import link_and_load
 import random
@@ -14,7 +15,8 @@ if __name__ == "__main__":
         username = sys.argv[2]
         files = [a.replace("\\", "/") for a in sum([glob.glob(f) for f in sys.argv[3:]], [])]
 
-        for f in files:
+        new_files = to_recompile(address, files)
+        for f in new_files:
             do_compile(address, f, str(random.random()), f, "CO")
 
         link_and_load(address, username, files)

scripts/make_parallel.py

@@ -1,5 +1,6 @@
 import sys
 
+from check_objects import to_recompile
 from compile import do_compile
 from link_and_load import link_and_load
 import random
@@ -20,6 +21,8 @@ def do_compile_wrapper(filename):
 
 if __name__ == "__main__":
     p = multiprocessing.Pool(multiprocessing.cpu_count() * 2)
-    p.map(do_compile_wrapper, files)
+
+    new_files = to_recompile(address, files)
+    p.map(do_compile_wrapper, new_files)
 
     link_and_load(address, username, files)