|
@@ -6,6 +6,7 @@ from grammar_compiler_visitor import GrammarCompilerVisitor
|
|
|
from hutnparser import Parser, Tree
|
|
|
from meta_grammar import Grammar
|
|
|
from cached_exception import CachedException
|
|
|
+import hashlib
|
|
|
|
|
|
global parsers
|
|
|
parsers = {}
|
|
@@ -16,20 +17,36 @@ def read(filename):
|
|
|
with open(filename, 'r') as f:
|
|
|
return f.read()
|
|
|
|
|
|
+def md5digest(filename):
|
|
|
+ hasher = hashlib.md5()
|
|
|
+ with open(filename, 'rb') as afile:
|
|
|
+ hasher.update(afile.read())
|
|
|
+ return hasher.hexdigest()
|
|
|
+
|
|
|
+def fetch_cached(filename):
|
|
|
+ try:
|
|
|
+ md5 = md5digest(filename)
|
|
|
+ cache_folder = os.path.abspath("%s/../caches/" % (os.path.dirname(os.path.abspath(__file__))))
|
|
|
+ picklefile = cache_folder + "/%s.pickle" % md5
|
|
|
+ if os.path.getmtime(picklefile) > os.path.getmtime(filename):
|
|
|
+ with open(picklefile, "rb") as f:
|
|
|
+ return pickle.load(f)
|
|
|
+ else:
|
|
|
+ return None
|
|
|
+ except:
|
|
|
+ return None
|
|
|
+
|
|
|
+def make_cached(filename, data):
|
|
|
+ md5 = md5digest(filename)
|
|
|
+ cache_folder = os.path.abspath("%s/../caches/" % (os.path.dirname(os.path.abspath(__file__))))
|
|
|
+ picklefile = cache_folder + "/%s.pickle" % md5
|
|
|
+ with open(picklefile, "wb") as f:
|
|
|
+ pickle.dump(data, f, pickle.HIGHEST_PROTOCOL)
|
|
|
+
|
|
|
def do_parse(inputfile, grammarfile):
|
|
|
- new_grammar = True
|
|
|
- picklefile = grammarfile + ".pickle"
|
|
|
if grammarfile not in parsers:
|
|
|
- try:
|
|
|
- if os.path.getmtime(picklefile) > os.path.getmtime(grammarfile):
|
|
|
- # Pickle is more recent than grammarfile, so use it
|
|
|
- with open(picklefile, 'rb') as f:
|
|
|
- grammar = pickle.load(f)
|
|
|
- new_grammar = False
|
|
|
- else:
|
|
|
- # Will be catched immediately
|
|
|
- raise Exception()
|
|
|
- except:
|
|
|
+ grammar = fetch_cached(grammarfile)
|
|
|
+ if grammar is None:
|
|
|
result = parser = Parser(Grammar(), hide_implicit = True).parse(read(grammarfile))
|
|
|
if result['status'] != Parser.Constants.Success:
|
|
|
print 'not a valid grammar!'
|
|
@@ -41,29 +58,18 @@ def do_parse(inputfile, grammarfile):
|
|
|
grammar = Grammar()
|
|
|
grammar.rules = structure['rules']
|
|
|
grammar.tokens = structure['tokens']
|
|
|
- with open(picklefile, 'wb') as f:
|
|
|
- pickle.dump(grammar, f, pickle.HIGHEST_PROTOCOL)
|
|
|
+ make_cached(grammarfile, grammar)
|
|
|
parsers[grammarfile] = grammar
|
|
|
else:
|
|
|
- new_grammar = False
|
|
|
grammar = parsers[grammarfile]
|
|
|
|
|
|
- picklefile = inputfile + ".pickle"
|
|
|
- try:
|
|
|
- if os.path.getmtime(picklefile) > os.path.getmtime(inputfile):
|
|
|
- # Pickle is more recent than inputfile, so use it
|
|
|
- with open(picklefile, 'rb') as f:
|
|
|
- result = pickle.load(f)
|
|
|
- else:
|
|
|
- # Inputfile has changed
|
|
|
- raise Exception()
|
|
|
- except:
|
|
|
+ result = fetch_cached(inputfile)
|
|
|
+ if result is None:
|
|
|
result = Parser(grammar, line_position = True).parse(read(inputfile))
|
|
|
if result['status'] != Parser.Constants.Success:
|
|
|
msg = "%s:%s:%s: %s" % (inputfile, result["line"], result["column"], result["text"])
|
|
|
raise Exception(msg)
|
|
|
- with open(picklefile, 'wb') as f:
|
|
|
- pickle.dump(result, f, pickle.HIGHEST_PROTOCOL)
|
|
|
+ make_cached(inputfile, result)
|
|
|
|
|
|
return result
|
|
|
|