import cPickle as pickle import os import sys from grammar_compiler_visitor import GrammarCompilerVisitor from hutnparser import Parser, Tree from meta_grammar import Grammar global parsers parsers = {} def read(filename): with open(filename, 'r') as f: return f.read() def do_parse(inputfile, grammarfile): new_grammar = True picklefile = grammarfile + ".pickle" if grammarfile not in parsers: try: if os.path.getmtime(picklefile) > os.path.getmtime(grammarfile): # Pickle is more recent than grammarfile, so use it grammar = pickle.load(open(picklefile, 'rb')) new_grammar = False else: # Will be catched immediately raise Exception("Pickle is invalid!") except: result = parser = Parser(Grammar(), hide_implicit = True).parse(read(grammarfile)) if result['status'] != Parser.Constants.Success: print 'not a valid grammar!' print result tree = result['tree'] visitor = GrammarCompilerVisitor() structure = visitor.visit(tree) grammar = Grammar() grammar.rules = structure['rules'] grammar.tokens = structure['tokens'] pickle.dump(grammar, open(picklefile, 'wb'), pickle.HIGHEST_PROTOCOL) parsers[grammarfile] = grammar else: new_grammar = False grammar = parsers[grammarfile] picklefile = inputfile + ".pickle" try: if new_grammar: # Stop anyway, as the grammar is new raise Exception() if os.path.getmtime(picklefile) > os.path.getmtime(inputfile): # Pickle is more recent than inputfile, so use it result = pickle.load(open(picklefile, 'rb')) else: # Inputfile has changed raise Exception() except: result = Parser(grammar, line_position = True).parse(read(inputfile)) if result['status'] != Parser.Constants.Success: print('not a valid input file: %s' % inputfile) print(result) pickle.dump(result, open(picklefile, 'wb'), pickle.HIGHEST_PROTOCOL) return result def find_file(filename, include_paths): import os.path include_paths = ["."] + \ [os.path.abspath(os.path.dirname(working_file))] + \ [os.path.abspath("%s/../includes/" % (os.path.dirname(os.path.abspath(__file__))))] + \ include_paths + \ [] attempts = [] for include in include_paths: testfile = include + os.sep + filename if os.path.isfile(os.path.abspath(testfile)): return os.path.abspath(testfile) else: attempts.append(os.path.abspath(testfile)) else: raise Exception("Could not resolve file %s. Tried: %s" % (filename, attempts)) def do_compile(inputfile, grammarfile, compile_visitor = None, compile_visitor2 = None, include_paths = []): import os.path global working_file working_file = os.path.abspath(inputfile) result = do_parse(inputfile, grammarfile) error = result["status"] != Parser.Constants.Success if error: print 'not a valid input file!' print '{}:{}: error: {}'.format(result['line'], result['column'], result['text']) for r in result['partialresults']: print r if isinstance(r['tree'], Tree): print r['tree'].head, [x.head for x in r['tree'].tail if hasattr(x, 'head')] # print "-"*20+"partial-result"+"-"*20 # from prettyprint_visitor import PrintVisitor # pv = PrintVisitor() # pv.visit(r['tree']) # print pv.dump() else: for child in result["tree"].tail: child.inputfile = inputfile included = set() while True: for i, v in enumerate(result["tree"].tail): if v.head == "include": # Expand this node for j in v.tail: if j.head == "STRVALUE": f = str(j.tail[0])[1:-1] if f in included: subtree = [] else: # print find_file(str(j.tail[0])[1:-1], include_paths) # print do_parse(find_file(str(j.tail[0])[1:-1], include_paths), grammarfile) name = str(j.tail[0])[1:-1] subtree = do_parse(find_file(name, include_paths), grammarfile)["tree"].tail if subtree is None: if compile_visitor is None: return False else: return None for t in subtree: t.inputfile = name included.add(f) # Found the string value, so break from the inner for ("searching for element") break # Merge all nodes in before = result["tree"].tail[:i] after = result["tree"].tail[i+1:] result["tree"].tail = before + subtree + after # Found an include node, but to prevent corruption of the tree, we need to start over again, so break from the outer for loop break else: # The outer for finally finished, so there were no includes remaining, thus terminate the infinite while loop break if compile_visitor is None: return not error else: if error: return None else: try: compile_visitor.visit(result["tree"]) if compile_visitor2: compile_visitor2.visit(result["tree"]) except RuntimeError as e: print e.message sys.exit(1) if compile_visitor2: return compile_visitor2.dump() else: return compile_visitor.dump() def main(input_file, grammar_file, mode, args): if mode == "None": visitor = None visitor2 = None elif mode == "PP": from prettyprint_visitor import PrettyPrintVisitor visitor = PrettyPrintVisitor() visitor2 = None elif mode == "P": from prettyprint_visitor import PrintVisitor visitor = PrintVisitor() visitor2 = None elif mode == "S": from semantics_visitor import SemanticsVisitor visitor = SemanticsVisitor() visitor2 = None elif mode == "SP": from semantics_visitor import SemanticsVisitor from prettyprint_visitor import PrettyPrintVisitor visitor = SemanticsVisitor() visitor2 = PrettyPrintVisitor() elif mode == "BS": from semantics_visitor import SemanticsVisitor from bootstrap_visitor import BootstrapVisitor visitor = SemanticsVisitor() visitor2 = BootstrapVisitor(args) elif mode == "PS": from semantics_visitor import SemanticsVisitor from primitives_visitor import PrimitivesVisitor visitor = SemanticsVisitor() visitor2 = PrimitivesVisitor(args) elif mode == "PO": from semantics_visitor import SemanticsVisitor from primitives_object_visitor import PrimitivesObjectVisitor visitor = SemanticsVisitor() visitor2 = PrimitivesObjectVisitor(args) elif mode == "CS": from semantics_visitor import SemanticsVisitor from constructors_visitor import ConstructorsVisitor visitor = SemanticsVisitor() visitor2 = ConstructorsVisitor(args) elif mode == "CO": from semantics_visitor import SemanticsVisitor from constructors_object_visitor import ConstructorsObjectVisitor visitor = SemanticsVisitor() visitor2 = ConstructorsObjectVisitor(args) else: print("Visitor not understood: " + str(mode)) sys.exit(1) return do_compile(input_file, grammar_file, visitor, visitor2) if __name__ == "__main__": if len(sys.argv) <= 2: print("Invocation: ") print(" %s input_file grammar_file [mode]" % sys.argv[0]) sys.exit(1) else: value = main(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4:]) if value is None: sys.exit(1) else: if isinstance(value, list): print '\n'.join([repr(x) for x in value]) sys.exit(0)