modelverse/interface/HUTN/hutn_compiler/compiler.py

import cPickle as pickle
import os
import sys

from grammar_compiler_visitor import GrammarCompilerVisitor
from hutnparser import Parser, Tree
from meta_grammar import Grammar
from cached_exception import CachedException
import hashlib

global parsers
parsers = {}

sys.setrecursionlimit(2000)

def read(filename):
  with open(filename, 'r') as f:
    return f.read()

def md5digest(filename):
    hasher = hashlib.md5()
    with open(filename, 'rb') as afile:
        hasher.update(afile.read())
    return hasher.hexdigest()

def fetch_cached(filename):
    try:
        md5 = md5digest(filename)
        cache_folder = os.path.abspath("%s/../caches/" % (os.path.dirname(os.path.abspath(__file__))))
        picklefile = cache_folder + "/%s.pickle" % md5
        if os.path.getmtime(picklefile) > os.path.getmtime(filename):
            with open(picklefile, "rb") as f:
                return pickle.load(f)
        else:
            return None
    except:
        return None

def make_cached(filename, data):
    md5 = md5digest(filename)
    cache_folder = os.path.abspath("%s/../caches/" % (os.path.dirname(os.path.abspath(__file__))))
    picklefile = cache_folder + "/%s.pickle" % md5
    with open(picklefile, "wb") as f:
        pickle.dump(data, f, pickle.HIGHEST_PROTOCOL)

def do_parse(inputfile, grammarfile):
    if grammarfile not in parsers:
        grammar = fetch_cached(grammarfile)
        if grammar is None:
            result = parser = Parser(Grammar(), hide_implicit = True).parse(read(grammarfile))
            if result['status'] != Parser.Constants.Success:
                print 'not a valid grammar!'
                print result

            tree = result['tree']
            visitor = GrammarCompilerVisitor()
            structure = visitor.visit(tree)
            grammar = Grammar()          
            grammar.rules = structure['rules']
            grammar.tokens = structure['tokens']        
            make_cached(grammarfile, grammar)
        parsers[grammarfile] = grammar
    else:
        grammar = parsers[grammarfile]

    result = fetch_cached(inputfile)
    if result is None:
        result = Parser(grammar, line_position = True).parse(read(inputfile))
        if result['status'] != Parser.Constants.Success:
            msg = "%s:%s:%s: %s" % (inputfile, result["line"], result["column"], result["text"])
            raise Exception(msg)
        make_cached(inputfile, result)

    return result

def find_file(filename, include_paths):
    import os.path
    include_paths = ["."] + \
                    [os.path.abspath(os.path.dirname(working_file))] + \
                    [os.path.abspath("%s/../includes/" % (os.path.dirname(os.path.abspath(__file__))))] + \
                    include_paths + \
                    []

    attempts = []
    for include in include_paths:
        testfile = include + os.sep + filename
        if os.path.isfile(os.path.abspath(testfile)):
            return os.path.abspath(testfile)
        else:
            attempts.append(os.path.abspath(testfile))
    else:
        raise Exception("Could not resolve file %s. Tried: %s" % (filename, attempts))

def do_compile(inputfile, grammarfile, visitors=[], include_paths = []):
    import os.path
    global working_file
    working_file = os.path.abspath(inputfile)

    result = do_parse(inputfile, grammarfile)
    error = result["status"] != Parser.Constants.Success
    if error:
        msg = "%s:%s:%s: %s" % (inputfile, result["line"], result["column"], result["text"])
        raise Exception(msg)
    else:
        for child in result["tree"].tail:
            child.inputfile = inputfile
        included = set()
        while True:
            for i, v in enumerate(result["tree"].tail):
                if v.head == "include":
                    # Expand this node
                    for j in v.tail:
                        if j.head == "STRVALUE":
                            f = str(j.tail[0])[1:-1]
                            if f in included:
                                subtree = []
                            else:
                                name = str(j.tail[0])[1:-1]
                                subtree = do_parse(find_file(name, include_paths), grammarfile)["tree"].tail
                                if subtree is None:
                                    raise Exception("Parsing error for included file %s" % find_file(name, include_paths))

                                for t in subtree:
                                    t.inputfile = name
                                included.add(f)
                            # Found the string value, so break from the inner for ("searching for element")
                            break

                    # Merge all nodes in
                    before = result["tree"].tail[:i]
                    after = result["tree"].tail[i+1:]
                    result["tree"].tail = before + subtree + after
                    # Found an include node, but to prevent corruption of the tree, we need to start over again, so break from the outer for loop
                    break
            else:
                # The outer for finally finished, so there were no includes remaining, thus terminate the infinite while loop
                break

    result["tree"].fix_tracability(inputfile)

    for visitor in visitors:
        visitor.visit(result["tree"])

    if visitors:
        return visitors[-1].dump()

def main(input_file, grammar_file, mode, args=[], symbols=None):
    from prettyprint_visitor import PrettyPrintVisitor
    from prettyprint_visitor import PrintVisitor
    from semantics_visitor import SemanticsVisitor
    from bootstrap_visitor import BootstrapVisitor
    from primitives_visitor import PrimitivesVisitor
    from primitives_object_visitor import PrimitivesObjectVisitor
    from constructors_visitor import ConstructorsVisitor
    from constructors_object_visitor import ConstructorsObjectVisitor
    from model_visitor import ModelVisitor
    from model_bootstrap_visitor import ModelBootstrapVisitor
    from model_object_visitor import ModelObjectVisitor

    modes = {
        "N" : [],
        "P" : [PrintVisitor],
        "PP" : [PrettyPrintVisitor],
        "S" : [SemanticsVisitor],
        "PS" : [SemanticsVisitor, PrimitivesVisitor],
        "PO" : [SemanticsVisitor, PrimitivesObjectVisitor],
        "BS" : [SemanticsVisitor, BootstrapVisitor],
        "CS" : [SemanticsVisitor, ConstructorsVisitor],
        "CO" : [SemanticsVisitor, ConstructorsObjectVisitor],
        "M" : [ModelVisitor],
        "MB" : [ModelBootstrapVisitor],
        "MO" : [ModelObjectVisitor],
    }
    try:
        visitors = [v(args) for v in modes[mode]]
        result = do_compile(input_file, grammar_file, visitors)
        if symbols is not None and mode == "BS":
            symbols.update(visitors[-1].object_symbols)
    except CachedException:
        return True
    return result

if __name__ == "__main__":
    if len(sys.argv) <= 2:
        print("Invocation: ")
        print("  %s input_file grammar_file mode [mode_params]*" % sys.argv[0])
        sys.exit(1)
    else:
        value = main(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4:])
        if value is not None:
            print(value)