modelverse/scripts/make_parallel.py

import sys

from check_objects import to_recompile
from compile import do_compile
from link_and_load import link_and_load
import random
import multiprocessing
import glob
import os

def do_compile_wrapper(args):
    address, taskname, filename = args
    do_compile(address, filename, str(random.random()), filename, "PO", ["--debug"])

def main(address, taskname, files):
    new_files = to_recompile(address, files)
    if os.name == "posix":
        # At least linux decently implements forking, so use as many cores as possible
        p = multiprocessing.Pool(multiprocessing.cpu_count() * 2)
        p.map(do_compile_wrapper, [[address, taskname, f] for f in new_files])
    else:
        # Other systems are less intelligent, so just do this sequentially
        map(do_compile_wrapper, [[address, taskname, f] for f in new_files])
    link_and_load(address, taskname, files)

if __name__ == "__main__":
    if len(sys.argv) < 4:
        sys.stderr.write("Expected different parameters!\n")
        sys.stderr.write("    %s address taskname [file]*\n" % sys.argv[0])
        sys.exit(1)

    address = sys.argv[1]
    taskname = sys.argv[2]
    files = [a.replace("\\", "/") for a in sum([glob.glob(f) for f in sys.argv[3:]], [])]

    main(address, taskname, files)