pyalma/al2/experiment.py

import pathlib as pl
import json
import importlib.machinery as impmach
import multiprocessing
import threading 
import concurrent.futures as concfut
import os

from . import batch
from . import plan

def execute(exp_file):
    dispatcher = load(exp_file)
    dispatcher.start()
    dispatcher.join()

def load(exp_file):
    exp_plan = plan.Plan(exp_file, multiprocessing.Lock())

    with open(exp_file) as efile:
        exp_obj = json.loads(efile.read())
        exp_obj["load"] = pl.Path(exp_obj["load"])

        exp_mod = impmach.SourceFileLoader(exp_obj["load"].stem,
                                           str(exp_obj["load"])).load_module()

    return Dispatcher(exp_mod.run, exp_plan, os.cpu_count())

class Dispatcher (threading.Thread):
    def __init__(self, exp_func, exp_plan, num_workers):
        threading.Thread.__init__(self)
        
        self.__exp_func = exp_func
        self.__plan = exp_plan

        self.__num_workers = num_workers
        self.__workers = []

        for i in range(self.__num_workers):
            self.__workers.append(multiprocessing.Process(target=self.__run_exp,
                                                          args=(self.__exp_func,
                                                                self.__plan)))

    def run(self):
        for worker in self.__workers:
            worker.start()
        
        for worker in self.__workers:
            worker.join()

    @staticmethod
    def __run_exp(exp_func, exp_plan):
        instance = exp_plan.next()

        while instance !=  None:
            exp_func(instance)

            exp_plan.done_with(instance)

            instance = exp_plan.next()