jeffemmett
/
cadCAD
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

parallelized runs

This commit is contained in:
Joshua E. Jodesty 2019-03-01 17:30:37 -05:00
parent d56e843fcc
commit e37601ae22
2 changed files with 13 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cadCAD/engine/__init__.py
View File

@ -1,5 +1,5 @@
from typing import Callable, Dict, List, Any, Tuple from typing import Callable, Dict, List, Any, Tuple
from pathos.multiprocessing import ProcessingPool as Pool from pathos.multiprocessing import ProcessingPool as PPool
from pandas.core.frame import DataFrame from pandas.core.frame import DataFrame
from cadCAD.utils import flatten from cadCAD.utils import flatten
@ -44,7 +44,7 @@ def parallelize_simulations(
Ns: List[int] Ns: List[int]
): ):
l = list(zip(simulation_execs, var_dict_list, states_lists, configs_structs, env_processes_list, Ts, Ns)) l = list(zip(simulation_execs, var_dict_list, states_lists, configs_structs, env_processes_list, Ts, Ns))
with Pool(len(configs_structs)) as p: with PPool(len(configs_structs)) as p:
results = p.map(lambda t: t[0](t[1], t[2], t[3], t[4], t[5], t[6]), l) results = p.map(lambda t: t[0](t[1], t[2], t[3], t[4], t[5], t[6]), l)
return results return results

15
cadCAD/engine/simulation.py
View File

@ -1,8 +1,10 @@
from typing import Any, Callable, Dict, List, Tuple from typing import Any, Callable, Dict, List, Tuple
from pathos.pools import ThreadPool as TPool
from copy import deepcopy from copy import deepcopy
from fn.op import foldr, call from fn.op import foldr, call
from cadCAD.engine.utils import engine_exception from cadCAD.engine.utils import engine_exception
from cadCAD.utils import flatten
id_exception: Callable = engine_exception(KeyError, KeyError, None) id_exception: Callable = engine_exception(KeyError, KeyError, None)
@ -142,7 +144,6 @@ class Executor:
return simulation_list return simulation_list
# ToDo: Muiltithreaded Runs
def simulation( def simulation(
self, self,
var_dict: Dict[str, List[Any]], var_dict: Dict[str, List[Any]],
@ -153,8 +154,7 @@ class Executor:
runs: int runs: int
) -> List[List[Dict[str, Any]]]: ) -> List[List[Dict[str, Any]]]:
pipe_run: List[List[Dict[str, Any]]] = [] def execute_run(var_dict, states_list, configs, env_processes, time_seq, run) -> List[Dict[str, Any]]:
for run in range(runs):
run += 1 run += 1
states_list_copy: List[Dict[str, Any]] = deepcopy(states_list) states_list_copy: List[Dict[str, Any]] = deepcopy(states_list)
head, *tail = self.run_pipeline(var_dict, states_list_copy, configs, env_processes, time_seq, run) head, *tail = self.run_pipeline(var_dict, states_list_copy, configs, env_processes, time_seq, run)
@ -163,6 +163,13 @@ class Executor:
genesis: Dict[str, Any] = head.pop() genesis: Dict[str, Any] = head.pop()
genesis['substep'], genesis['timestep'], genesis['run'] = 0, 0, run genesis['substep'], genesis['timestep'], genesis['run'] = 0, 0, run
first_timestep_per_run: List[Dict[str, Any]] = [genesis] + tail.pop(0) first_timestep_per_run: List[Dict[str, Any]] = [genesis] + tail.pop(0)
pipe_run += [first_timestep_per_run] + tail return [first_timestep_per_run] + tail
pipe_run: List[List[Dict[str, Any]]] = flatten(
TPool().map(
lambda run: execute_run(var_dict, states_list, configs, env_processes, time_seq, run),
list(range(runs))
)
)
return pipe_run return pipe_run