jeffemmett
/
cadCAD
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

init

This commit is contained in:
Joshua E. Jodesty 2019-09-07 17:39:10 -04:00
parent 99495d08dc
commit 903069f23b
12 changed files with 122 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -3,6 +3,7 @@ jupyter notebook
.ipynb_checkpoints
.DS_Store
.idea
.pytest_cache
notebooks
*.egg-info
__pycache__

49
cadCAD/configuration/__init__.py
View File

@ -1,3 +1,4 @@
from copy import deepcopy
from typing import Dict, Callable, List, Tuple
from functools import reduce
import pandas as pd
@ -11,9 +12,9 @@ from cadCAD.configuration.utils.depreciationHandler import sanitize_partial_stat
class Configuration(object):
def __init__(self, sim_config={}, initial_state={}, seeds={}, env_processes={},
def __init__(self, user_id, sim_config={}, initial_state={}, seeds={}, env_processes={},
exogenous_states={}, partial_state_update_blocks={}, policy_ops=[lambda a, b: a + b],
**kwargs) -> None:
session_id=0, simulation_id=0, run_id=1, **kwargs) -> None:
# print(exogenous_states)
self.sim_config = sim_config
self.initial_state = initial_state
@ -24,11 +25,18 @@ class Configuration(object):
self.policy_ops = policy_ops
self.kwargs = kwargs
self.user_id = user_id
self.session_id = session_id
self.simulation_id = simulation_id
self.run_id = run_id
sanitize_config(self)
def append_configs(sim_configs={}, initial_state={}, seeds={}, raw_exogenous_states={}, env_processes={},
partial_state_update_blocks={}, policy_ops=[lambda a, b: a + b], _exo_update_per_ts: bool = True) -> None:
def append_configs(user_id='cadCAD_user', session_id=0, #ToDo: change to string
sim_configs={}, initial_state={}, seeds={}, raw_exogenous_states={}, env_processes={},
partial_state_update_blocks={}, policy_ops=[lambda a, b: a + b], _exo_update_per_ts: bool = True
) -> None:
if _exo_update_per_ts is True:
exogenous_states = exo_update_per_ts(raw_exogenous_states)
else:
@ -37,7 +45,27 @@ def append_configs(sim_configs={}, initial_state={}, seeds={}, raw_exogenous_sta
if isinstance(sim_configs, dict):
sim_configs = [sim_configs]
for sim_config in sim_configs:
new_sim_configs = []
for t in list(zip(sim_configs, list(range(len(sim_configs))))):
sim_config, simulation_id = t[0], t[1]
N = sim_config['N']
if N > 1:
for n in range(N):
sim_config['simulation_id'] = simulation_id
sim_config['run_id'] = n
sim_config['N'] = 1
new_sim_configs.append(deepcopy(sim_config))
del sim_config
else:
sim_config['simulation_id'] = simulation_id
sim_config['run_id'] = 0
new_sim_configs.append(deepcopy(sim_config))
print(new_sim_configs)
print()
# for sim_config in sim_configs:
for sim_config in new_sim_configs:
config = Configuration(
sim_config=sim_config,
initial_state=initial_state,
@ -45,10 +73,15 @@ def append_configs(sim_configs={}, initial_state={}, seeds={}, raw_exogenous_sta
exogenous_states=exogenous_states,
env_processes=env_processes,
partial_state_update_blocks=partial_state_update_blocks,
policy_ops=policy_ops
policy_ops=policy_ops,
user_id=user_id,
session_id=session_id,
simulation_id=sim_config['simulation_id'],
run_id=sim_config['run_id']
)
print(sim_configs)
#for each sim config create new config
# print(sim_configs)
# for each sim config create new config
configs.append(config)

52
cadCAD/engine/__init__.py
View File

@ -1,3 +1,4 @@
from pprint import pprint
from typing import Callable, Dict, List, Any, Tuple
from pathos.multiprocessing import ProcessingPool as PPool
from pandas.core.frame import DataFrame
@ -25,11 +26,18 @@ def single_proc_exec(
configs_structs: List[ConfigsType],
env_processes_list: List[EnvProcessesType],
Ts: List[range],
Ns: List[int]
Ns: List[int],
userIDs,
sessionIDs,
simulationIDs,
runIDs: List[int],
):
l = [simulation_execs, states_lists, configs_structs, env_processes_list, Ts, Ns]
simulation_exec, states_list, config, env_processes, T, N = list(map(lambda x: x.pop(), l))
result = simulation_exec(var_dict_list, states_list, config, env_processes, T, N)
params = [simulation_execs, states_lists, configs_structs, env_processes_list, Ts, Ns,
userIDs, sessionIDs, simulationIDs, runIDs]
simulation_exec, states_list, config, env_processes, T, N, user_id, session_id, simulation_id, run_id = \
list(map(lambda x: x.pop(), params))
result = simulation_exec(var_dict_list, states_list, config, env_processes, T, N,
user_id, session_id, simulation_id, run_id)
return flatten(result)
@ -40,11 +48,16 @@ def parallelize_simulations(
configs_structs: List[ConfigsType],
env_processes_list: List[EnvProcessesType],
Ts: List[range],
Ns: List[int]
Ns: List[int],
userIDs,
sessionIDs,
simulationIDs,
runIDs: List[int],
):
l = list(zip(simulation_execs, var_dict_list, states_lists, configs_structs, env_processes_list, Ts, Ns))
params = list(zip(simulation_execs, var_dict_list, states_lists, configs_structs, env_processes_list, Ts, Ns,
userIDs, sessionIDs, simulationIDs, runIDs))
with PPool(len(configs_structs)) as p:
results = p.map(lambda t: t[0](t[1], t[2], t[3], t[4], t[5], t[6]), l)
results = p.map(lambda t: t[0](t[1], t[2], t[3], t[4], t[5], t[6], t[7], t[8], t[9], t[10]), params)
return results
@ -70,7 +83,6 @@ class Executor:
config_proc = Processor()
create_tensor_field = TensorFieldReport(config_proc).create_tensor_field
print(r'''
__________ ____
________ __ _____/ ____/ | / __ \
@ -82,19 +94,27 @@ class Executor:
print(f'Execution Mode: {self.exec_context + ": " + str(self.configs)}')
print(f'Configurations: {self.configs}')
var_dict_list, states_lists, Ts, Ns, eps, configs_structs, env_processes_list, partial_state_updates, simulation_execs = \
[], [], [], [], [], [], [], [], []
userIDs, sessionIDs, simulationIDs, runIDs, \
var_dict_list, states_lists, \
Ts, Ns, \
eps, configs_structs, env_processes_list, \
partial_state_updates, simulation_execs = \
[], [], [], [], [], [], [], [], [], [], [], [], []
config_idx = 0
for x in self.configs:
userIDs.append(x.user_id)
sessionIDs.append(x.session_id)
simulationIDs.append(x.simulation_id)
runIDs.append(x.run_id)
Ts.append(x.sim_config['T'])
Ns.append(x.sim_config['N'])
var_dict_list.append(x.sim_config['M'])
states_lists.append([x.initial_state])
eps.append(list(x.exogenous_states.values()))
configs_structs.append(config_proc.generate_config(x.initial_state, x.partial_state_updates, eps[config_idx]))
# print(env_processes_list)
env_processes_list.append(x.env_processes)
partial_state_updates.append(x.partial_state_updates)
simulation_execs.append(SimExecutor(x.policy_ops).simulation)
@ -105,11 +125,17 @@ class Executor:
if self.exec_context == ExecutionMode.single_proc:
tensor_field = create_tensor_field(partial_state_updates.pop(), eps.pop())
result = self.exec_method(simulation_execs, var_dict_list, states_lists, configs_structs, env_processes_list, Ts, Ns)
result = self.exec_method(
simulation_execs, var_dict_list, states_lists, configs_structs, env_processes_list, Ts, Ns,
userIDs, sessionIDs, simulationIDs, runIDs
)
final_result = result, tensor_field
elif self.exec_context == ExecutionMode.multi_proc:
# if len(self.configs) > 1:
simulations = self.exec_method(simulation_execs, var_dict_list, states_lists, configs_structs, env_processes_list, Ts, Ns)
simulations = self.exec_method(
simulation_execs, var_dict_list, states_lists, configs_structs, env_processes_list, Ts, Ns,
userIDs, sessionIDs, simulationIDs, runIDs
)
results = []
for result, partial_state_updates, ep in list(zip(simulations, partial_state_updates, eps)):
results.append((flatten(result), create_tensor_field(partial_state_updates, ep)))

9
cadCAD/engine/simulation.py
View File

@ -202,7 +202,11 @@ class Executor:
configs: List[Tuple[List[Callable], List[Callable]]],
env_processes: Dict[str, Callable],
time_seq: range,
runs: int
runs: int,
user_id,
session_id,
simulation_id,
run_id
) -> List[List[Dict[str, Any]]]:
def execute_run(sweep_dict, states_list, configs, env_processes, time_seq, run) -> List[Dict[str, Any]]:
@ -211,6 +215,7 @@ class Executor:
def generate_init_sys_metrics(genesis_states_list):
for d in genesis_states_list:
d['run'], d['substep'], d['timestep'] = run, 0, 0
d['user_id'], d['simulation_id'], d['session_id'], d['run_id'] = user_id, simulation_id, session_id, run_id
yield d
states_list_copy: List[Dict[str, Any]] = list(generate_init_sys_metrics(deepcopy(states_list)))
@ -222,6 +227,8 @@ class Executor:
return first_timestep_per_run
# print(type(run_id))
# print(runs)
tp = TPool(runs)
pipe_run: List[List[Dict[str, Any]]] = flatten(
tp.map(

8
cadCAD/utils/__init__.py
View File

@ -17,6 +17,14 @@ def append_dict(dict, new_dict):
return dict
def arrange_cols(df, reverse):
session_metrics = ['user_id', 'session_id', 'simulation_id', 'run_id']
sys_metrics = ['run', 'timestep', 'substep']
result_cols = list(set(df.columns) - set(session_metrics) - set(sys_metrics))
result_cols.sort(reverse=reverse)
return df[session_metrics + sys_metrics + result_cols]
class IndexCounter:
def __init__(self):
self.i = 0

1
simulations/regression_tests/config1.py
View File

@ -152,6 +152,7 @@ sim_config = config_sim(
)
append_configs(
user_id='user_a',
sim_configs=sim_config,
initial_state=genesis_states,
env_processes=env_processes,

1
simulations/regression_tests/config2.py
View File

@ -140,6 +140,7 @@ sim_config = config_sim(
)
append_configs(
user_id='user_b',
sim_configs=sim_config,
initial_state=genesis_states,
env_processes=env_processes,

1
simulations/regression_tests/sweep_config.py
View File

@ -144,6 +144,7 @@ sim_config = config_sim(
# New Convention
partial_state_update_blocks = psub_list(psu_block, psu_steps)
append_configs(
user_id='user_a',
sim_configs=sim_config,
initial_state=genesis_states,
seeds=seeds,

6
simulations/test_executions/config1_test.py
View File

@ -3,6 +3,7 @@ from typing import List
from tabulate import tabulate
# The following imports NEED to be in the exact order
from cadCAD.engine import ExecutionMode, ExecutionContext, Executor
from cadCAD.utils import arrange_cols
from simulations.regression_tests import config1
from cadCAD import configs
@ -14,8 +15,11 @@ first_config = configs # only contains config1
single_proc_ctx = ExecutionContext(context=exec_mode.single_proc)
run = Executor(exec_context=single_proc_ctx, configs=first_config)
# print(set(result.columns) - set(['user_id', 'session_id', 'simulation_id', 'run_id']) - set(['run', 'timestep', 'substep']))
# print(['run', 'timestep', 'substep'])
raw_result, tensor_field = run.execute()
result = pd.DataFrame(raw_result)
result = arrange_cols(pd.DataFrame(raw_result), False)
print()
print("Tensor Field: config1")
# print(raw_result)

23
simulations/test_executions/config2_test.py
View File

@ -2,6 +2,7 @@ import pandas as pd
from tabulate import tabulate
# The following imports NEED to be in the exact order
from cadCAD.engine import ExecutionMode, ExecutionContext, Executor
from cadCAD.utils import arrange_cols
from simulations.regression_tests import config2
from cadCAD import configs
@ -9,15 +10,15 @@ exec_mode = ExecutionMode()
print("Simulation Execution: Single Configuration")
print()
first_config = configs # only contains config2
single_proc_ctx = ExecutionContext(context=exec_mode.single_proc)
run = Executor(exec_context=single_proc_ctx, configs=first_config)
single_proc_ctx = ExecutionContext(context=exec_mode.multi_proc)
run = Executor(exec_context=single_proc_ctx, configs=configs)
raw_result, tensor_field = run.execute()
result = pd.DataFrame(raw_result)
print()
print("Tensor Field: config1")
print(tabulate(tensor_field, headers='keys', tablefmt='psql'))
print("Output:")
print(tabulate(result, headers='keys', tablefmt='psql'))
print()
for raw_result, tensor_field in run.execute():
result = arrange_cols(pd.DataFrame(raw_result), False)
print()
# print("Tensor Field: " + config_names[i])
print(tabulate(tensor_field, headers='keys', tablefmt='psql'))
print("Output:")
print(tabulate(result, headers='keys', tablefmt='psql'))
print()
# i += 1

5
simulations/test_executions/multi_config_test.py
View File

@ -2,6 +2,7 @@ import pandas as pd
from tabulate import tabulate
# The following imports NEED to be in the exact order
from cadCAD.engine import ExecutionMode, ExecutionContext, Executor
from cadCAD.utils import arrange_cols
from simulations.regression_tests import config1, config2
from cadCAD import configs
@ -15,9 +16,9 @@ run = Executor(exec_context=multi_proc_ctx, configs=configs)
i = 0
config_names = ['config1', 'config2']
for raw_result, tensor_field in run.execute():
result = pd.DataFrame(raw_result)
result = arrange_cols(pd.DataFrame(raw_result), False)
print()
print(f"Tensor Field: {config_names[i]}")
# print(f"Tensor Field: {config_names[i]}")
print(tabulate(tensor_field, headers='keys', tablefmt='psql'))
print("Output:")
print(tabulate(result, headers='keys', tablefmt='psql'))

4
simulations/test_executions/param_sweep_test.py
View File

@ -14,11 +14,11 @@ multi_proc_ctx = ExecutionContext(context=exec_mode.multi_proc)
run = Executor(exec_context=multi_proc_ctx, configs=configs)
i = 0
config_names = ['sweep_config_A', 'sweep_config_B']
# config_names = ['sweep_config_A', 'sweep_config_B']
for raw_result, tensor_field in run.execute():
result = pd.DataFrame(raw_result)
print()
print("Tensor Field: " + config_names[i])
# print("Tensor Field: " + config_names[i])
print(tabulate(tensor_field, headers='keys', tablefmt='psql'))
print("Output:")
print(tabulate(result, headers='keys', tablefmt='psql'))