From ddc67531bd43e24a7b8951aa72ed7b789044d6d6 Mon Sep 17 00:00:00 2001 From: "Joshua E. Jodesty" Date: Wed, 13 Feb 2019 00:38:15 -0500 Subject: [PATCH] param sweep full spec working --- SimCAD/configuration/__init__.py | 27 +++++-- SimCAD/configuration/utils/__init__.py | 6 +- SimCAD/engine/__init__.py | 19 ++--- SimCAD/engine/simulation.py | 30 ++++--- SimCAD/utils/__init__.py | 27 +++++++ simulations/validation/config1.py | 106 +++++++++++++++++-------- 6 files changed, 150 insertions(+), 65 deletions(-) diff --git a/SimCAD/configuration/__init__.py b/SimCAD/configuration/__init__.py index ab9f5fd..248c9b0 100644 --- a/SimCAD/configuration/__init__.py +++ b/SimCAD/configuration/__init__.py @@ -7,6 +7,8 @@ from SimCAD.configuration.utils.parameterSweep import ParamSweep from SimCAD.utils import key_filter from SimCAD.configuration.utils.behaviorAggregation import dict_elemwise_sum +from SimCAD.configuration.utils import exo_update_per_ts + class Configuration(object): @@ -22,14 +24,25 @@ class Configuration(object): def append_configs(sim_config, genesis_states, seed, raw_exogenous_states, env_processes, mechanisms, _exo_update_per_ts=True): - param_sweep = ParamSweep( - sweep_list=sim_config['M'], - mechs=mechanisms, - raw_exogenous_states=raw_exogenous_states, - _exo_update_per_ts=_exo_update_per_ts - ) + if 'M' in sim_config.keys(): + + for mechanisms, exogenous_states in sim_config['M']: + configs.append( + Configuration( + sim_config=sim_config, + state_dict=genesis_states, + seed=seed, + exogenous_states=exogenous_states, + env_processes=env_processes, + mechanisms=mechanisms + ) + ) + else: + if _exo_update_per_ts is True: + exogenous_states = exo_update_per_ts(raw_exogenous_states) + else: + exogenous_states = raw_exogenous_states - for mechanisms, exogenous_states in zip(param_sweep.mechanisms(), param_sweep.exogenous_states()): configs.append( Configuration( sim_config=sim_config, diff --git a/SimCAD/configuration/utils/__init__.py b/SimCAD/configuration/utils/__init__.py index c119190..c21f57c 100644 --- a/SimCAD/configuration/utils/__init__.py +++ b/SimCAD/configuration/utils/__init__.py @@ -4,7 +4,7 @@ from copy import deepcopy from fn.func import curried import pandas as pd -from SimCAD.utils import dict_filter, contains_type, curry_pot +from SimCAD.utils import dict_filter, contains_type, flatten_tabulated_dict, tabulate_dict, curry_pot # import pprint # pp = pprint.PrettyPrinter(indent=4) @@ -120,3 +120,7 @@ def exo_update_per_ts(ep): return y, s[y] return {es: ep_decorator(f, es) for es, f in ep.items()} + + +def process_variables(d): + return flatten_tabulated_dict(tabulate_dict(d)) \ No newline at end of file diff --git a/SimCAD/engine/__init__.py b/SimCAD/engine/__init__.py index 66a0773..3a31be6 100644 --- a/SimCAD/engine/__init__.py +++ b/SimCAD/engine/__init__.py @@ -16,16 +16,16 @@ class ExecutionContext: self.name = context self.method = None - def single_proc_exec(simulation_execs, states_lists, configs_structs, env_processes_list, Ts, Ns): + def single_proc_exec(simulation_execs, var_dict, states_lists, configs_structs, env_processes_list, Ts, Ns): l = [simulation_execs, states_lists, configs_structs, env_processes_list, Ts, Ns] simulation, states_list, config, env_processes, T, N = list(map(lambda x: x.pop(), l)) - result = simulation(states_list, config, env_processes, T, N) + result = simulation(var_dict, states_list, config, env_processes, T, N) return flatten(result) - def parallelize_simulations(fs, states_list, configs, env_processes, Ts, Ns): - l = list(zip(fs, states_list, configs, env_processes, Ts, Ns)) + def parallelize_simulations(fs, var_dict_list, states_list, configs, env_processes, Ts, Ns): + l = list(zip(fs, var_dict_list, states_list, configs, env_processes, Ts, Ns)) with Pool(len(configs)) as p: - results = p.map(lambda t: t[0](t[1], t[2], t[3], t[4], t[5]), l) + results = p.map(lambda t: t[0](t[1], t[2], t[3], t[4], t[5], t[6]), l) return results if context == 'single_proc': @@ -47,10 +47,11 @@ class Executor: create_tensor_field = TensorFieldReport(config_proc).create_tensor_field print(self.exec_context+": "+str(self.configs)) - states_lists, Ts, Ns, eps, configs_structs, env_processes_list, mechanisms, simulation_execs = \ - [], [], [], [], [], [], [], [] + var_dict_list, states_lists, Ts, Ns, eps, configs_structs, env_processes_list, mechanisms, simulation_execs = \ + [], [], [], [], [], [], [], [], [] config_idx = 0 for x in self.configs: + var_dict_list.append(x.sim_config['M']) states_lists.append([x.state_dict]) Ts.append(x.sim_config['T']) Ns.append(x.sim_config['N']) @@ -64,11 +65,11 @@ class Executor: if self.exec_context == ExecutionMode.single_proc: tensor_field = create_tensor_field(mechanisms.pop(), eps.pop()) - result = self.exec_method(simulation_execs, states_lists, configs_structs, env_processes_list, Ts, Ns) + result = self.exec_method(simulation_execs, var_dict_list, states_lists, configs_structs, env_processes_list, Ts, Ns) return result, tensor_field elif self.exec_context == ExecutionMode.multi_proc: if len(self.configs) > 1: - simulations = self.exec_method(simulation_execs, states_lists, configs_structs, env_processes_list, Ts, Ns) + simulations = self.exec_method(simulation_execs, var_dict_list, states_lists, configs_structs, env_processes_list, Ts, Ns) results = [] for result, mechanism, ep in list(zip(simulations, mechanisms, eps)): results.append((flatten(result), create_tensor_field(mechanism, ep))) diff --git a/SimCAD/engine/simulation.py b/SimCAD/engine/simulation.py index 7c1f5bc..ed3dc06 100644 --- a/SimCAD/engine/simulation.py +++ b/SimCAD/engine/simulation.py @@ -4,6 +4,8 @@ from fn.op import foldr, call from SimCAD.utils import curry_pot from SimCAD.engine.utils import engine_exception +import pprint as pp + id_exception = engine_exception(KeyError, KeyError, None) @@ -13,14 +15,15 @@ class Executor: self.state_update_exception = state_update_exception self.behavior_update_exception = behavior_update_exception - def get_behavior_input(self, step, sL, s, funcs): + def get_behavior_input(self, var_dict, step, sL, s, funcs): ops = self.behavior_ops[::-1] - def get_col_results(step, sL, s, funcs): - return list(map(lambda f: curry_pot(f, step, sL, s), funcs)) + def get_col_results(var_dict, step, sL, s, funcs): + # return list(map(lambda f: curry_pot(f, step, sL, s), funcs)) + return list(map(lambda f: f(var_dict, step, sL, s), funcs)) # print(get_col_results(step, sL, s, funcs)) - return foldr(call, get_col_results(step, sL, s, funcs))(ops) + return foldr(call, get_col_results(var_dict, step, sL, s, funcs))(ops) def apply_env_proc(self, env_processes, state_dict, step): for state in state_dict.keys(): @@ -31,16 +34,17 @@ class Executor: else: state_dict[state] = env_state(state_dict[state]) - def mech_step(self, m_step, sL, state_funcs, behavior_funcs, env_processes, t_step, run): + def mech_step(self, var_dict, m_step, sL, state_funcs, behavior_funcs, env_processes, t_step, run): last_in_obj = sL[-1] - _input = self.behavior_update_exception(self.get_behavior_input(m_step, sL, last_in_obj, behavior_funcs)) + _input = self.behavior_update_exception(self.get_behavior_input(var_dict, m_step, sL, last_in_obj, behavior_funcs)) # print(_input) # ToDo: add env_proc generator to `last_in_copy` iterator as wrapper function last_in_copy = dict( [ - self.state_update_exception(curry_pot(f, m_step, sL, last_in_obj, _input)) for f in state_funcs + # self.state_update_exception(curry_pot(f, m_step, sL, last_in_obj, _input)) for f in state_funcs + self.state_update_exception(f(var_dict, m_step, sL, last_in_obj, _input)) for f in state_funcs ] ) @@ -58,7 +62,7 @@ class Executor: return sL - def mech_pipeline(self, states_list, configs, env_processes, t_step, run): + def mech_pipeline(self, var_dict, states_list, configs, env_processes, t_step, run): m_step = 0 states_list_copy = deepcopy(states_list) genesis_states = states_list_copy[-1] @@ -68,7 +72,7 @@ class Executor: m_step += 1 for config in configs: s_conf, b_conf = config[0], config[1] - states_list = self.mech_step(m_step, states_list, s_conf, b_conf, env_processes, t_step, run) + states_list = self.mech_step(var_dict, m_step, states_list, s_conf, b_conf, env_processes, t_step, run) m_step += 1 t_step += 1 @@ -76,11 +80,11 @@ class Executor: return states_list # ToDo: Rename Run Pipeline - def block_pipeline(self, states_list, configs, env_processes, time_seq, run): + def block_pipeline(self, var_dict, states_list, configs, env_processes, time_seq, run): time_seq = [x + 1 for x in time_seq] simulation_list = [states_list] for time_step in time_seq: - pipe_run = self.mech_pipeline(simulation_list[-1], configs, env_processes, time_step, run) + pipe_run = self.mech_pipeline(var_dict, simulation_list[-1], configs, env_processes, time_step, run) _, *pipe_run = pipe_run simulation_list.append(pipe_run) @@ -88,12 +92,12 @@ class Executor: # ToDo: Muiltithreaded Runs - def simulation(self, states_list, configs, env_processes, time_seq, runs): + def simulation(self, var_dict, states_list, configs, env_processes, time_seq, runs): pipe_run = [] for run in range(runs): run += 1 states_list_copy = deepcopy(states_list) - head, *tail = self.block_pipeline(states_list_copy, configs, env_processes, time_seq, run) + head, *tail = self.block_pipeline(var_dict, states_list_copy, configs, env_processes, time_seq, run) genesis = head.pop() genesis['mech_step'], genesis['time_step'], genesis['run'] = 0, 0, run first_timestep_per_run = [genesis] + tail.pop(0) diff --git a/SimCAD/utils/__init__.py b/SimCAD/utils/__init__.py index 0258e9e..495d09e 100644 --- a/SimCAD/utils/__init__.py +++ b/SimCAD/utils/__init__.py @@ -41,6 +41,33 @@ def dict_filter(dictionary, condition): return dict([(k, v) for k, v in dictionary.items() if condition(v)]) +def get_max_dict_val_len(g): + return len(max(g.values(), key=len)) + + +def tabulate_dict(d): + max_len = get_max_dict_val_len(d) + _d = {} + for k, vl in d.items(): + if len(vl) != max_len: + _d[k] = vl + list([vl[-1]] * (max_len-1)) + else: + _d[k] = vl + + return _d + + +def flatten_tabulated_dict(d): + max_len = get_max_dict_val_len(d) + dl = [{} for i in range(max_len)] + + for k, vl in d.items(): + for v, i in zip(vl, list(range(len(vl)))): + dl[i][k] = v + + return dl + + def contains_type(_collection, type): return any(isinstance(x, type) for x in _collection) diff --git a/simulations/validation/config1.py b/simulations/validation/config1.py index 9ad579c..b67d04e 100644 --- a/simulations/validation/config1.py +++ b/simulations/validation/config1.py @@ -3,8 +3,9 @@ import numpy as np from datetime import timedelta import pprint -from SimCAD.configuration import append_configs -from SimCAD.configuration.utils import proc_trigger, bound_norm_random, ep_time_step, exo_update_per_ts +from SimCAD import configs +from SimCAD.configuration import Configuration +from SimCAD.configuration.utils import proc_trigger, ep_time_step, process_variables, exo_update_per_ts pp = pprint.PrettyPrinter(indent=4) @@ -15,57 +16,66 @@ seed = { 'c': np.random.RandomState(3) } + +g = { + 'alpha': [1], + 'beta': [2, 5], + 'gamma': [3, 4], + 'omega': [7] +} + + # beta = 1 # middleware(f1,f2,f3,f4) # Behaviors per Mechanism -def b1m1(step, sL, s): +def b1m1(_g, step, sL, s): return {'param1': 1} -def b2m1(step, sL, s): +def b2m1(_g, step, sL, s): return {'param2': 4} -def b1m2(_beta, step, sL, s): - return {'param1': 'a', 'param2': _beta} +def b1m2(_g, step, sL, s): + return {'param1': 'a', 'param2': _g['beta']} -def b2m2(step, sL, s): +def b2m2(_g, step, sL, s): return {'param1': 'b', 'param2': 0} # @curried -def b1m3(step, sL, s): +def b1m3(_g, step, sL, s): return {'param1': np.array([10, 100])} # @curried -def b2m3(step, sL, s): +def b2m3(_g, step, sL, s): return {'param1': np.array([20, 200])} # Internal States per Mechanism # @curried -def s1m1(step, sL, s, _input): +def s1m1(_g, step, sL, s, _input): y = 's1' x = 0 return (y, x) -def s2m1(sweep_param, step, sL, s, _input): +def s2m1(_g, step, sL, s, _input): y = 's2' - x = sweep_param + x = _g['beta'] return (y, x) -def s1m2(step, sL, s, _input): +def s1m2(_g, step, sL, s, _input): y = 's1' x = _input['param2'] return (y, x) -def s2m2(step, sL, s, _input): +def s2m2(_g, step, sL, s, _input): y = 's2' x = _input['param2'] return (y, x) -def s1m3(step, sL, s, _input): +def s1m3(_g, step, sL, s, _input): y = 's1' x = 0 return (y, x) -def s2m3(step, sL, s, _input): +def s2m3(_g, step, sL, s, _input): y = 's2' x = 0 return (y, x) @@ -76,19 +86,19 @@ proc_one_coef_A = 0.7 proc_one_coef_B = 1.3 -def es3p1(param, step, sL, s, _input): +def es3p1(_g, step, sL, s, _input): y = 's3' - x = param + x = _g['gamma'] return (y, x) # @curried -def es4p2(param, step, sL, s, _input): +def es4p2(_g, step, sL, s, _input): y = 's4' - x = param + x = _g['gamma'] return (y, x) ts_format = '%Y-%m-%d %H:%M:%S' t_delta = timedelta(days=0, minutes=0, seconds=1) -def es5p2(step, sL, s, _input): +def es5p2(_g, step, sL, s, _input): y = 'timestamp' x = ep_time_step(s, dt_str=s['timestamp'], fromat_str=ts_format, _timedelta=t_delta) return (y, x) @@ -174,18 +184,44 @@ mechanisms = { } } -sim_config = { - "N": 2, - "T": range(5), - "M": [Decimal(1), Decimal(2), Decimal(3)] # dict read from dict -} -append_configs( - sim_config=sim_config, - genesis_states=genesis_states, - seed=seed, - raw_exogenous_states=raw_exogenous_states, - env_processes=env_processes, - mechanisms=mechanisms, - _exo_update_per_ts=True #Default -) \ No newline at end of file +# process_variables(g) +def gen_sim_configs(N, T, Ms): + return [ + { + "N": 2, + "T": range(5), + "M": M + } + for M in process_variables(Ms) + ] + + +sim_configs = gen_sim_configs( + N=2, + T=range(5), + Ms=g +) + + +for sim_config in sim_configs: + configs.append( + Configuration( + sim_config=sim_config, + state_dict=genesis_states, + seed=seed, + exogenous_states=raw_exogenous_states, # exo_update_per_ts + env_processes=env_processes, + mechanisms=mechanisms + ) + ) + +# append_configs( +# sim_config=sim_config, +# genesis_states=genesis_states, +# seed=seed, +# raw_exogenous_states=raw_exogenous_states, +# env_processes=env_processes, +# mechanisms=mechanisms, +# _exo_update_per_ts=True #Default +# ) \ No newline at end of file