From 2310d5042c2e4c907882f72560d2953450e20a7c Mon Sep 17 00:00:00 2001 From: "Joshua E. Jodesty" Date: Thu, 14 Feb 2019 16:06:31 -0500 Subject: [PATCH] Revert "Renaming PR hack" --- .gitignore | 4 +- README.md | 5 +- SimCAD/__init__.py | 2 +- SimCAD/configuration/__init__.py | 94 +-- SimCAD/configuration/utils/__init__.py | 95 +-- ...yAggregation.py => behaviorAggregation.py} | 15 +- SimCAD/configuration/utils/parameterSweep.py | 20 - SimCAD/engine/__init__.py | 34 +- SimCAD/engine/simulation.py | 64 +- SimCAD/engine/utils.py | 9 - SimCAD/utils/__init__.py | 117 +--- Simulation.md | 4 +- dist/SimCAD-0.1-py3-none-any.whl | Bin 7445 -> 11629 bytes notebooks/jjodesty/multithreading.ipynb | 576 ------------------ requirements.txt | 2 - simulations/example_run.py | 32 +- simulations/validation/base_config1.py | 171 ++++++ simulations/validation/base_config2.py | 180 ++++++ simulations/validation/config1.py | 98 +-- simulations/validation/config2.py | 99 +-- simulations/validation/config_1.py | 178 ++++++ simulations/validation/config_2.py | 180 ++++++ simulations/validation/sweep_config.py | 196 ------ 23 files changed, 930 insertions(+), 1245 deletions(-) rename SimCAD/configuration/utils/{policyAggregation.py => behaviorAggregation.py} (77%) delete mode 100644 SimCAD/configuration/utils/parameterSweep.py delete mode 100644 notebooks/jjodesty/multithreading.ipynb create mode 100644 simulations/validation/base_config1.py create mode 100644 simulations/validation/base_config2.py create mode 100644 simulations/validation/config_1.py create mode 100644 simulations/validation/config_2.py delete mode 100644 simulations/validation/sweep_config.py diff --git a/.gitignore b/.gitignore index b70d197..93f9edb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,6 @@ .idea .ipynb_checkpoints .DS_Store -.idea -notebooks -SimCAD.egg-info __pycache__ Pipfile Pipfile.lock @@ -12,6 +9,7 @@ results *.csv *.txt simulations/.ipynb_checkpoints +dist/SimCAD-0.1.tar.gz build SimCAD.egg-info \ No newline at end of file diff --git a/README.md b/README.md index 825c846..db87c42 100644 --- a/README.md +++ b/README.md @@ -15,13 +15,10 @@ Simulations may be run with a range of initial conditions and parameters for sta and environmental processes to understand and visualize network behavior under various conditions. Support for \ A/B testing policies, monte carlo analysis and other common numerical methods is provided. -SimCAD is written in Python 3. - **1. Install Dependencies:** ```bash -pip3 install -r requirements.txt +pip install -r requirements.txt python3 setup.py sdist bdist_wheel -pip3 install dist/SimCAD-0.1-py3-none-any.whl ``` **2. Configure Simulation:** diff --git a/SimCAD/__init__.py b/SimCAD/__init__.py index b4234cb..0b7fa28 100644 --- a/SimCAD/__init__.py +++ b/SimCAD/__init__.py @@ -1,2 +1,2 @@ name = "SimCAD" -configs = [] \ No newline at end of file +configs = [] diff --git a/SimCAD/configuration/__init__.py b/SimCAD/configuration/__init__.py index 1440b54..223de64 100644 --- a/SimCAD/configuration/__init__.py +++ b/SimCAD/configuration/__init__.py @@ -2,70 +2,36 @@ from functools import reduce from fn.op import foldr import pandas as pd -from SimCAD import configs from SimCAD.utils import key_filter -from SimCAD.configuration.utils.policyAggregation import dict_elemwise_sum -from SimCAD.configuration.utils import exo_update_per_ts +from SimCAD.configuration.utils.behaviorAggregation import dict_elemwise_sum -class Configuration(object): - def __init__(self, sim_config=None, initial_state=None, seeds=None, env_processes=None, - exogenous_states=None, partial_state_updates=None, policy_ops=[foldr(dict_elemwise_sum())]): +class Configuration: + def __init__(self, sim_config, state_dict, seed, exogenous_states, env_processes, mechanisms, behavior_ops=[foldr(dict_elemwise_sum())]): self.sim_config = sim_config - self.initial_state = initial_state - self.seeds = seeds - self.env_processes = env_processes + self.state_dict = state_dict + self.seed = seed self.exogenous_states = exogenous_states - self.partial_state_updates = partial_state_updates - self.policy_ops = policy_ops - - -def append_configs(sim_configs, initial_state, seeds, raw_exogenous_states, env_processes, partial_state_updates, _exo_update_per_ts=True): - if _exo_update_per_ts is True: - exogenous_states = exo_update_per_ts(raw_exogenous_states) - else: - exogenous_states = raw_exogenous_states - - if isinstance(sim_configs, list): - for sim_config in sim_configs: - configs.append( - Configuration( - sim_config=sim_config, - initial_state=initial_state, - seeds=seeds, - exogenous_states=exogenous_states, - env_processes=env_processes, - partial_state_updates=partial_state_updates - ) - ) - elif isinstance(sim_configs, dict): - configs.append( - Configuration( - sim_config=sim_configs, - initial_state=initial_state, - seeds=seeds, - exogenous_states=exogenous_states, - env_processes=env_processes, - partial_state_updates=partial_state_updates - ) - ) + self.env_processes = env_processes + self.behavior_ops = behavior_ops + self.mechanisms = mechanisms class Identity: - def __init__(self, policy_id={'identity': 0}): - self.beh_id_return_val = policy_id + def __init__(self, behavior_id={'identity': 0}): + self.beh_id_return_val = behavior_id - def p_identity(self, var_dict, sub_step, sL, s): + def b_identity(self, step, sL, s): return self.beh_id_return_val - def policy_identity(self, k): - return self.p_identity + def behavior_identity(self, k): + return self.b_identity - def no_state_identity(self, var_dict, sub_step, sL, s, _input): + def no_state_identity(self, step, sL, s, _input): return None def state_identity(self, k): - return lambda var_dict, sub_step, sL, s, _input: (k, s[k]) + return lambda step, sL, s, _input: (k, s[k]) def apply_identity_funcs(self, identity, df, cols): def fillna_with_id_func(identity, df, col): @@ -77,34 +43,34 @@ class Identity: class Processor: def __init__(self, id=Identity()): self.id = id - self.p_identity = id.p_identity - self.policy_identity = id.policy_identity + self.b_identity = id.b_identity + self.behavior_identity = id.behavior_identity self.no_state_identity = id.no_state_identity self.state_identity = id.state_identity self.apply_identity_funcs = id.apply_identity_funcs - def create_matrix_field(self, partial_state_updates, key): + def create_matrix_field(self, mechanisms, key): if key == 'states': identity = self.state_identity - elif key == 'policies': - identity = self.policy_identity - df = pd.DataFrame(key_filter(partial_state_updates, key)) + elif key == 'behaviors': + identity = self.behavior_identity + df = pd.DataFrame(key_filter(mechanisms, key)) col_list = self.apply_identity_funcs(identity, df, list(df.columns)) if len(col_list) != 0: return reduce((lambda x, y: pd.concat([x, y], axis=1)), col_list) else: return pd.DataFrame({'empty': []}) - def generate_config(self, initial_state, partial_state_updates, exo_proc): + def generate_config(self, state_dict, mechanisms, exo_proc): def no_update_handler(bdf, sdf): if (bdf.empty == False) and (sdf.empty == True): bdf_values = bdf.values.tolist() - sdf_values = [[self.no_state_identity] * len(bdf_values) for m in range(len(partial_state_updates))] + sdf_values = [[self.no_state_identity] * len(bdf_values) for m in range(len(mechanisms))] return sdf_values, bdf_values elif (bdf.empty == True) and (sdf.empty == False): sdf_values = sdf.values.tolist() - bdf_values = [[self.b_identity] * len(sdf_values) for m in range(len(partial_state_updates))] + bdf_values = [[self.b_identity] * len(sdf_values) for m in range(len(mechanisms))] return sdf_values, bdf_values else: sdf_values = sdf.values.tolist() @@ -113,19 +79,19 @@ class Processor: def only_ep_handler(state_dict): sdf_functions = [ - lambda sub_step, sL, s, _input: (k, v) for k, v in zip(state_dict.keys(), state_dict.values()) + lambda step, sL, s, _input: (k, v) for k, v in zip(state_dict.keys(), state_dict.values()) ] sdf_values = [sdf_functions] - bdf_values = [[self.p_identity] * len(sdf_values)] + bdf_values = [[self.b_identity] * len(sdf_values)] return sdf_values, bdf_values - if len(partial_state_updates) != 0: - bdf = self.create_matrix_field(partial_state_updates, 'policies') - sdf = self.create_matrix_field(partial_state_updates, 'states') + if len(mechanisms) != 0: + bdf = self.create_matrix_field(mechanisms, 'behaviors') + sdf = self.create_matrix_field(mechanisms, 'states') sdf_values, bdf_values = no_update_handler(bdf, sdf) zipped_list = list(zip(sdf_values, bdf_values)) else: - sdf_values, bdf_values = only_ep_handler(initial_state) + sdf_values, bdf_values = only_ep_handler(state_dict) zipped_list = list(zip(sdf_values, bdf_values)) return list(map(lambda x: (x[0] + exo_proc, x[1]), zipped_list)) diff --git a/SimCAD/configuration/utils/__init__.py b/SimCAD/configuration/utils/__init__.py index ab5113d..53dad43 100644 --- a/SimCAD/configuration/utils/__init__.py +++ b/SimCAD/configuration/utils/__init__.py @@ -1,18 +1,15 @@ from datetime import datetime, timedelta from decimal import Decimal -from copy import deepcopy from fn.func import curried import pandas as pd -from SimCAD.utils import dict_filter, contains_type - class TensorFieldReport: def __init__(self, config_proc): self.config_proc = config_proc - def create_tensor_field(self, partial_state_updates, exo_proc, keys=['policies', 'states']): - dfs = [self.config_proc.create_matrix_field(partial_state_updates, k) for k in keys] + def create_tensor_field(self, mechanisms, exo_proc, keys=['behaviors', 'states']): + dfs = [self.config_proc.create_matrix_field(mechanisms, k) for k in keys] df = pd.concat(dfs, axis=1) for es, i in zip(exo_proc, range(len(exo_proc))): df['es' + str(i + 1)] = es @@ -20,103 +17,41 @@ class TensorFieldReport: return df -# def s_update(y, x): -# return lambda step, sL, s, _input: (y, x) -# -# -def state_update(y, x): - return lambda sub_step, sL, s, _input: (y, x) - - def bound_norm_random(rng, low, high): - res = rng.normal((high+low)/2, (high-low)/6) - if res < low or res > high: + res = rng.normal((high+low)/2,(high-low)/6) + if (reshigh): res = bound_norm_random(rng, low, high) return Decimal(res) @curried -def proc_trigger(trigger_time, update_f, time): - if time == trigger_time: +def proc_trigger(trigger_step, update_f, step): + if step == trigger_step: return update_f else: return lambda x: x -tstep_delta = timedelta(days=0, minutes=0, seconds=30) -def time_step(dt_str, dt_format='%Y-%m-%d %H:%M:%S', _timedelta = tstep_delta): +t_delta = timedelta(days=0, minutes=0, seconds=30) +def time_step(dt_str, dt_format='%Y-%m-%d %H:%M:%S', _timedelta = t_delta): dt = datetime.strptime(dt_str, dt_format) t = dt + _timedelta return t.strftime(dt_format) -ep_t_delta = timedelta(days=0, minutes=0, seconds=1) -def ep_time_step(s, dt_str, fromat_str='%Y-%m-%d %H:%M:%S', _timedelta = ep_t_delta): - if s['sub_step'] == 0: +t_delta = timedelta(days=0, minutes=0, seconds=1) +def ep_time_step(s, dt_str, fromat_str='%Y-%m-%d %H:%M:%S', _timedelta = t_delta): + if s['mech_step'] == 0: return time_step(dt_str, fromat_str, _timedelta) else: return dt_str -# mech_sweep_filter -def partial_state_sweep_filter(state_field, partial_state_updates): - partial_state_dict = dict([(k, v[state_field]) for k, v in partial_state_updates.items()]) - return dict([ - (k, dict_filter(v, lambda v: isinstance(v, list))) for k, v in partial_state_dict.items() - if contains_type(list(v.values()), list) - ]) - - -def state_sweep_filter(raw_exogenous_states): - return dict([(k, v) for k, v in raw_exogenous_states.items() if isinstance(v, list)]) - -# sweep_mech_states -@curried -def sweep_partial_states(_type, in_config): - configs = [] - # filtered_mech_states - filtered_partial_states = partial_state_sweep_filter(_type, in_config.partial_state_updates) - if len(filtered_partial_states) > 0: - for partial_state, state_dict in filtered_partial_states.items(): - for state, state_funcs in state_dict.items(): - for f in state_funcs: - config = deepcopy(in_config) - config.partial_state_updates[partial_state][_type][state] = f - configs.append(config) - del config - else: - configs = [in_config] - - return configs - - -@curried -def sweep_states(state_type, states, in_config): - configs = [] - filtered_states = state_sweep_filter(states) - if len(filtered_states) > 0: - for state, state_funcs in filtered_states.items(): - for f in state_funcs: - config = deepcopy(in_config) - exploded_states = deepcopy(states) - exploded_states[state] = f - if state_type == 'exogenous': - config.exogenous_states = exploded_states - elif state_type == 'environmental': - config.env_processes = exploded_states - configs.append(config) - del config, exploded_states - else: - configs = [in_config] - - return configs - def exo_update_per_ts(ep): @curried - def ep_decorator(f, y, var_dict, sub_step, sL, s, _input): - if s['sub_step'] + 1 == 1: - return f(var_dict, sub_step, sL, s, _input) + def ep_decorator(f, y, step, sL, s, _input): + if s['mech_step'] + 1 == 1: + return f(step, sL, s, _input) else: - return y, s[y] - + return (y, s[y]) return {es: ep_decorator(f, es) for es, f in ep.items()} diff --git a/SimCAD/configuration/utils/policyAggregation.py b/SimCAD/configuration/utils/behaviorAggregation.py similarity index 77% rename from SimCAD/configuration/utils/policyAggregation.py rename to SimCAD/configuration/utils/behaviorAggregation.py index eac845d..5f8b5f6 100644 --- a/SimCAD/configuration/utils/policyAggregation.py +++ b/SimCAD/configuration/utils/behaviorAggregation.py @@ -2,18 +2,17 @@ from fn.op import foldr from fn.func import curried -def get_base_value(x): - if isinstance(x, str): +def get_base_value(datatype): + if datatype is str: return '' - elif isinstance(x, int): + elif datatype is int: return 0 - elif isinstance(x, list): + elif datatype is list: return [] - else: - return 0 + return 0 -def policy_to_dict(v): +def behavior_to_dict(v): return dict(list(zip(map(lambda n: 'b' + str(n + 1), list(range(len(v)))), v))) @@ -33,7 +32,7 @@ def sum_dict_values(): def dict_op(f, d1, d2): def set_base_value(target_dict, source_dict, key): if key not in target_dict: - return get_base_value(source_dict[key]) + return get_base_value(type(source_dict[key])) else: return target_dict[key] diff --git a/SimCAD/configuration/utils/parameterSweep.py b/SimCAD/configuration/utils/parameterSweep.py deleted file mode 100644 index 7d5024c..0000000 --- a/SimCAD/configuration/utils/parameterSweep.py +++ /dev/null @@ -1,20 +0,0 @@ -from SimCAD.utils import flatten_tabulated_dict, tabulate_dict - - -def process_variables(d): - return flatten_tabulated_dict(tabulate_dict(d)) - - -def config_sim(d): - if "M" in d: - return [ - { - "N": d["N"], - "T": d["T"], - "M": M - } - for M in process_variables(d["M"]) - ] - else: - d["M"] = [{}] - return d diff --git a/SimCAD/engine/__init__.py b/SimCAD/engine/__init__.py index 139d126..66a0773 100644 --- a/SimCAD/engine/__init__.py +++ b/SimCAD/engine/__init__.py @@ -16,16 +16,16 @@ class ExecutionContext: self.name = context self.method = None - def single_proc_exec(simulation_execs, var_dict, states_lists, configs_structs, env_processes_list, Ts, Ns): + def single_proc_exec(simulation_execs, states_lists, configs_structs, env_processes_list, Ts, Ns): l = [simulation_execs, states_lists, configs_structs, env_processes_list, Ts, Ns] simulation, states_list, config, env_processes, T, N = list(map(lambda x: x.pop(), l)) - result = simulation(var_dict, states_list, config, env_processes, T, N) + result = simulation(states_list, config, env_processes, T, N) return flatten(result) - def parallelize_simulations(fs, var_dict_list, states_list, configs, env_processes, Ts, Ns): - l = list(zip(fs, var_dict_list, states_list, configs, env_processes, Ts, Ns)) + def parallelize_simulations(fs, states_list, configs, env_processes, Ts, Ns): + l = list(zip(fs, states_list, configs, env_processes, Ts, Ns)) with Pool(len(configs)) as p: - results = p.map(lambda t: t[0](t[1], t[2], t[3], t[4], t[5], t[6]), l) + results = p.map(lambda t: t[0](t[1], t[2], t[3], t[4], t[5]), l) return results if context == 'single_proc': @@ -47,32 +47,30 @@ class Executor: create_tensor_field = TensorFieldReport(config_proc).create_tensor_field print(self.exec_context+": "+str(self.configs)) - var_dict_list, states_lists, Ts, Ns, eps, configs_structs, env_processes_list, partial_state_updates, simulation_execs = \ - [], [], [], [], [], [], [], [], [] + states_lists, Ts, Ns, eps, configs_structs, env_processes_list, mechanisms, simulation_execs = \ + [], [], [], [], [], [], [], [] config_idx = 0 for x in self.configs: - + states_lists.append([x.state_dict]) Ts.append(x.sim_config['T']) Ns.append(x.sim_config['N']) - var_dict_list.append(x.sim_config['M']) - states_lists.append([x.initial_state]) eps.append(list(x.exogenous_states.values())) - configs_structs.append(config_proc.generate_config(x.initial_state, x.partial_state_updates, eps[config_idx])) + configs_structs.append(config_proc.generate_config(x.state_dict, x.mechanisms, eps[config_idx])) env_processes_list.append(x.env_processes) - partial_state_updates.append(x.partial_state_updates) - simulation_execs.append(SimExecutor(x.policy_ops).simulation) + mechanisms.append(x.mechanisms) + simulation_execs.append(SimExecutor(x.behavior_ops).simulation) config_idx += 1 if self.exec_context == ExecutionMode.single_proc: - tensor_field = create_tensor_field(partial_state_updates.pop(), eps.pop()) - result = self.exec_method(simulation_execs, var_dict_list, states_lists, configs_structs, env_processes_list, Ts, Ns) + tensor_field = create_tensor_field(mechanisms.pop(), eps.pop()) + result = self.exec_method(simulation_execs, states_lists, configs_structs, env_processes_list, Ts, Ns) return result, tensor_field elif self.exec_context == ExecutionMode.multi_proc: if len(self.configs) > 1: - simulations = self.exec_method(simulation_execs, var_dict_list, states_lists, configs_structs, env_processes_list, Ts, Ns) + simulations = self.exec_method(simulation_execs, states_lists, configs_structs, env_processes_list, Ts, Ns) results = [] - for result, partial_state_updates, ep in list(zip(simulations, partial_state_updates, eps)): - results.append((flatten(result), create_tensor_field(partial_state_updates, ep))) + for result, mechanism, ep in list(zip(simulations, mechanisms, eps)): + results.append((flatten(result), create_tensor_field(mechanism, ep))) return results diff --git a/SimCAD/engine/simulation.py b/SimCAD/engine/simulation.py index 4d855b1..2660d2a 100644 --- a/SimCAD/engine/simulation.py +++ b/SimCAD/engine/simulation.py @@ -1,47 +1,39 @@ from copy import deepcopy from fn.op import foldr, call - from SimCAD.engine.utils import engine_exception id_exception = engine_exception(KeyError, KeyError, None) class Executor: - def __init__(self, policy_ops, policy_update_exception=id_exception, state_update_exception=id_exception): - self.policy_ops = policy_ops # behavior_ops + def __init__(self, behavior_ops, behavior_update_exception=id_exception, state_update_exception=id_exception): + self.behavior_ops = behavior_ops self.state_update_exception = state_update_exception - self.policy_update_exception = policy_update_exception # behavior_update_exception + self.behavior_update_exception = behavior_update_exception - # get_behavior_input - def get_policy_input(self, var_dict, sub_step, sL, s, funcs): - ops = self.policy_ops[::-1] + def get_behavior_input(self, step, sL, s, funcs): + ops = self.behavior_ops[::-1] - def get_col_results(var_dict, sub_step, sL, s, funcs): - return list(map(lambda f: f(var_dict, sub_step, sL, s), funcs)) + def get_col_results(step, sL, s, funcs): + return list(map(lambda f: f(step, sL, s), funcs)) - return foldr(call, get_col_results(var_dict, sub_step, sL, s, funcs))(ops) + return foldr(call, get_col_results(step, sL, s, funcs))(ops) - def apply_env_proc(self, env_processes, state_dict, sub_step): + def apply_env_proc(self, env_processes, state_dict, step): for state in state_dict.keys(): if state in list(env_processes.keys()): env_state = env_processes[state] if (env_state.__name__ == '_curried') or (env_state.__name__ == 'proc_trigger'): - state_dict[state] = env_state(sub_step)(state_dict[state]) + state_dict[state] = env_state(step)(state_dict[state]) else: state_dict[state] = env_state(state_dict[state]) - # mech_step - def partial_state_update(self, var_dict, sub_step, sL, state_funcs, policy_funcs, env_processes, time_step, run): + def mech_step(self, m_step, sL, state_funcs, behavior_funcs, env_processes, t_step, run): last_in_obj = sL[-1] - _input = self.policy_update_exception(self.get_policy_input(var_dict, sub_step, sL, last_in_obj, policy_funcs)) + _input = self.state_update_exception(self.get_behavior_input(m_step, sL, last_in_obj, behavior_funcs)) - # ToDo: add env_proc generator to `last_in_copy` iterator as wrapper function - last_in_copy = dict( - [ - self.state_update_exception(f(var_dict, sub_step, sL, last_in_obj, _input)) for f in state_funcs - ] - ) + last_in_copy = dict([self.behavior_update_exception(f(m_step, sL, last_in_obj, _input)) for f in state_funcs]) for k in last_in_obj: if k not in last_in_copy: @@ -51,49 +43,47 @@ class Executor: self.apply_env_proc(env_processes, last_in_copy, last_in_copy['timestamp']) - last_in_copy["sub_step"], last_in_copy["time_step"], last_in_copy['run'] = sub_step, time_step, run + last_in_copy["mech_step"], last_in_copy["time_step"], last_in_copy['run'] = m_step, t_step, run sL.append(last_in_copy) del last_in_copy return sL - # mech_pipeline - def state_update_pipeline(self, var_dict, states_list, configs, env_processes, time_step, run): - sub_step = 0 + def mech_pipeline(self, states_list, configs, env_processes, t_step, run): + m_step = 0 states_list_copy = deepcopy(states_list) genesis_states = states_list_copy[-1] - genesis_states['sub_step'], genesis_states['time_step'] = sub_step, time_step + genesis_states['mech_step'], genesis_states['time_step'] = m_step, t_step states_list = [genesis_states] - sub_step += 1 + m_step += 1 for config in configs: - s_conf, p_conf = config[0], config[1] - states_list = self.partial_state_update(var_dict, sub_step, states_list, s_conf, p_conf, env_processes, time_step, run) - sub_step += 1 + s_conf, b_conf = config[0], config[1] + states_list = self.mech_step(m_step, states_list, s_conf, b_conf, env_processes, t_step, run) + m_step += 1 - time_step += 1 + t_step += 1 return states_list - def run_pipeline(self, var_dict, states_list, configs, env_processes, time_seq, run): + def block_pipeline(self, states_list, configs, env_processes, time_seq, run): time_seq = [x + 1 for x in time_seq] simulation_list = [states_list] for time_step in time_seq: - pipe_run = self.state_update_pipeline(var_dict, simulation_list[-1], configs, env_processes, time_step, run) + pipe_run = self.mech_pipeline(simulation_list[-1], configs, env_processes, time_step, run) _, *pipe_run = pipe_run simulation_list.append(pipe_run) return simulation_list - # ToDo: Muiltithreaded Runs - def simulation(self, var_dict, states_list, configs, env_processes, time_seq, runs): + def simulation(self, states_list, configs, env_processes, time_seq, runs): pipe_run = [] for run in range(runs): run += 1 states_list_copy = deepcopy(states_list) - head, *tail = self.run_pipeline(var_dict, states_list_copy, configs, env_processes, time_seq, run) + head, *tail = self.block_pipeline(states_list_copy, configs, env_processes, time_seq, run) genesis = head.pop() - genesis['sub_step'], genesis['time_step'], genesis['run'] = 0, 0, run + genesis['mech_step'], genesis['time_step'], genesis['run'] = 0, 0, run first_timestep_per_run = [genesis] + tail.pop(0) pipe_run += [first_timestep_per_run] + tail del states_list_copy diff --git a/SimCAD/engine/utils.py b/SimCAD/engine/utils.py index c0f3a76..bcf1507 100644 --- a/SimCAD/engine/utils.py +++ b/SimCAD/engine/utils.py @@ -24,8 +24,6 @@ def retrieve_state(l, offset): return l[last_index(l) + offset + 1] -# exception_function = f(sub_step, sL, sL[-2], _input) -# try_function = f(sub_step, sL, last_mut_obj, _input) @curried def engine_exception(ErrorType, error_message, exception_function, try_function): try: @@ -33,10 +31,3 @@ def engine_exception(ErrorType, error_message, exception_function, try_function) except ErrorType: print(error_message) return exception_function - - -@curried -def fit_param(param, x): - return x + param - -# fit_param = lambda param: lambda x: x + param diff --git a/SimCAD/utils/__init__.py b/SimCAD/utils/__init__.py index b29ba08..720ab6a 100644 --- a/SimCAD/utils/__init__.py +++ b/SimCAD/utils/__init__.py @@ -1,7 +1,3 @@ -from collections import defaultdict -from itertools import product -import warnings - def pipe(x): return x @@ -11,125 +7,14 @@ def print_pipe(x): return x -def flattenDict(l): - def tupalize(k, vs): - l = [] - if isinstance(vs, list): - for v in vs: - l.append((k, v)) - else: - l.append((k, vs)) - return l - - flat_list = [tupalize(k, vs) for k, vs in l.items()] - flat_dict = [dict(items) for items in product(*flat_list)] - return flat_dict - - def flatten(l): - if isinstance(l, list): - return [item for sublist in l for item in sublist] - elif isinstance(l, dict): - return flattenDict(l) - - -def flatMap(f, collection): - return flatten(list(map(f, collection))) - - -def dict_filter(dictionary, condition): - return dict([(k, v) for k, v in dictionary.items() if condition(v)]) - - -def get_max_dict_val_len(g): - return len(max(g.values(), key=len)) - - -def tabulate_dict(d): - max_len = get_max_dict_val_len(d) - _d = {} - for k, vl in d.items(): - if len(vl) != max_len: - _d[k] = vl + list([vl[-1]] * (max_len-1)) - else: - _d[k] = vl - - return _d - - -def flatten_tabulated_dict(d): - max_len = get_max_dict_val_len(d) - dl = [{} for i in range(max_len)] - - for k, vl in d.items(): - for v, i in zip(vl, list(range(len(vl)))): - dl[i][k] = v - - return dl - - -def contains_type(_collection, type): - return any(isinstance(x, type) for x in _collection) - - -def drop_right(l, n): - return l[:len(l) - n] + return [item for sublist in l for item in sublist] def key_filter(l, keyname): - if (type(l) == list): - return [v[keyname] for v in l] - # Keeping support to dictionaries for backwards compatibility - # Should be removed in the future - warnings.warn( - "The use of a dictionary to describe Partial State Update Blocks will be deprecated. Use a list instead.", - FutureWarning) return [v[keyname] for k, v in l.items()] -def groupByKey(l): - d = defaultdict(list) - for key, value in l: - d[key].append(value) - return list(dict(d).items()).pop() - - -# @curried def rename(new_name, f): f.__name__ = new_name return f - - -def curry_pot(f, *argv): - sweep_ind = f.__name__[0:5] == 'sweep' - arg_len = len(argv) - if sweep_ind is True and arg_len == 4: - return f(argv[0])(argv[1])(argv[2])(argv[3]) - elif sweep_ind is False and arg_len == 4: - return f(argv[0], argv[1], argv[2], argv[3]) - elif sweep_ind is True and arg_len == 3: - return f(argv[0])(argv[1])(argv[2]) - elif sweep_ind is False and arg_len == 3: - return f(argv[0], argv[1], argv[2]) - else: - raise TypeError('curry_pot() needs 3 or 4 positional arguments') - -# def curry_pot(f, *argv): -# sweep_ind = f.__name__[0:5] == 'sweep' -# arg_len = len(argv) -# if sweep_ind is True and arg_len == 4: -# return f(argv[0])(argv[1])(argv[2])(argv[3]) -# elif sweep_ind is False and arg_len == 4: -# return f(argv[0])(argv[1])(argv[2])(argv[3]) -# elif sweep_ind is True and arg_len == 3: -# return f(argv[0])(argv[1])(argv[2]) -# elif sweep_ind is False and arg_len == 3: -# return f(argv[0])(argv[1])(argv[2]) -# else: -# raise TypeError('curry_pot() needs 3 or 4 positional arguments') - -# def rename(newname): -# def decorator(f): -# f.__name__ = newname -# return f -# return decorator diff --git a/Simulation.md b/Simulation.md index a6e1347..81d1ece 100644 --- a/Simulation.md +++ b/Simulation.md @@ -1,10 +1,10 @@ -# SimCAD Documentation +# SimmCAD Documentation ## Introduction A blockchain is a distributed ledger with economic agents transacting in a network. The state of the network evolves with every new transaction, which can be a result of user behaviors, protocol-defined system mechanisms, or external processes. -It is not uncommon today for blockchain projects to announce a set of rules for their network and make claims about their system level behavior. However, the validity of those claims is hardly validated. Furthermore, it is difficult to know the potential system-level impact when the network is considering an upgrade to their system rules and parameters. +It is not uncommon today for blockchain projects to announce a set of rules for their network and make claims about their system level behvaior. However, the validity of those claims is hardly validated. Furthermore, it is difficult to know the potential system-level impact when the network is considering an upgrade to their system rules and prameters. To rigorously and reliably analyze, design, and improve cryptoeconomic networks, we are introducing this Computer Aided Design Engine where we define a cryptoeconomic network with its state and exogneous variables, model transactions as a result of agent behaviors, state mechanisms, and environmental processes. We can then run simulations with different initial states, mechanisms, environmental processes to understand and visualize network behavior under different conditions. diff --git a/dist/SimCAD-0.1-py3-none-any.whl b/dist/SimCAD-0.1-py3-none-any.whl index d6a18cf3b4ece9d96dc8275acf765fab7dcc56cd..e42e99a2bd65a1c085c532d44b7331fe1b5f83b4 100644 GIT binary patch delta 5790 zcmZXYbx_q$zsC=)lER_SAtVpoAl)V1-H4PlNaz=&8$5K#k2t`gM7lwwyOB<5=@6+a zckVOKbMM`m&%9=K-?RJAXLn{_$-=*ka5P?_p@Tsn&=b%}Xd;dZ>PhHVBR2HEsE$Ct zM15o+))>kowVqa@JrW)F`oH-=0Dqi^ClU4#%7wA2{Tx9Q7T|+GEes$K1O$JeBn4EI zWaQMejyEcA+vni$CkYc}Of<@~q$3FA0uCCMG zZ1L2d28j|;rfo7uya95Zc~J6b*8HBd@Fr5E8~>u|)y7g^*;~h3_wqWJti1o zoe{VoG()^&?Xo##OJVB()rBovXrjGl)pc^^Q>^fUCJxe>owBuHMVE+0QaYHD!;DWU z%i&&NU`=7L%V0RB&~r4-BVs@}c2Y*ivAEv!&pzh^5*)7^Ry|1MJ0Y1u8NL?bSEOtT z;6-a=lt|@P?&0>ZMWk^@J0(| zDVED_NvZ|6scH&M)#5iT4*f$*7Gfx#p#z`VTRg=5efd;+94M*&ToKvO@RD9bfnoH7 zW1zl3#!QmUzUy?ce(L9PWAry7S(PAdSA0h?4QEcwcbMoLN-%>=fb65d#tZd{rVWd} ztKxc1IMZr%+>>`}Z1yviqM|4#`VzjM+w!1_j9xZk_MS;+u};dgR&B_(|_$E({P5lT{d zrm+l}d5f&?ist!zRe-y(bp+iAaw3B;;|Wa`$fT0RtAyys!<~Vc-*Qq=NJ8H1ec@+L zoa@X&0?=pQBl3X_t5MwPK6|KT!8#8DRw4#5%2TfnT@udqGa;URW<%y%lN9lf=Dv$a zHnb`K8>Y-eKXZ+{q$jm)_TCU<=`?fv(}a@om^Y0O`yt-rV1O2%uF2S+P*tbXBTCfB z?)I8<3>mb~<}u*eTq+{y*jVVRLWcO(B}D;Rp~)4eElYJxLuY2b)K3h$jqv3S-k@ZW zF0PnjeKLqV)}t_vvyXHSM|iO>P`nIXYwlaE`TmbwvX@@{I1~G;_iR>CaoVMx*w};x zx@bC=>poI-2tdv$MS)0oNSJcY)~5(Y4nUf0C9?=7-r9$ZFN8a_B#3q?-3l2B3@UzH>n zJLXZe(X0ylsja-Zhr#OZ#q|J17yd3`9qkp#OF$S*%XtP;UKV-}>?j-pp88)pS+72nI zRl6pO%Ntr}?T#gd=Y5=u&U*UI?~nv%lKje5TAPEpRl=tGsatoD<&8%0#S$t1?-$FO z{QE4b6F{+zgLT?F+WrjpaswVNqZ;OB!tiTJo^n~VNRtqf-?Iy-O(g#9`h$b-Q;#iM zM3RtV>vh4wCnK~3j5zm+i<82WrZ9aTc0{AKmnRhK#IoRWRo&SOPv-R$bN7y5jmm}8 zX`PRhE%6EcHxzk~vmS-iF?c`BhNm~74Rg9?pUCvyu`T4#%dQP{de)+qpKdwn- z2H&n$-Xs;F;$4a!UQ2Q_Aw2A%iSmIuv!jh z8KF3!rWNDk;E_N<_appb;>x9YJ()G%$(V-eh@iy)i|TpE+n__V;}n`w>90zjG!#`C zsFKa7p`Uo7cF&$&KZQUuyu#f5Hgzf+?trQ_5;$7}Ok$5{gJ_(df%%^GMS!s4OTj$z zg$DXRC~7mf_WNJLKPF6Ft7WBxqVaIAH`|Ow*cP1`fnuGg3`l+o(tomP;w0lK=aX-= zo9CR-T~vG?Kqx35fy~!(;C0n=b0a|!PDaDTH}vEwTH~iv@_S#QG%UBsg-teg2?LgN zCL@FZNGXz9?RCY&0mkXAnkvaC=izO&pZCLIs6(5EM5EN|NNO8t+?2Zd@@YSI(p5aE z4SUORj`{00K`7F7mqUM0?%0qWwRSZx=XUHn9&p`-bG5FT;?St^+kHkf1_FqumlxjO zZYYYCr3kr29{fJwY=zCl+XUbA0~dCO!4hYwISPBd$%E9QJKfk+q~EGVa^9(Ar*`#b z^NFqi=S+uAmvi>Ec*IoxhJMe}lFMz-PJUk4aOeC{nFhCQXIXLRrFg&xju!nb`v~1J zLpoT%;=Y~MOw;-8S;4p%zF>6MV1$I>ut{ITGc%Gpoixs^mG!W6Ab5cRo; zimd}b$SCwsA^)Z$n}YGEuv3$@l8oiJ*yh;6vpT>%Rewh7Lvag@0s2b27$#NaSwFs> zY3$l2VE@Umfk%d~&xy&IFp^zO>?QLgB*Vgnnk~YPs^}^H z+Xz*52P4%;>^6OmQBqD`l~8rr-C(SWMq|(mS8qF1DL}b){R&$Ls6LmFQmC}txohb3 zPo(XrLQ;>R6&>;(Fc6NASniz9=W4E%@X@rem<)ZL%z1Do4v_c0Lfh}8ae)rwPj#7& zcjsz8L%mAJrsb|!K=PpcdHbW(a&v7s1`b+!auJlc$j7maWhoB^*aN@Nu zvn=aN;a3xfBzNop78oEGhU+$%h)y=)=Y+0z9FN`Pkg;yX=c$;lW3RNr|JZA7QcfPz zX5KWOdlG7;!>T{#QX^{B5Wg+RJ+!9demz%pU$#~ z!tFf}&<^}YTQ@4x!AbP$<68S02U-_W?{mZ(IYL%LyY>{I?1lU1t_tX#@FxXxj?DW7 zay~rJFJpJ24u9yJ9vhSyz1xZYdjYR=S`f)Vwuw+8VWc%4xmzEbW*FrglRq(i!tMQ& zLRDgOMsSuiiUNgG6xcBjm8-)_hBaok2Zo$&aM=m*nc# zKm8Pv!dZ2IT@{9hgM(Y!?%wS1sEu<3WIDi6t5z z2GNaoAHkM8lX-4CP$*=(0Tv2ofW!H-*`b_wNG~9)FEKLzKFTW2d~R=SSdqVEo{+Qx zwKW}PVw8SOrh$c^d>@>@Lxvz1`4M-{xVMd(x(&DrC8ELX*#%U4<^z^y?&@U%J!QvL zQ}5X;6tvFu@xb~dJcHc4lN{w5?_W~J2#O+X zJpnGNM)Hr#&*-!$SM2k7&W&~~2=_nO1@TA!kP!l9W?$jEib$IvF=T#48{6KPDO zRuWO>DPww>H>ra>*-_L%W2_B(1GWks-U7>Jj?(5<+?1@tYQ*gn0@%hRSB*MLJ7UkD zSUshgAz<9`;4-hVCWyi+tD#6hlQl{|s|7&M_ohyd#;I$yCuAWzg}PUx%7`|TH*C5VO2ak z@+6sSOw9qa8^XO`+*PI36KGG>%5Qp<|CN|-X)rKvhDTDqX-9fzOv7q-UDR2`<>ePy z8nW{lMqS9zrx4mQAsBDs5*?w|H^3!Dn`d@Y$jieWU0K>Bg0IJ_jACyG`vqmjYFhP_ z#Ne>Z?;vLgQwbz%#3j{A=9PC_rvBD}j&lCui|-@bBQ47#bPV&!J%gB}h9i|XRIEo6 z=#D?sE_s3Hx3ACb$IY|j`#99z+DQqD8@=u?Wda|1x4*^=u&A(gQ z>KzT2q$$1A)|>@qu#3)>8&wl^A@&I0@1<43!|!CKl0Df)bN58JY4;Lp6uTB6I*t%n z=HSbtd5z{idBj_XmBi+WXmXFKU9&+x`q{n1rPlf2YSUCyXZlj?ZPX(=5ubx3uje8h z!S_h&5kqE^p)Gkk9|7p>2!N5B%S-5x8sOa2`_Nrui zXgPCj36a2K3gP*&e}CY;RRL`C|AY~JMDJDh|$0rV}Z!-Hgq8RX0ProdK$@a9`DpzQOV>-zBP!$sq= z&U&%FEqm>bA2$?X+{p5`OX!t2#Z9l=^cVut_Vq6sncLG|irWFI<*eJc-iHEcic)gk zeA`O36-1ZxJ;9-EQ{d@iG7#A^&ghQW50v`JdrisdGRq?Y|Jy|x2ZlO%XGaVfl9R2h zuLWaURtqB?@h0ulYMitU%_!z z8C0KYq=wtbvG9j-44G!6+#bdk*e&Z;|qw0Ju5AVQ>6~mi^@LrdiO1$I z!4oMFFE3jSW%Xr=5VA7iC7ntXIV%!9oVd{&Koy%)0#-N07MUSUJON0<&HA zcvL4OFHNJxL%t3zfpyOcB3-*sE9||2)EPNA^eZxpQ;ycH@&~P44|?m?Q(JEb$Wfc@ z$QSk!s@H}HePJ@@F1v~Puz6Nr6rvM`QSxp=*-3J==;|sye8c*L+z+4~);Yg8J~`gK z$z@IvFZljYy_e(6V(;>>s4|pC8rRStX%q3z%$NM#__la0-@Fx*_j0Ur>P?=W)uFZH zY?POEU+wAHX`O}Y0e_BPv6zpMSJG6hOt!JvaZGVsIo|8gO8Dgn#XF2r%%h|b8cX<1 zXRMUY6#U#n$!FrEARACHEsV}X#IB{Qp>9?7ys=P-2ur6GK38_h3=#dtH%9QO|H66% z^l75ULBq%n2VUo!I{k_0H1XBX*!N1KG0@-$Fd4$q0FLYEmSBuNh+}i})NnzOW^7-m zhB%}dQSmvA(G)=Yi<8>Ci)hp5YydW? zA7EQQpNvc8td~Xeqt1{KXUH0Ruh#5s!9SR4Z(M~orG~+Y9rm~w@q#V_K%_vqV+$A> zOOk;SJaC@rNydHJ9#YmN`{!A!&eoc8zV`-(;RcrwHE;giVVopPgr~u5W74#@PY7Ft z_+~km_{w7}nvpFPw}%>wr7sSXynv0i+QUncdPrEn%Mje{^1nN_dVeY&UP`2mHo-ds zM*$0<{cU5oDUm$hV>@s1N{G9nfA{SF|{~hm(-AwvLSGl zA**eyGuD-?t)6JyoGuFE!2}_ml%O->T8GZ) zl4*uyGjA$qTw5>jSnE%S9JOUl#+7>Naw(X^cY$I1oxQUdY5uj3)#`9$2x3;8Pj&7C z43rzpCH7-==-UBxCFSiZyy6sSAGbgc6jWR+H~OMx$fN|bO)}n1@33G+sZZ;unUbyIS?law)u>z*!KzaYQcU$^3{He&EI zAB^JvJ(z((EoHhqR$J-`_?^H}tc@a_T1`}(lV-2j7 z&0ygx(`kk`Q5!&{SV`EaPhZ79HA*UX;?(NWSS7tfk$&OZ2NE<2+KX%p8oF6v+Vv0Az$EFvqV ztM?J{Yj`8QfWcW~%6M>#Zo%r+>u^?RfcTgrfuOTNrwH51Nkoq8@#eeiM~pd+U@fxEj_GPj1l#JUsn`VFxvlUSd>tB7`gbPS?wWz z^s#U-h1Mh9P?$cVmqP9lc$5l{(5IAsL?)%|^Z(~y;m4GWkpEGz|9Qf{U+_Q(F)9b$ zqpF2lfS<7a-_YZwG5PC;AB*B|{c98iff(WH5N4XkolvT{RiFM{!TMV-{+0O4f}kYE pzdLjNcj(_Y3_pk5w^EtTEUXkblfg(X5 zkQ~U4QKsk#mS;Q&HiLo}=$u063K+aj=j3d8E(2C#aQx&OAz)nw2dcCN0k70@z9Rq` zuq6Yi*v5zWUa6FC&9URsiW>(7G@vyAD>McovJwHO&^GcLFU|^s1*L$C zfeF>CM7RgRoemg+pE~^RrRsQyA@sCNea@n&&M=y^pIuuanWLoscGNZV-ML%tMUL5v zICNG;Z?$TCr$-^b-I04IEP(Kk%`_r?fy2eiQ)=FI_*m@F)gH&aWTEx8-d#P8&5tx^ zdVQ@AlKSgZ=lzzv?2{0vK*uCg(WW8)!T#ESrCqTAEFao3XiT!Fmu z@UVH-K=1dHVy7{GVfQ+hM!Th3lK@x^UXyezuqAZMtR`!IxKT=XNnF5J`cc1`$)@&IMl&6PBd+y{ejG+yD_9$8BMJ}udVsdVF zJU~^9YW74Pe-+g{m3MZ^k3tF)8j$++^=0uCjgiR7;2rPPyD^ZyY}XMB6}cs}09)MA z1RW5Gpz#5=2l#O3h=Sa$dm^wQ>2W_rSx*M{s!~Ds<}0xI;SxPHe8ISOu3uuLgxk$*zf!iWoNA)XC78}n zpHlbYXcbS4LG3)U=U~&0s?7mHZ#IV zjyiUR<&1>A`?L0wt&AyC#$rWQ?kH3ZE0<1k#!p#ax6+j?wUdRn$+OGuyzgOFPX%)q z=-e7_?32M6>sZ3pFw#-b_p{ocO$I&^?68=yCZ&;BBmXZKeR=6K{xH-OKoYZ#9C)zs z)@i;V`ONcO!)=+K3$XPMA6#gKj!s2+RGWOSyv&gXS%L7{7Tthhd;4XR8YT@1SLrKq z&d^UYY>>p`f;8^cgf`_#hGNduK6cGlGq^fkq~JQ&d;0Fr<}9At>D{oLlLDu2(9lU- z{JeYBc=!r#O}+B>N$q0Vljghle$R`7(et*VzU%z1tjeyMBl!MS^Ol10@*qhi73y1M zuo_DVI^l5j?IZ^kH!Pr&l|CK%HnLqSlVFd;#>f#>zUKQR-rK!ym>8NS9A?+$k}p}RCJVQ8j(%9mpMcWVig-?aws26}9IPeDbrp1hsyfS$ zL{@Wj$k&YrJ4OwXSeV45F(P%_aCV|k=Or)EvHCflmR#!2n&GVNSZo~Xmf%RxY5iZy zniD!xp22Ph&cu>4aHMGR75Q3*$0GZMztBQ-v_A;`%-~$0sjiA{@Gi~25*f~@xDZ+f diff --git a/notebooks/jjodesty/multithreading.ipynb b/notebooks/jjodesty/multithreading.ipynb deleted file mode 100644 index c3a41ad..0000000 --- a/notebooks/jjodesty/multithreading.ipynb +++ /dev/null @@ -1,576 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "import _thread\n", - "import time\n", - "from fn.func import curried" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Define a function for the thread\n", - "def f(threadName, delay):\n", - " count = 0\n", - " print(count)\n", - " # while count < 5:\n", - " # time.sleep(delay)\n", - " # count += 1\n", - " # print(count)\n", - " \n", - "def pipe(x):\n", - " print(x)\n", - " return x" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "00\n", - "\n" - ] - } - ], - "source": [ - "# Create two threads as follows\n", - "try:\n", - " _thread.start_new_thread( f, (\"Thread-1\", 2, ) )\n", - " _thread.start_new_thread( f, (\"Thread-2\", 4, ) )\n", - "except:\n", - " print (\"Error: unable to start thread\")\n", - "\n", - "while 1:\n", - " pass\n" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[1, 2]\n", - "('s2', . at 0x1099efae8>)\n", - "('s2', . at 0x1099ef9d8>)\n" - ] - } - ], - "source": [ - "from SimCAD.engine.utils import sweep\n", - "from SimCAD.utils import rename\n", - "from SimCAD.configuration.utils import s_update\n", - "\n", - "# @curried\n", - "def fit_param(param):\n", - " return lambda x: x + param\n", - "\n", - "# xf = lambda param: lambda x: x + param\n", - "\n", - "def sweep(params, y, xf):\n", - " op = [rename('sweep', s_update(y, xf(param))) for param in params]\n", - " print(params)\n", - " # print()\n", - " return op\n", - "\n", - "for f in sweep([1,2], 's2', fit_param):\n", - " print(f(1,2,3,4))\n", - "# sweep([1,2], 's2', xf)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[1, 64, 2187, 65536]\n" - ] - } - ], - "source": [ - "# instantiate and configure the worker pool\n", - "from pathos.threading import ThreadPool\n", - "pool = ThreadPool(nodes=4)\n", - "\n", - "# do a blocking map on the chosen function\n", - "print(pool.map(pow, [1,2,3,4], [5,6,7,8]))" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "ename": "SyntaxError", - "evalue": "invalid syntax (, line 3)", - "traceback": [ - "\u001b[0;36m File \u001b[0;32m\"\"\u001b[0;36m, line \u001b[0;32m3\u001b[0m\n\u001b[0;31m [for f in fs]\u001b[0m\n\u001b[0m ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n" - ], - "output_type": "error" - } - ], - "source": [ - "with Pool(len(configs)) as p:\n", - " results = p.map(lambda t: t[0](t[1], t[2], t[3], t[4], t[5]), l)\n", - " \n", - "\n", - "def state_multithreading(self, fs, m_step, sL, last_in_obj, _input):\n", - " if type(fs) == 'list':\n", - " pool.map(f(m_step, sL, last_in_obj, _input), fs)\n", - " else:\n", - " f(m_step, sL, last_in_obj, _input)" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[('s2', [11, 23])]" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from itertools import groupby\n", - "l = [('s2', 11), ('s2', 23)]\n", - "l.sort(key = lambda i : i[0])\n", - "[(key, [i[1] for i in values]) for key, values in groupby(l, lambda i: i[0])]" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [], - "source": [ - "def groupByKV(l):\n", - " l.sort(key = lambda i : i[0])\n", - " return [(key, [i[1] for i in values]) for key, values in groupby(l, lambda i: i[0])]" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[('s2', [11, 23])]" - ] - }, - "execution_count": 18, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "groupByKV(l)" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [ - { - "ename": "SyntaxError", - "evalue": "invalid syntax (, line 2)", - "traceback": [ - "\u001b[0;36m File \u001b[0;32m\"\"\u001b[0;36m, line \u001b[0;32m2\u001b[0m\n\u001b[0;31m collect = lambda tuplist: reduce(lambda acc, (k,v): acc[k].append(v) or acc,tuplist, defaultdict(list))\u001b[0m\n\u001b[0m ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n" - ], - "output_type": "error" - } - ], - "source": [ - "from collections import defaultdict \n", - "collect = lambda tuplist: reduce(lambda acc, (k,v): acc[k].append(v) or acc,tuplist, defaultdict(list))" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [], - "source": [ - "from collections import defaultdict\n", - "d = defaultdict(list)\n", - "for key, value in [('s2', 11), ('s2', 23)]:\n", - " d[key].append(value)" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": { - "collapsed": true - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Help on defaultdict object:\n", - "\n", - "class defaultdict(builtins.dict)\n", - " | defaultdict(default_factory[, ...]) --> dict with default factory\n", - " | \n", - " | The default factory is called without arguments to produce\n", - " | a new value when a key is not present, in __getitem__ only.\n", - " | A defaultdict compares equal to a dict with the same items.\n", - " | All remaining arguments are treated the same as if they were\n", - " | passed to the dict constructor, including keyword arguments.\n", - " | \n", - " | Method resolution order:\n", - " | defaultdict\n", - " | builtins.dict\n", - " | builtins.object\n", - " | \n", - " | Methods defined here:\n", - " | \n", - " | __copy__(...)\n", - " | D.copy() -> a shallow copy of D.\n", - " | \n", - " | __getattribute__(self, name, /)\n", - " | Return getattr(self, name).\n", - " | \n", - " | __init__(self, /, *args, **kwargs)\n", - " | Initialize self. See help(type(self)) for accurate signature.\n", - " | \n", - " | __missing__(...)\n", - " | __missing__(key) # Called by __getitem__ for missing key; pseudo-code:\n", - " | if self.default_factory is None: raise KeyError((key,))\n", - " | self[key] = value = self.default_factory()\n", - " | return value\n", - " | \n", - " | __reduce__(...)\n", - " | Return state information for pickling.\n", - " | \n", - " | __repr__(self, /)\n", - " | Return repr(self).\n", - " | \n", - " | copy(...)\n", - " | D.copy() -> a shallow copy of D.\n", - " | \n", - " | ----------------------------------------------------------------------\n", - " | Data descriptors defined here:\n", - " | \n", - " | default_factory\n", - " | Factory for default value called by __missing__().\n", - " | \n", - " | ----------------------------------------------------------------------\n", - " | Methods inherited from builtins.dict:\n", - " | \n", - " | __contains__(self, key, /)\n", - " | True if D has a key k, else False.\n", - " | \n", - " | __delitem__(self, key, /)\n", - " | Delete self[key].\n", - " | \n", - " | __eq__(self, value, /)\n", - " | Return self==value.\n", - " | \n", - " | __ge__(self, value, /)\n", - " | Return self>=value.\n", - " | \n", - " | __getitem__(...)\n", - " | x.__getitem__(y) <==> x[y]\n", - " | \n", - " | __gt__(self, value, /)\n", - " | Return self>value.\n", - " | \n", - " | __iter__(self, /)\n", - " | Implement iter(self).\n", - " | \n", - " | __le__(self, value, /)\n", - " | Return self<=value.\n", - " | \n", - " | __len__(self, /)\n", - " | Return len(self).\n", - " | \n", - " | __lt__(self, value, /)\n", - " | Return self size of D in memory, in bytes\n", - " | \n", - " | clear(...)\n", - " | D.clear() -> None. Remove all items from D.\n", - " | \n", - " | fromkeys(iterable, value=None, /) from builtins.type\n", - " | Returns a new dict with keys from iterable and values equal to value.\n", - " | \n", - " | get(...)\n", - " | D.get(k[,d]) -> D[k] if k in D, else d. d defaults to None.\n", - " | \n", - " | items(...)\n", - " | D.items() -> a set-like object providing a view on D's items\n", - " | \n", - " | keys(...)\n", - " | D.keys() -> a set-like object providing a view on D's keys\n", - " | \n", - " | pop(...)\n", - " | D.pop(k[,d]) -> v, remove specified key and return the corresponding value.\n", - " | If key is not found, d is returned if given, otherwise KeyError is raised\n", - " | \n", - " | popitem(...)\n", - " | D.popitem() -> (k, v), remove and return some (key, value) pair as a\n", - " | 2-tuple; but raise KeyError if D is empty.\n", - " | \n", - " | setdefault(...)\n", - " | D.setdefault(k[,d]) -> D.get(k,d), also set D[k]=d if k not in D\n", - " | \n", - " | update(...)\n", - " | D.update([E, ]**F) -> None. Update D from dict/iterable E and F.\n", - " | If E is present and has a .keys() method, then does: for k in E: D[k] = E[k]\n", - " | If E is present and lacks a .keys() method, then does: for k, v in E: D[k] = v\n", - " | In either case, this is followed by: for k in F: D[k] = F[k]\n", - " | \n", - " | values(...)\n", - " | D.values() -> an object providing a view on D's values\n", - " | \n", - " | ----------------------------------------------------------------------\n", - " | Data and other attributes inherited from builtins.dict:\n", - " | \n", - " | __hash__ = None\n", - "\n" - ] - } - ], - "source": [ - "help(d)" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'s2': [11, 23]}" - ] - }, - "execution_count": 27, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "dict(d)" - ] - }, - { - "cell_type": "code", - "execution_count": 55, - "metadata": {}, - "outputs": [], - "source": [ - "def groupByKey(l):\n", - " d = defaultdict(list)\n", - " for key, value in l:\n", - " d[key].append(value)\n", - " return list(dict(d).items()).pop()" - ] - }, - { - "cell_type": "code", - "execution_count": 56, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "('s2', [11, 23])" - ] - }, - "execution_count": 56, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "r = groupByKey([('s2', 11), ('s2', 23)])\n", - "r" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# xf = lambda param: 1.0 + param\n", - "# def xf(y, param, s):\n", - "# return s[y] + param\n", - "\n", - "# def fit_param(param):\n", - "# y = 's2'\n", - "# x = 1 + param\n", - "# return lambda step, sL, s, _input: (y, x)\n", - "#\n", - "# def fit_param(param):\n", - "# return lambda step, sL, s, _input: (\n", - "# 's2',\n", - "# s['s2'] + param\n", - "# )\n", - "#\n", - "# s2m1 = sweep(\n", - "# params = [Decimal(11.0), Decimal(22.0)],\n", - "# sweep_f = fit_param\n", - "# )" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [], - "source": [ - "from decimal import Decimal\n", - "from itertools import product\n", - "\n", - "# def \n", - "\n", - "l = {\n", - " 's1': 1, \n", - " 's2': [Decimal('11'), Decimal('22')], \n", - " 's3': [Decimal('12'), Decimal('23')], \n", - " 's4': 10, \n", - " 'timestamp': '2018-10-01 15:16:25', \n", - " 'mech_step': 0, \n", - " 'time_step': 1\n", - "}\n", - "\n", - "def flattenDict(l):\n", - " def tupalize(k, vs):\n", - " l = []\n", - " if isinstance(vs, list):\n", - " for v in vs:\n", - " l.append((k, v))\n", - " else:\n", - " l.append((k, vs))\n", - " return l\n", - "\n", - " flat_list = [tupalize(k, vs) for k, vs in l.items()]\n", - " flat_dict = [dict(items) for items in product(*flat_list)]\n", - " return flat_dict" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[{'s1': 1,\n", - " 's2': Decimal('11'),\n", - " 's3': Decimal('12'),\n", - " 's4': 10,\n", - " 'timestamp': '2018-10-01 15:16:25',\n", - " 'mech_step': 0,\n", - " 'time_step': 1},\n", - " {'s1': 1,\n", - " 's2': Decimal('11'),\n", - " 's3': Decimal('23'),\n", - " 's4': 10,\n", - " 'timestamp': '2018-10-01 15:16:25',\n", - " 'mech_step': 0,\n", - " 'time_step': 1},\n", - " {'s1': 1,\n", - " 's2': Decimal('22'),\n", - " 's3': Decimal('12'),\n", - " 's4': 10,\n", - " 'timestamp': '2018-10-01 15:16:25',\n", - " 'mech_step': 0,\n", - " 'time_step': 1},\n", - " {'s1': 1,\n", - " 's2': Decimal('22'),\n", - " 's3': Decimal('23'),\n", - " 's4': 10,\n", - " 'timestamp': '2018-10-01 15:16:25',\n", - " 'mech_step': 0,\n", - " 'time_step': 1}]" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "flattenDict(l)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.5" - } - }, - "nbformat": 4, - "nbformat_minor": 1 -} diff --git a/requirements.txt b/requirements.txt index 0418284..48a300b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,3 @@ -wheel -pandas pathos fn tabulate \ No newline at end of file diff --git a/simulations/example_run.py b/simulations/example_run.py index b179e49..3a18378 100644 --- a/simulations/example_run.py +++ b/simulations/example_run.py @@ -1,27 +1,29 @@ import pandas as pd from tabulate import tabulate + # The following imports NEED to be in the exact order from SimCAD.engine import ExecutionMode, ExecutionContext, Executor -from simulations.validation import sweep_config, config1, config2 +from validation import config1, config2 from SimCAD import configs exec_mode = ExecutionMode() -# print("Simulation Execution 1") -# print() -# first_config = [configs[0]] # from config1 -# single_proc_ctx = ExecutionContext(context=exec_mode.single_proc) -# run1 = Executor(exec_context=single_proc_ctx, configs=first_config) -# run1_raw_result, tensor_field = run1.main() -# result = pd.DataFrame(run1_raw_result) -# print() -# print("Tensor Field:") -# print(tabulate(tensor_field, headers='keys', tablefmt='psql')) -# print("Output:") -# print(tabulate(result, headers='keys', tablefmt='psql')) -# print() +print("Simulation Execution 1") +print() +first_config = [configs[0]] # from config1 +single_proc_ctx = ExecutionContext(context=exec_mode.single_proc) +run1 = Executor(exec_context=single_proc_ctx, configs=first_config) +run1_raw_result, tensor_field = run1.main() +result = pd.DataFrame(run1_raw_result) +print() +print("Tensor Field:") +print(tabulate(tensor_field, headers='keys', tablefmt='psql')) +print("Output:") +print(tabulate(result, headers='keys', tablefmt='psql')) +print() -print("Simulation Execution 2: Concurrent Execution") +print("Simulation Execution 2: Pairwise Execution") +print() multi_proc_ctx = ExecutionContext(context=exec_mode.multi_proc) run2 = Executor(exec_context=multi_proc_ctx, configs=configs) for raw_result, tensor_field in run2.main(): diff --git a/simulations/validation/base_config1.py b/simulations/validation/base_config1.py new file mode 100644 index 0000000..3bf83ba --- /dev/null +++ b/simulations/validation/base_config1.py @@ -0,0 +1,171 @@ +from decimal import Decimal +import numpy as np +from datetime import timedelta + +from SimCAD import configs +from SimCAD.configuration import Configuration +from SimCAD.configuration.utils import exo_update_per_ts, proc_trigger, bound_norm_random, \ + ep_time_step + +seed = { + 'z': np.random.RandomState(1), + 'a': np.random.RandomState(2), + 'b': np.random.RandomState(3), + 'c': np.random.RandomState(3) +} + +# Behaviors per Mechanism +# Different return types per mechanism ?? *** No *** +def b1m1(step, sL, s): + return {'param1': 1} +def b2m1(step, sL, s): + return {'param1': 1} + +def b1m2(step, sL, s): + return {'param1': 1, 'param2': 2} +def b2m2(step, sL, s): + return {'param1': 1, 'param2': 4} + +def b1m3(step, sL, s): + return {'param1': 1, 'param2': np.array([10, 100])} +def b2m3(step, sL, s): + return {'param1': 1, 'param2': np.array([20, 200])} + +# deff not more than 2 +# Internal States per Mechanism +def s1m1(step, sL, s, _input): + y = 's1' + x = s['s1'] + _input['param1'] + return (y, x) +def s2m1(step, sL, s, _input): + y = 's2' + x = s['s2'] + _input['param1'] + return (y, x) + +def s1m2(step, sL, s, _input): + y = 's1' + x = s['s1'] + _input['param1'] + return (y, x) +def s2m2(step, sL, s, _input): + y = 's2' + x = s['s2'] + _input['param1'] + return (y, x) + +def s1m3(step, sL, s, _input): + y = 's1' + x = s['s1'] + _input['param1'] + return (y, x) +def s2m3(step, sL, s, _input): + y = 's2' + x = s['s2'] + _input['param1'] + return (y, x) + +# Exogenous States +proc_one_coef_A = 0.7 +proc_one_coef_B = 1.3 + +def es3p1(step, sL, s, _input): + y = 's3' + x = s['s3'] * bound_norm_random(seed['a'], proc_one_coef_A, proc_one_coef_B) + return (y, x) + +def es4p2(step, sL, s, _input): + y = 's4' + x = s['s4'] * bound_norm_random(seed['b'], proc_one_coef_A, proc_one_coef_B) + return (y, x) + +ts_format = '%Y-%m-%d %H:%M:%S' +t_delta = timedelta(days=0, minutes=0, seconds=1) +def es5p2(step, sL, s, _input): + y = 'timestamp' + x = ep_time_step(s, dt_str=s['timestamp'], fromat_str=ts_format, _timedelta=t_delta) + return (y, x) + + +# Environment States +def env_a(x): + return 10 +def env_b(x): + return 10 +# def what_ever(x): +# return x + 1 + +# Genesis States +genesis_states = { + 's1': Decimal(0.0), + 's2': Decimal(0.0), + 's3': Decimal(1.0), + 's4': Decimal(1.0), + 'timestamp': '2018-10-01 15:16:24' +} + +# remove `exo_update_per_ts` to update every ts +exogenous_states = exo_update_per_ts( + { + "s3": es3p1, + "s4": es4p2, + "timestamp": es5p2 + } +) + +# make env proc trigger field agnostic + +# ToDo: Bug - Can't use environments without proc_trigger. TypeError: 'int' object is not callable +# "/Users/jjodesty/Projects/DiffyQ-SimCAD/SimCAD/engine/simulation.py" +env_processes = { + # "s3": env_a, + # "s4": env_b + "s3": proc_trigger('2018-10-01 15:16:25', env_a), + "s4": proc_trigger('2018-10-01 15:16:25', env_b) +} + +# need at least 1 behaviour and 1 state function for the 1st mech with behaviors +# mechanisms = {} +mechanisms = { + "m1": { + "behaviors": { + "b1": b1m1, # lambda step, sL, s: s['s1'] + 1, + "b2": b2m1 + }, + "states": { # exclude only. TypeError: reduce() of empty sequence with no initial value + "s1": s1m1, + "s2": s2m1 + } + }, + "m2": { + "behaviors": { + "b1": b1m2, + "b2": b2m2 + }, + "states": { + "s1": s1m2, + "s2": s2m2 + } + }, + "m3": { + "behaviors": { + "b1": b1m3, + "b2": b2m3 + }, + "states": { + "s1": s1m3, + "s2": s2m3 + } + } +} + +sim_config = { + "N": 2, + "T": range(5) +} + +configs.append( + Configuration( + sim_config=sim_config, + state_dict=genesis_states, + seed=seed, + exogenous_states=exogenous_states, + env_processes=env_processes, + mechanisms=mechanisms + ) +) \ No newline at end of file diff --git a/simulations/validation/base_config2.py b/simulations/validation/base_config2.py new file mode 100644 index 0000000..6b9469e --- /dev/null +++ b/simulations/validation/base_config2.py @@ -0,0 +1,180 @@ +from decimal import Decimal +import numpy as np +from datetime import timedelta + +from SimCAD import configs +from SimCAD.configuration import Configuration +from SimCAD.configuration.utils import exo_update_per_ts, proc_trigger, bound_norm_random, \ + ep_time_step + + +seed = { + 'z': np.random.RandomState(1), + 'a': np.random.RandomState(2), + 'b': np.random.RandomState(3), + 'c': np.random.RandomState(3) +} + +# Behaviors per Mechanism +# Different return types per mechanism ?? *** No *** +def b1m1(step, sL, s): + return {'param1': 1} +def b2m1(step, sL, s): + return {'param2': 4} + +def b1m2(step, sL, s): + return {'param1': 'a', 'param2': 2} +def b2m2(step, sL, s): + return {'param1': 'b', 'param2': 4} + + +def b1m3(step, sL, s): + return {'param1': ['c'], 'param2': np.array([10, 100])} +def b2m3(step, sL, s): + return {'param1': ['d'], 'param2': np.array([20, 200])} + + +# Internal States per Mechanism +def s1m1(step, sL, s, _input): + y = 's1' + x = _input['param1'] + return (y, x) +def s2m1(step, sL, s, _input): + y = 's2' + x = _input['param2'] + return (y, x) + +def s1m2(step, sL, s, _input): + y = 's1' + x = _input['param1'] + return (y, x) +def s2m2(step, sL, s, _input): + y = 's2' + x = _input['param2'] + return (y, x) + +def s1m3(step, sL, s, _input): + y = 's1' + x = _input['param1'] + return (y, x) +def s2m3(step, sL, s, _input): + y = 's2' + x = _input['param2'] + return (y, x) + +# Exogenous States +proc_one_coef_A = 0.7 +proc_one_coef_B = 1.3 + +def es3p1(step, sL, s, _input): + y = 's3' + x = s['s3'] * bound_norm_random(seed['a'], proc_one_coef_A, proc_one_coef_B) + return (y, x) + +def es4p2(step, sL, s, _input): + y = 's4' + x = s['s4'] * bound_norm_random(seed['b'], proc_one_coef_A, proc_one_coef_B) + return (y, x) + +ts_format = '%Y-%m-%d %H:%M:%S' +t_delta = timedelta(days=0, minutes=0, seconds=1) +def es5p2(step, sL, s, _input): + y = 'timestamp' + x = ep_time_step(s, dt_str=s['timestamp'], fromat_str=ts_format, _timedelta=t_delta) + return (y, x) + + +# Environment States +def env_a(x): + return 10 +def env_b(x): + return 10 +# def what_ever(x): +# return x + 1 + +# Genesis States +genesis_states = { + 's1': Decimal(0.0), + 's2': Decimal(0.0), + 's3': Decimal(1.0), + 's4': Decimal(1.0), + 'timestamp': '2018-10-01 15:16:24' +} + +# remove `exo_update_per_ts` to update every ts +# why `exo_update_per_ts` here instead of `env_processes` +exogenous_states = exo_update_per_ts( + { + "s3": es3p1, + "s4": es4p2, + "timestamp": es5p2 + } +) + +# make env proc trigger field agnostic +env_processes = { + "s3": proc_trigger('2018-10-01 15:16:25', env_a), + "s4": proc_trigger('2018-10-01 15:16:25', env_b) +} + +# lambdas +# genesis Sites should always be there +# [1, 2] +# behavior_ops = [ foldr(_ + _), lambda x: x + 0 ] + + +# [1, 2] = {'b1': ['a'], 'b2', [1]} = +# behavior_ops = [behavior_to_dict, print_fwd, sum_dict_values] +# behavior_ops = [foldr(dict_elemwise_sum())] +# behavior_ops = [] + +# need at least 1 behaviour and 1 state function for the 1st mech with behaviors +# mechanisms = {} +mechanisms = { + "m1": { + "behaviors": { + "b1": b1m1, # lambda step, sL, s: s['s1'] + 1, + # "b2": b2m1 + }, + "states": { # exclude only. TypeError: reduce() of empty sequence with no initial value + "s1": s1m1, + # "s2": s2m1 + } + }, + "m2": { + "behaviors": { + "b1": b1m2, + # "b2": b2m2 + }, + "states": { + "s1": s1m2, + # "s2": s2m2 + } + }, + "m3": { + "behaviors": { + "b1": b1m3, + "b2": b2m3 + }, + "states": { + "s1": s1m3, + "s2": s2m3 + } + } +} + +sim_config = { + "N": 2, + "T": range(5) +} + +configs.append( + Configuration( + sim_config=sim_config, + state_dict=genesis_states, + seed=seed, + exogenous_states=exogenous_states, + env_processes=env_processes, + mechanisms=mechanisms + ) +) \ No newline at end of file diff --git a/simulations/validation/config1.py b/simulations/validation/config1.py index ce74604..a016842 100644 --- a/simulations/validation/config1.py +++ b/simulations/validation/config1.py @@ -2,12 +2,13 @@ from decimal import Decimal import numpy as np from datetime import timedelta -from SimCAD.configuration import append_configs -from SimCAD.configuration.utils import proc_trigger, bound_norm_random, ep_time_step -from SimCAD.configuration.utils.parameterSweep import config_sim +from SimCAD import configs +from SimCAD.configuration import Configuration +from SimCAD.configuration.utils import exo_update_per_ts, proc_trigger, bound_norm_random, \ + ep_time_step -seeds = { +seed = { 'z': np.random.RandomState(1), 'a': np.random.RandomState(2), 'b': np.random.RandomState(3), @@ -15,47 +16,47 @@ seeds = { } -# Policies per Mechanism -def p1m1(_g, step, sL, s): +# Behaviors per Mechanism +def b1m1(step, sL, s): return {'param1': 1} -def p2m1(_g, step, sL, s): +def b2m1(step, sL, s): return {'param2': 4} -def p1m2(_g, step, sL, s): +def b1m2(step, sL, s): return {'param1': 'a', 'param2': 2} -def p2m2(_g, step, sL, s): +def b2m2(step, sL, s): return {'param1': 'b', 'param2': 4} -def p1m3(_g, step, sL, s): +def b1m3(step, sL, s): return {'param1': ['c'], 'param2': np.array([10, 100])} -def p2m3(_g, step, sL, s): +def b2m3(step, sL, s): return {'param1': ['d'], 'param2': np.array([20, 200])} # Internal States per Mechanism -def s1m1(_g, step, sL, s, _input): +def s1m1(step, sL, s, _input): y = 's1' x = _input['param1'] return (y, x) -def s2m1(_g, step, sL, s, _input): +def s2m1(step, sL, s, _input): y = 's2' x = _input['param2'] return (y, x) -def s1m2(_g, step, sL, s, _input): +def s1m2(step, sL, s, _input): y = 's1' x = _input['param1'] return (y, x) -def s2m2(_g, step, sL, s, _input): +def s2m2(step, sL, s, _input): y = 's2' x = _input['param2'] return (y, x) -def s1m3(_g, step, sL, s, _input): +def s1m3(step, sL, s, _input): y = 's1' x = _input['param1'] return (y, x) -def s2m3(_g, step, sL, s, _input): +def s2m3(step, sL, s, _input): y = 's2' x = _input['param2'] return (y, x) @@ -65,19 +66,19 @@ def s2m3(_g, step, sL, s, _input): proc_one_coef_A = 0.7 proc_one_coef_B = 1.3 -def es3p1(_g, step, sL, s, _input): +def es3p1(step, sL, s, _input): y = 's3' - x = s['s3'] * bound_norm_random(seeds['a'], proc_one_coef_A, proc_one_coef_B) + x = s['s3'] * bound_norm_random(seed['a'], proc_one_coef_A, proc_one_coef_B) return (y, x) -def es4p2(_g, step, sL, s, _input): +def es4p2(step, sL, s, _input): y = 's4' - x = s['s4'] * bound_norm_random(seeds['b'], proc_one_coef_A, proc_one_coef_B) + x = s['s4'] * bound_norm_random(seed['b'], proc_one_coef_A, proc_one_coef_B) return (y, x) ts_format = '%Y-%m-%d %H:%M:%S' t_delta = timedelta(days=0, minutes=0, seconds=1) -def es5p2(_g, step, sL, s, _input): +def es5p2(step, sL, s, _input): y = 'timestamp' x = ep_time_step(s, dt_str=s['timestamp'], fromat_str=ts_format, _timedelta=t_delta) return (y, x) @@ -102,11 +103,14 @@ genesis_states = { } -raw_exogenous_states = { +# remove `exo_update_per_ts` to update every ts +exogenous_states = exo_update_per_ts( + { "s3": es3p1, "s4": es4p2, "timestamp": es5p2 -} + } +) env_processes = { @@ -115,11 +119,11 @@ env_processes = { } -partial_state_update_block = { +mechanisms = { "m1": { - "policies": { - "b1": p1m1, - "b2": p2m1 + "behaviors": { + "b1": b1m1, + "b2": b2m1 }, "states": { "s1": s1m1, @@ -127,9 +131,9 @@ partial_state_update_block = { } }, "m2": { - "policies": { - "b1": p1m2, - "b2": p2m2 + "behaviors": { + "b1": b1m2, + "b2": b2m2 }, "states": { "s1": s1m2, @@ -137,9 +141,9 @@ partial_state_update_block = { } }, "m3": { - "policies": { - "b1": p1m3, - "b2": p2m3 + "behaviors": { + "b1": b1m3, + "b2": b2m3 }, "states": { "s1": s1m3, @@ -149,19 +153,19 @@ partial_state_update_block = { } -sim_config = config_sim( - { - "N": 2, - "T": range(5), - } -) +sim_config = { + "N": 2, + "T": range(5) +} -append_configs( - sim_configs=sim_config, - initial_state=genesis_states, - seeds=seeds, - raw_exogenous_states=raw_exogenous_states, - env_processes=env_processes, - partial_state_updates=partial_state_update_block +configs.append( + Configuration( + sim_config=sim_config, + state_dict=genesis_states, + seed=seed, + exogenous_states=exogenous_states, + env_processes=env_processes, + mechanisms=mechanisms + ) ) diff --git a/simulations/validation/config2.py b/simulations/validation/config2.py index 987e8c0..703dace 100644 --- a/simulations/validation/config2.py +++ b/simulations/validation/config2.py @@ -2,11 +2,13 @@ from decimal import Decimal import numpy as np from datetime import timedelta -from SimCAD.configuration import append_configs -from SimCAD.configuration.utils import proc_trigger, bound_norm_random, ep_time_step -from SimCAD.configuration.utils.parameterSweep import config_sim +from SimCAD import configs +from SimCAD.configuration import Configuration +from SimCAD.configuration.utils import exo_update_per_ts, proc_trigger, bound_norm_random, \ + ep_time_step -seeds = { + +seed = { 'z': np.random.RandomState(1), 'a': np.random.RandomState(2), 'b': np.random.RandomState(3), @@ -14,47 +16,47 @@ seeds = { } -# Policies per Mechanism -def p1m1(_g, step, sL, s): +# Behaviors per Mechanism +def b1m1(step, sL, s): return {'param1': 1} -def p2m1(_g, step, sL, s): +def b2m1(step, sL, s): return {'param2': 4} -def p1m2(_g, step, sL, s): +def b1m2(step, sL, s): return {'param1': 'a', 'param2': 2} -def p2m2(_g, step, sL, s): +def b2m2(step, sL, s): return {'param1': 'b', 'param2': 4} -def p1m3(_g, step, sL, s): +def b1m3(step, sL, s): return {'param1': ['c'], 'param2': np.array([10, 100])} -def p2m3(_g, step, sL, s): +def b2m3(step, sL, s): return {'param1': ['d'], 'param2': np.array([20, 200])} # Internal States per Mechanism -def s1m1(_g, step, sL, s, _input): +def s1m1(step, sL, s, _input): y = 's1' x = _input['param1'] return (y, x) -def s2m1(_g, step, sL, s, _input): +def s2m1(step, sL, s, _input): y = 's2' x = _input['param2'] return (y, x) -def s1m2(_g, step, sL, s, _input): +def s1m2(step, sL, s, _input): y = 's1' x = _input['param1'] return (y, x) -def s2m2(_g, step, sL, s, _input): +def s2m2(step, sL, s, _input): y = 's2' x = _input['param2'] return (y, x) -def s1m3(_g, step, sL, s, _input): +def s1m3(step, sL, s, _input): y = 's1' x = _input['param1'] return (y, x) -def s2m3(_g, step, sL, s, _input): +def s2m3(step, sL, s, _input): y = 's2' x = _input['param2'] return (y, x) @@ -64,19 +66,19 @@ def s2m3(_g, step, sL, s, _input): proc_one_coef_A = 0.7 proc_one_coef_B = 1.3 -def es3p1(_g, step, sL, s, _input): +def es3p1(step, sL, s, _input): y = 's3' - x = s['s3'] * bound_norm_random(seeds['a'], proc_one_coef_A, proc_one_coef_B) + x = s['s3'] * bound_norm_random(seed['a'], proc_one_coef_A, proc_one_coef_B) return (y, x) -def es4p2(_g, step, sL, s, _input): +def es4p2(step, sL, s, _input): y = 's4' - x = s['s4'] * bound_norm_random(seeds['b'], proc_one_coef_A, proc_one_coef_B) + x = s['s4'] * bound_norm_random(seed['b'], proc_one_coef_A, proc_one_coef_B) return (y, x) ts_format = '%Y-%m-%d %H:%M:%S' t_delta = timedelta(days=0, minutes=0, seconds=1) -def es5p2(_g, step, sL, s, _input): +def es5p2(step, sL, s, _input): y = 'timestamp' x = ep_time_step(s, dt_str=s['timestamp'], fromat_str=ts_format, _timedelta=t_delta) return (y, x) @@ -101,11 +103,14 @@ genesis_states = { } -raw_exogenous_states = { +# remove `exo_update_per_ts` to update every ts +exogenous_states = exo_update_per_ts( + { "s3": es3p1, "s4": es4p2, "timestamp": es5p2 -} + } +) env_processes = { @@ -114,11 +119,11 @@ env_processes = { } -partial_state_update_block = { +mechanisms = { "m1": { - "policies": { - "b1": p1m1, - # "b2": p2m1 + "behaviors": { + "b1": b1m1, + # "b2": b2m1 }, "states": { "s1": s1m1, @@ -126,9 +131,9 @@ partial_state_update_block = { } }, "m2": { - "policies": { - "b1": p1m2, - # "b2": p2m2 + "behaviors": { + "b1": b1m2, + # "b2": b2m2 }, "states": { "s1": s1m2, @@ -136,9 +141,9 @@ partial_state_update_block = { } }, "m3": { - "policies": { - "b1": p1m3, - "b2": p2m3 + "behaviors": { + "b1": b1m3, + "b2": b2m3 }, "states": { "s1": s1m3, @@ -148,19 +153,19 @@ partial_state_update_block = { } -sim_config = config_sim( - { - "N": 2, - "T": range(5), - } -) +sim_config = { + "N": 2, + "T": range(5) +} -append_configs( - sim_configs=sim_config, - initial_state=genesis_states, - seeds=seeds, - raw_exogenous_states=raw_exogenous_states, - env_processes=env_processes, - partial_state_updates=partial_state_update_block +configs.append( + Configuration( + sim_config=sim_config, + state_dict=genesis_states, + seed=seed, + exogenous_states=exogenous_states, + env_processes=env_processes, + mechanisms=mechanisms + ) ) diff --git a/simulations/validation/config_1.py b/simulations/validation/config_1.py new file mode 100644 index 0000000..7bdd5ac --- /dev/null +++ b/simulations/validation/config_1.py @@ -0,0 +1,178 @@ +from decimal import Decimal +import numpy as np +from datetime import timedelta + +from SimCAD import configs +from SimCAD.configuration import Configuration +from SimCAD.configuration.utils import exo_update_per_ts, proc_trigger, bound_norm_random, \ + ep_time_step + +seed = { + 'z': np.random.RandomState(1), + 'a': np.random.RandomState(2), + 'b': np.random.RandomState(3), + 'c': np.random.RandomState(3) +} + +# Behaviors per Mechanism +# Different return types per mechanism ?? *** No *** +def b1m1(step, sL, s): + return {'param1': 1} +def b2m1(step, sL, s): + return {'param2': 4} + +def b1m2(step, sL, s): + return {'param1': 'a', 'param2': 2} +def b2m2(step, sL, s): + return {'param1': 'b', 'param2': 4} + +def b1m3(step, sL, s): + return {'param1': ['c'], 'param2': np.array([10, 100])} +def b2m3(step, sL, s): + return {'param1': ['d'], 'param2': np.array([20, 200])} + +# deff not more than 2 +# Internal States per Mechanism +def s1m1(step, sL, s, _input): + y = 's1' + x = _input['param1'] #+ [Coef1 x 5] + return (y, x) +def s2m1(step, sL, s, _input): + y = 's2' + x = _input['param2'] #+ [Coef2 x 5] + return (y, x) + +def s1m2(step, sL, s, _input): + y = 's1' + x = _input['param1'] + return (y, x) +def s2m2(step, sL, s, _input): + y = 's2' + x = _input['param2'] + return (y, x) + +def s1m3(step, sL, s, _input): + y = 's1' + x = _input['param1'] + return (y, x) +def s2m3(step, sL, s, _input): + y = 's2' + x = _input['param2'] + return (y, x) + +# Exogenous States +proc_one_coef_A = 0.7 +proc_one_coef_B = 1.3 + +def es3p1(step, sL, s, _input): + y = 's3' + x = s['s3'] * bound_norm_random(seed['a'], proc_one_coef_A, proc_one_coef_B) + return (y, x) + +def es4p2(step, sL, s, _input): + y = 's4' + x = s['s4'] * bound_norm_random(seed['b'], proc_one_coef_A, proc_one_coef_B) + return (y, x) + +ts_format = '%Y-%m-%d %H:%M:%S' +t_delta = timedelta(days=0, minutes=0, seconds=1) +def es5p2(step, sL, s, _input): + y = 'timestamp' + x = ep_time_step(s, dt_str=s['timestamp'], fromat_str=ts_format, _timedelta=t_delta) + return (y, x) + + +# Environment States +def env_a(x): + return 5 +def env_b(x): + return 10 +# def what_ever(x): +# return x + 1 + +# Genesis States +genesis_states = { + 's1': Decimal(0.0), + 's2': Decimal(0.0), + 's3': Decimal(1.0), + 's4': Decimal(1.0), + 'timestamp': '2018-10-01 15:16:24' +} + +# remove `exo_update_per_ts` to update every ts +exogenous_states = exo_update_per_ts( + { + "s3": es3p1, + "s4": es4p2, + "timestamp": es5p2 + } +) + +# ToDo: make env proc trigger field agnostic +# ToDo: input json into function renaming __name__ +env_processes = { + "s3": env_a, + "s4": proc_trigger('2018-10-01 15:16:25', env_b) +} + +# lambdas +# genesis Sites should always be there +# [1, 2] +# behavior_ops = [ foldr(_ + _), lambda x: x + 0 ] + +# [1, 2] = {'b1': ['a'], 'b2', [1]} = +# behavior_ops = [ behavior_to_dict, print_fwd, sum_dict_values ] +# behavior_ops = [foldr(dict_elemwise_sum())] +# behavior_ops = [foldr(lambda a, b: a + b)] + +# need at least 1 behaviour and 1 state function for the 1st mech with behaviors +# mechanisms = {} + +mechanisms = { + "m1": { + "behaviors": { + "b1": b1m1, # lambda step, sL, s: s['s1'] + 1, + "b2": b2m1 + }, + "states": { # exclude only. TypeError: reduce() of empty sequence with no initial value + "s1": s1m1, + "s2": s2m1 + } + }, + "m2": { + "behaviors": { + "b1": b1m2, + "b2": b2m2 + }, + "states": { + "s1": s1m2, + "s2": s2m2 + } + }, + "m3": { + "behaviors": { + "b1": b1m3, + "b2": b2m3 + }, + "states": { + "s1": s1m3, + "s2": s2m3 + } + } +} + +sim_config = { + "N": 2, + "T": range(5) +} + +configs.append( + Configuration( + sim_config=sim_config, + state_dict=genesis_states, + seed=seed, + exogenous_states=exogenous_states, + env_processes=env_processes, + mechanisms=mechanisms + ) +) \ No newline at end of file diff --git a/simulations/validation/config_2.py b/simulations/validation/config_2.py new file mode 100644 index 0000000..6b9469e --- /dev/null +++ b/simulations/validation/config_2.py @@ -0,0 +1,180 @@ +from decimal import Decimal +import numpy as np +from datetime import timedelta + +from SimCAD import configs +from SimCAD.configuration import Configuration +from SimCAD.configuration.utils import exo_update_per_ts, proc_trigger, bound_norm_random, \ + ep_time_step + + +seed = { + 'z': np.random.RandomState(1), + 'a': np.random.RandomState(2), + 'b': np.random.RandomState(3), + 'c': np.random.RandomState(3) +} + +# Behaviors per Mechanism +# Different return types per mechanism ?? *** No *** +def b1m1(step, sL, s): + return {'param1': 1} +def b2m1(step, sL, s): + return {'param2': 4} + +def b1m2(step, sL, s): + return {'param1': 'a', 'param2': 2} +def b2m2(step, sL, s): + return {'param1': 'b', 'param2': 4} + + +def b1m3(step, sL, s): + return {'param1': ['c'], 'param2': np.array([10, 100])} +def b2m3(step, sL, s): + return {'param1': ['d'], 'param2': np.array([20, 200])} + + +# Internal States per Mechanism +def s1m1(step, sL, s, _input): + y = 's1' + x = _input['param1'] + return (y, x) +def s2m1(step, sL, s, _input): + y = 's2' + x = _input['param2'] + return (y, x) + +def s1m2(step, sL, s, _input): + y = 's1' + x = _input['param1'] + return (y, x) +def s2m2(step, sL, s, _input): + y = 's2' + x = _input['param2'] + return (y, x) + +def s1m3(step, sL, s, _input): + y = 's1' + x = _input['param1'] + return (y, x) +def s2m3(step, sL, s, _input): + y = 's2' + x = _input['param2'] + return (y, x) + +# Exogenous States +proc_one_coef_A = 0.7 +proc_one_coef_B = 1.3 + +def es3p1(step, sL, s, _input): + y = 's3' + x = s['s3'] * bound_norm_random(seed['a'], proc_one_coef_A, proc_one_coef_B) + return (y, x) + +def es4p2(step, sL, s, _input): + y = 's4' + x = s['s4'] * bound_norm_random(seed['b'], proc_one_coef_A, proc_one_coef_B) + return (y, x) + +ts_format = '%Y-%m-%d %H:%M:%S' +t_delta = timedelta(days=0, minutes=0, seconds=1) +def es5p2(step, sL, s, _input): + y = 'timestamp' + x = ep_time_step(s, dt_str=s['timestamp'], fromat_str=ts_format, _timedelta=t_delta) + return (y, x) + + +# Environment States +def env_a(x): + return 10 +def env_b(x): + return 10 +# def what_ever(x): +# return x + 1 + +# Genesis States +genesis_states = { + 's1': Decimal(0.0), + 's2': Decimal(0.0), + 's3': Decimal(1.0), + 's4': Decimal(1.0), + 'timestamp': '2018-10-01 15:16:24' +} + +# remove `exo_update_per_ts` to update every ts +# why `exo_update_per_ts` here instead of `env_processes` +exogenous_states = exo_update_per_ts( + { + "s3": es3p1, + "s4": es4p2, + "timestamp": es5p2 + } +) + +# make env proc trigger field agnostic +env_processes = { + "s3": proc_trigger('2018-10-01 15:16:25', env_a), + "s4": proc_trigger('2018-10-01 15:16:25', env_b) +} + +# lambdas +# genesis Sites should always be there +# [1, 2] +# behavior_ops = [ foldr(_ + _), lambda x: x + 0 ] + + +# [1, 2] = {'b1': ['a'], 'b2', [1]} = +# behavior_ops = [behavior_to_dict, print_fwd, sum_dict_values] +# behavior_ops = [foldr(dict_elemwise_sum())] +# behavior_ops = [] + +# need at least 1 behaviour and 1 state function for the 1st mech with behaviors +# mechanisms = {} +mechanisms = { + "m1": { + "behaviors": { + "b1": b1m1, # lambda step, sL, s: s['s1'] + 1, + # "b2": b2m1 + }, + "states": { # exclude only. TypeError: reduce() of empty sequence with no initial value + "s1": s1m1, + # "s2": s2m1 + } + }, + "m2": { + "behaviors": { + "b1": b1m2, + # "b2": b2m2 + }, + "states": { + "s1": s1m2, + # "s2": s2m2 + } + }, + "m3": { + "behaviors": { + "b1": b1m3, + "b2": b2m3 + }, + "states": { + "s1": s1m3, + "s2": s2m3 + } + } +} + +sim_config = { + "N": 2, + "T": range(5) +} + +configs.append( + Configuration( + sim_config=sim_config, + state_dict=genesis_states, + seed=seed, + exogenous_states=exogenous_states, + env_processes=env_processes, + mechanisms=mechanisms + ) +) \ No newline at end of file diff --git a/simulations/validation/sweep_config.py b/simulations/validation/sweep_config.py deleted file mode 100644 index c93a703..0000000 --- a/simulations/validation/sweep_config.py +++ /dev/null @@ -1,196 +0,0 @@ -from decimal import Decimal -import numpy as np -from datetime import timedelta -import pprint - -from SimCAD.configuration import append_configs -from SimCAD.configuration.utils import proc_trigger, ep_time_step -from SimCAD.configuration.utils.parameterSweep import config_sim - -pp = pprint.PrettyPrinter(indent=4) - -seeds = { - 'z': np.random.RandomState(1), - 'a': np.random.RandomState(2), - 'b': np.random.RandomState(3), - 'c': np.random.RandomState(3) -} - - -g = { - 'alpha': [1], - 'beta': [2, 5], - 'gamma': [3, 4], - 'omega': [7] -} - -# Policies per Mechanism -def p1m1(_g, step, sL, s): - return {'param1': 1} - -def p2m1(_g, step, sL, s): - return {'param2': 4} - -def p1m2(_g, step, sL, s): - return {'param1': 'a', 'param2': _g['beta']} - -def p2m2(_g, step, sL, s): - return {'param1': 'b', 'param2': 0} - -def p1m3(_g, step, sL, s): - return {'param1': np.array([10, 100])} - -def p2m3(_g, step, sL, s): - return {'param1': np.array([20, 200])} - -# Internal States per Mechanism -def s1m1(_g, step, sL, s, _input): - y = 's1' - x = 0 - return (y, x) - -def s2m1(_g, step, sL, s, _input): - y = 's2' - x = _g['beta'] - return (y, x) - -def s1m2(_g, step, sL, s, _input): - y = 's1' - x = _input['param2'] - return (y, x) - -def s2m2(_g, step, sL, s, _input): - y = 's2' - x = _input['param2'] - return (y, x) - -def s1m3(_g, step, sL, s, _input): - y = 's1' - x = 0 - return (y, x) - -def s2m3(_g, step, sL, s, _input): - y = 's2' - x = 0 - return (y, x) - - -# Exogenous States -proc_one_coef_A = 0.7 -proc_one_coef_B = 1.3 - - -def es3p1(_g, step, sL, s, _input): - y = 's3' - x = _g['gamma'] - return (y, x) -# @curried -def es4p2(_g, step, sL, s, _input): - y = 's4' - x = _g['gamma'] - return (y, x) - -ts_format = '%Y-%m-%d %H:%M:%S' -t_delta = timedelta(days=0, minutes=0, seconds=1) -def es5p2(_g, step, sL, s, _input): - y = 'timestamp' - x = ep_time_step(s, dt_str=s['timestamp'], fromat_str=ts_format, _timedelta=t_delta) - return (y, x) - - -# Environment States -# @curried -# def env_a(param, x): -# return x + param -def env_a(x): - return x -def env_b(x): - return 10 - - -# Genesis States -genesis_states = { - 's1': Decimal(0.0), - 's2': Decimal(0.0), - 's3': Decimal(1.0), - 's4': Decimal(1.0), - 'timestamp': '2018-10-01 15:16:24' -} - - -# remove `exo_update_per_ts` to update every ts -raw_exogenous_states = { - "s3": es3p1, - "s4": es4p2, - "timestamp": es5p2 -} - - -# ToDo: make env proc trigger field agnostic -# ToDo: input json into function renaming __name__ -triggered_env_b = proc_trigger('2018-10-01 15:16:25', env_b) -env_processes = { - "s3": env_a, #sweep(beta, env_a), - "s4": triggered_env_b #rename('parameterized', triggered_env_b) #sweep(beta, triggered_env_b) -} -# parameterized_env_processes = parameterize_states(env_processes) -# -# pp.pprint(parameterized_env_processes) -# exit() - -# ToDo: The number of values entered in sweep should be the # of config objs created, -# not dependent on the # of times the sweep is applied -# sweep exo_state func and point to exo-state in every other funtion -# param sweep on genesis states - -partial_state_update_block = { - "m1": { - "policies": { - "b1": p1m1, - "b2": p2m1 - }, - "states": { - "s1": s1m1, - "s2": s2m1 - } - }, - "m2": { - "policies": { - "b1": p1m2, - "b2": p2m2, - }, - "states": { - "s1": s1m2, - "s2": s2m2 - } - }, - "m3": { - "policies": { - "b1": p1m3, - "b2": p2m3 - }, - "states": { - "s1": s1m3, - "s2": s2m3 - } - } -} - - -sim_config = config_sim( - { - "N": 2, - "T": range(5), - "M": g - } -) - - -append_configs( - sim_configs=sim_config, - initial_state=genesis_states, - seeds=seeds, - raw_exogenous_states=raw_exogenous_states, - env_processes=env_processes, - partial_state_updates=partial_state_update_block -) \ No newline at end of file