udo dirty publish to 'side branch'

cadCAD/configuration/utils/__init__.py

@@ -1,7 +1,10 @@
 from datetime import datetime, timedelta
 from decimal import Decimal
 from copy import deepcopy
+from functools import reduce
+
 from fn.func import curried
+from funcy import curry
 import pandas as pd
 
 # Temporary
@@ -39,7 +42,7 @@ def bound_norm_random(rng, low, high):
 
 
 @curried
-def proc_trigger(trigger_time, update_f, time):
+def env_proc_trigger(trigger_time, update_f, time):
     if time == trigger_time:
         return update_f
     else:
@@ -48,14 +51,17 @@ def proc_trigger(trigger_time, update_f, time):
 
 tstep_delta = timedelta(days=0, minutes=0, seconds=30)
 def time_step(dt_str, dt_format='%Y-%m-%d %H:%M:%S', _timedelta = tstep_delta):
+    # print(dt_str)
     dt = datetime.strptime(dt_str, dt_format)
     t = dt + _timedelta
     return t.strftime(dt_format)
 
 
+# ToDo: Inject in first elem of last PSUB from Historical state
 ep_t_delta = timedelta(days=0, minutes=0, seconds=1)
-def ep_time_step(s, dt_str, fromat_str='%Y-%m-%d %H:%M:%S', _timedelta = ep_t_delta):
+def ep_time_step(s_condition, dt_str, fromat_str='%Y-%m-%d %H:%M:%S', _timedelta = ep_t_delta):
-    if s['substep'] == 0:
+    # print(dt_str)
+    if s_condition:
         return time_step(dt_str, fromat_str, _timedelta)
     else:
         return dt_str
@@ -124,6 +130,28 @@ def exo_update_per_ts(ep):
 
     return {es: ep_decorator(f, es) for es, f in ep.items()}
 
+def trigger_condition(s, conditions, cond_opp):
+    condition_bools = [s[field] in precondition_values for field, precondition_values in conditions.items()]
+    return reduce(cond_opp, condition_bools)
+
+def apply_state_condition(conditions, cond_opp, y, f, _g, step, sL, s, _input):
+    if trigger_condition(s, conditions, cond_opp):
+        return f(_g, step, sL, s, _input)
+    else:
+        return y, s[y]
+
+def proc_trigger(y, f, conditions, cond_op):
+    return lambda _g, step, sL, s, _input: apply_state_condition(conditions, cond_op, y, f, _g, step, sL, s, _input)
+
+
+def timestep_trigger(end_substep, y, f):
+    conditions = {'substep': [0, end_substep]}
+    cond_opp = lambda a, b: a and b
+    return proc_trigger(y, f, conditions, cond_opp)
+
+# trigger = curry(_trigger)
+# print(timestep_trigger)
+
 
 # param sweep enabling middleware
 def config_sim(d):
@@ -141,3 +169,19 @@ def config_sim(d):
     else:
         d["M"] = [{}]
         return d
+
+
+def psub(policies, state_updates):
+    return {
+        'policies': policies,
+        'states': state_updates
+    }
+
+def genereate_psubs(policy_grid, states_grid, policies, state_updates):
+    PSUBS = []
+    for policy_ids, state_list in zip(policy_grid, states_grid):
+        filtered_policies = {k: v for (k, v) in policies.items() if k in policy_ids}
+        filtered_state_updates = {k: v for (k, v) in state_updates.items() if k in state_list}
+        PSUBS.append(psub(filtered_policies, filtered_state_updates))
+
+    return PSUBS

cadCAD/engine/simulation.py

@@ -107,7 +107,7 @@ class Executor:
         last_in_obj: Dict[str, Any] = sL[-1]
         # last_in_obj: Dict[str, Any] = sH[-1]
         # print(last_in_obj)
-        print(sH[-1])
+        # print(sH[-1])
 
         _input: Dict[str, Any] = self.policy_update_exception(self.get_policy_input(var_dict, sub_step, sH, last_in_obj, policy_funcs))
 
@@ -125,6 +125,7 @@ class Executor:
             return destination
 
         last_in_copy: Dict[str, Any] = transfer_missing_fields(last_in_obj, dict(generate_record(state_funcs)))
+        # ToDo: Remove
         last_in_copy: Dict[str, Any] = self.apply_env_proc(env_processes, last_in_copy, last_in_copy['timestep'])
         # ToDo: make 'substep' & 'timestep' reserve fields
         last_in_copy['substep'], last_in_copy['timestep'], last_in_copy['run'] = sub_step, time_step, run
@@ -152,7 +153,8 @@ class Executor:
         # states_list_copy: List[Dict[str, Any]] = deepcopy(states_list)
         # states_list_copy: List[Dict[str, Any]] = states_list
         # ToDo: flatten first
-        states_list_copy: List[Dict[str, Any]] = simulation_list[-1]
+        # states_list_copy: List[Dict[str, Any]] = simulation_list[-1]
+        states_list_copy: List[Dict[str, Any]] = deepcopy(simulation_list[-1])
         # print(states_list_copy)
 
         # ToDo: Causes Substep repeats in sL:
@@ -169,7 +171,9 @@ class Executor:
 
         # ToDo: Causes Substep repeats in sL, use for yield
         sub_step += 1
+
         for [s_conf, p_conf] in configs: # tensor field
+
             states_list: List[Dict[str, Any]] = self.partial_state_update(
                 var_dict, sub_step, states_list, simulation_list, s_conf, p_conf, env_processes, time_step, run
             )
@@ -177,6 +181,7 @@ class Executor:
             # print(simulation_list)
             # print(flatten(simulation_list))
             sub_step += 1
+            # print(sub_step)
 
         time_step += 1
 

cadCAD/utils/sys_config.py

@@ -1,5 +1,7 @@
-from cadCAD.configuration.utils import ep_time_step
+from cadCAD.configuration.utils import ep_time_step, time_step
-
+from funcy import curry
+# from fn import _
+from functools import reduce
 
 def increment(y, incr_by):
     return lambda _g, step, sL, s, _input: (y, s[y] + incr_by)
@@ -19,12 +21,106 @@ def update_timestamp(y, timedelta, format):
         ep_time_step(s, dt_str=s[y], fromat_str=format, _timedelta=timedelta)
     )
 
-def add(y, x):
+
-    return lambda _g, step, sH, s, _input: (y, s[y] + x)
+def apply(f, y: str, incr_by: int):
+    return lambda _g, step, sL, s, _input: (y, curry(f)(s[y])(incr_by))
+
+def add(y: str, incr_by: int):
+    return apply(lambda a, b: a + b, y, incr_by)
+
+def increment_state_by_int(y: str, incr_by: int):
+    return lambda _g, step, sL, s, _input: (y, s[y] + incr_by)
 
 def s(y, x):
     return lambda _g, step, sH, s, _input: (y, x)
 
+def time_model(y, substeps, time_delta, ts_format='%Y-%m-%d %H:%M:%S'):
+    def apply_incriment_condition(s):
+        if s['substep'] == 0 or s['substep'] == substeps:
+            return y, time_step(dt_str=s[y], dt_format=ts_format, _timedelta=time_delta)
+        else:
+            return y, s[y]
+    return lambda _g, step, sL, s, _input: apply_incriment_condition(s)
+
+
+# ToDo: Impliment Matrix reduction
+#
+# [
+#     {'conditions': [123], 'opp': lambda a, b: a and b},
+#     {'conditions': [123], 'opp': lambda a, b: a and b}
+# ]
+
+# def trigger_condition2(s, conditions, cond_opp):
+#     # print(conditions)
+#     condition_bools = [s[field] in precondition_values for field, precondition_values in conditions.items()]
+#     return reduce(cond_opp, condition_bools)
+#
+# def trigger_multi_conditions(s, multi_conditions, multi_cond_opp):
+#     # print([(d['conditions'], d['reduction_opp']) for d in multi_conditions])
+#     condition_bools = [
+#         trigger_condition2(s, conditions, opp) for conditions, opp in [
+#             (d['conditions'], d['reduction_opp']) for d in multi_conditions
+#         ]
+#     ]
+#     return reduce(multi_cond_opp, condition_bools)
+#
+# def apply_state_condition2(multi_conditions, multi_cond_opp, y, f, _g, step, sL, s, _input):
+#     if trigger_multi_conditions(s, multi_conditions, multi_cond_opp):
+#         return f(_g, step, sL, s, _input)
+#     else:
+#         return y, s[y]
+#
+# def proc_trigger2(y, f, multi_conditions, multi_cond_opp):
+#     return lambda _g, step, sL, s, _input: apply_state_condition2(multi_conditions, multi_cond_opp, y, f, _g, step, sL, s, _input)
+#
+# def timestep_trigger2(end_substep, y, f):
+#     multi_conditions = [
+#         {
+#             'condition': {
+#                 'substep': [0, end_substep]
+#             },
+#             'reduction_opp': lambda a, b: a and b
+#         }
+#     ]
+#     multi_cond_opp = lambda a, b: a and b
+#     return proc_trigger2(y, f, multi_conditions, multi_cond_opp)
+
+
+
+def env_trigger(trigger_field, trigger_val, input, funct_list):
+    y, x = input
+    if trigger_field == trigger_val:
+        i = 0
+        for g in funct_list:
+            x = g(x)
+    return y, x
+
+
+
+
+# def p1m1(_g, step, sL, s):
+#     return {'param1': 1}
+#
+# def apply_policy_condition(policies, policy_id, f, conditions, _g, step, sL, s):
+#     if trigger_condition(s, conditions):
+#         policies[policy_id] = f(_g, step, sL, s)
+#         return policies
+#     else:
+#         return policies
+#
+# def proc_trigger2(policies, conditions, policy_id, f):
+#     return lambda _g, step, sL, s: apply_policy_condition(policies, policy_id, f, conditions,_g, step, sL, s)
+
+# policies_updates = {"p1": udo_policyA, "p2": udo_policyB}
+
+
+# @curried
+# def proc_trigger(trigger_time, update_f, time):
+#     if time == trigger_time:
+#         return update_f
+#     else:
+#         return lambda x: x
+
 
 # def repr(_g, step, sL, s, _input):
 #     y = 'z'

simulations/historical_state_access_run.py

@@ -0,0 +1,30 @@
+import pandas as pd
+from tabulate import tabulate
+# The following imports NEED to be in the exact order
+from cadCAD.engine import ExecutionMode, ExecutionContext, Executor
+from simulations.validation import historical_state_access
+from cadCAD import configs
+
+exec_mode = ExecutionMode()
+
+print("Simulation Execution: Single Configuration")
+print()
+first_config = configs # only contains config1
+single_proc_ctx = ExecutionContext(context=exec_mode.single_proc)
+run = Executor(exec_context=single_proc_ctx, configs=first_config)
+
+raw_result, tensor_field = run.main()
+result = pd.DataFrame(raw_result)
+def delSH(d):
+    print(d)
+    if 'sh' in d.keys():
+        del d['sh']
+    return d
+result['sh'] = result['sh'].apply(lambda sh: list(map(lambda d: delSH(d), sh)))
+
+print()
+print("Tensor Field: config1")
+print(tabulate(tensor_field, headers='keys', tablefmt='psql'))
+print("Output:")
+print(tabulate(result, headers='keys', tablefmt='psql'))
+print()

simulations/param_sweep_test.py

@@ -0,0 +1,24 @@
+import pandas as pd
+from tabulate import tabulate
+# The following imports NEED to be in the exact order
+from cadCAD.engine import ExecutionMode, ExecutionContext, Executor
+from simulations.validation import new_sweep_config
+from cadCAD import configs
+
+exec_mode = ExecutionMode()
+
+print("Simulation Execution: Concurrent Execution")
+multi_proc_ctx = ExecutionContext(context=exec_mode.multi_proc)
+run = Executor(exec_context=multi_proc_ctx, configs=configs)
+
+i = 0
+config_names = ['sweep_config_A', 'sweep_config_B']
+for raw_result, tensor_field in run.main():
+    result = pd.DataFrame(raw_result)
+    print()
+    print("Tensor Field: " + config_names[i])
+    print(tabulate(tensor_field, headers='keys', tablefmt='psql'))
+    print("Output:")
+    print(tabulate(result, headers='keys', tablefmt='psql'))
+    print()
+    i += 1

simulations/regression_tests/sweep_config.py

@@ -0,0 +1,182 @@
+from decimal import Decimal
+import numpy as np
+from datetime import timedelta
+import pprint
+
+from cadCAD.configuration import append_configs
+from cadCAD.configuration.utils import proc_trigger, ep_time_step, config_sim
+
+from typing import Dict, List
+
+pp = pprint.PrettyPrinter(indent=4)
+
+seeds = {
+    'z': np.random.RandomState(1),
+    'a': np.random.RandomState(2),
+    'b': np.random.RandomState(3),
+    'c': np.random.RandomState(3)
+}
+
+# Optional
+g: Dict[str, List[int]] = {
+    'alpha': [1],
+    'beta': [2, 5],
+    'gamma': [3, 4],
+    'omega': [7]
+}
+
+# Policies per Mechanism
+def p1m1(_g, step, sL, s):
+    return {'param1': 1}
+
+def p2m1(_g, step, sL, s):
+    return {'param2': 4}
+
+def p1m2(_g, step, sL, s):
+    return {'param1': 'a', 'param2': _g['beta']}
+
+def p2m2(_g, step, sL, s):
+    return {'param1': 'b', 'param2': 0}
+
+def p1m3(_g, step, sL, s):
+    return {'param1': np.array([10, 100])}
+
+def p2m3(_g, step, sL, s):
+    return {'param1': np.array([20, 200])}
+
+
+# Internal States per Mechanism
+def s1m1(_g, step, sL, s, _input):
+    return 's1', 0
+
+def s2m1(_g, step, sL, s, _input):
+    return 's2', _g['beta']
+
+def s1m2(_g, step, sL, s, _input):
+    return 's1', _input['param2']
+
+def s2m2(_g, step, sL, s, _input):
+    return 's2', _input['param2']
+
+def s1m3(_g, step, sL, s, _input):
+    return 's1', 0
+
+def s2m3(_g, step, sL, s, _input):
+    return 's2', 0
+
+
+# Exogenous States
+proc_one_coef_A = 0.7
+proc_one_coef_B = 1.3
+
+
+def es3p1(_g, step, sL, s, _input):
+    return 's3', _g['gamma']
+# @curried
+def es4p2(_g, step, sL, s, _input):
+    return 's4', _g['gamma']
+
+ts_format = '%Y-%m-%d %H:%M:%S'
+t_delta = timedelta(days=0, minutes=0, seconds=1)
+def es5p2(_g, step, sL, s, _input):
+    y = 'timestep'
+    x = ep_time_step(s, dt_str=s['timestep'], fromat_str=ts_format, _timedelta=t_delta)
+    return (y, x)
+
+
+# Environment States
+# @curried
+# def env_a(param, x):
+#     return x + param
+def env_a(x):
+    return x
+def env_b(x):
+    return 10
+
+
+# Genesis States
+genesis_states = {
+    's1': Decimal(0.0),
+    's2': Decimal(0.0),
+    's3': Decimal(1.0),
+    's4': Decimal(1.0),
+#     'timestep': '2018-10-01 15:16:24'
+}
+
+
+# remove `exo_update_per_ts` to update every ts
+raw_exogenous_states = {
+    "s3": es3p1,
+    "s4": es4p2,
+#     "timestep": es5p2
+}
+
+
+# ToDo: make env proc trigger field agnostic
+# ToDo: input json into function renaming __name__
+triggered_env_b = proc_trigger(1, env_b)
+env_processes = {
+    "s3": env_a, #sweep(beta, env_a),
+    "s4": triggered_env_b #rename('parameterized', triggered_env_b) #sweep(beta, triggered_env_b)
+}
+# parameterized_env_processes = parameterize_states(env_processes)
+#
+# pp.pprint(parameterized_env_processes)
+# exit()
+
+# ToDo: The number of values entered in sweep should be the # of config objs created,
+# not dependent on the # of times the sweep is applied
+# sweep exo_state func and point to exo-state in every other funtion
+# param sweep on genesis states
+
+partial_state_update_block = {
+    "m1": {
+        "policies": {
+            "b1": p1m1,
+            "b2": p2m1
+        },
+        "variables": {
+            "s1": s1m1,
+            "s2": s2m1
+        }
+    },
+    "m2": {
+        "policies": {
+            "b1": p1m2,
+            "b2": p2m2,
+        },
+        "variables": {
+            "s1": s1m2,
+            "s2": s2m2
+        }
+    },
+    "m3": {
+        "policies": {
+            "b1": p1m3,
+            "b2": p2m3
+        },
+        "variables": {
+            "s1": s1m3,
+            "s2": s2m3
+        }
+    }
+}
+
+# config_sim Necessary
+sim_config = config_sim(
+    {
+        "N": 2,
+        "T": range(5),
+        "M": g # Optional
+    }
+)
+
+# New Convention
+append_configs(
+    sim_configs=sim_config,
+    initial_state=genesis_states,
+    seeds=seeds,
+    raw_exogenous_states=raw_exogenous_states,
+    env_processes=env_processes,
+    partial_state_update_blocks=partial_state_update_block
+)

simulations/regression_tests/udo.py

@@ -0,0 +1,194 @@
+from copy import deepcopy
+from datetime import timedelta
+from functools import reduce
+
+from cadCAD.utils import SilentDF #, val_switch
+from cadCAD.configuration import append_configs
+from cadCAD.configuration.utils import time_step, config_sim, proc_trigger, timestep_trigger, genereate_psubs
+from cadCAD.configuration.utils.userDefinedObject import udoPipe, UDO
+import pandas as pd
+
+from fn.func import curried
+
+import pprint as pp
+
+from cadCAD.utils.sys_config import add
+
+DF = SilentDF(pd.read_csv('/Users/jjodesty/Projects/DiffyQ-SimCAD/simulations/external_data/output.csv'))
+
+
+class udoExample(object):
+    def __init__(self, x, dataset=None):
+        self.x = x
+        self.mem_id = str(hex(id(self)))
+        self.ds = dataset # for setting ds initially or querying
+        self.perception = {}
+
+    def anon(self, f):
+        return f(self)
+
+    def updateX(self):
+        self.x += 1
+        return self
+
+    def perceive(self, s):
+        self.perception = self.ds[
+            (self.ds['run'] == s['run']) & (self.ds['substep'] == s['substep']) & (self.ds['timestep'] == s['timestep'])
+        ].drop(columns=['run', 'substep']).to_dict()
+        return self
+
+    def read(self, ds_uri):
+        self.ds = SilentDF(pd.read_csv(ds_uri))
+        return self
+
+    def write(self, ds_uri):
+        pd.to_csv(ds_uri)
+
+    # ToDo: Generic update function
+
+    pass
+
+
+state_udo = UDO(udo=udoExample(0, DF), masked_members=['obj', 'perception'])
+policy_udoA = UDO(udo=udoExample(0, DF), masked_members=['obj', 'perception'])
+policy_udoB = UDO(udo=udoExample(0, DF), masked_members=['obj', 'perception'])
+
+
+sim_config = config_sim({
+    "N": 2,
+    "T": range(4)
+})
+
+# ToDo: DataFrame Column order
+state_dict = {
+    'increment': 0,
+    'state_udo': state_udo, 'state_udo_tracker': 0,
+    'state_udo_perception_tracker': {"ds1": None, "ds2": None, "ds3": None, "timestep": None},
+    'udo_policies': {'udo_A': policy_udoA, 'udo_B': policy_udoB},
+    'udo_policy_tracker': (0, 0),
+    'timestamp': '2019-01-01 00:00:00'
+}
+
+policies, state_updates = {}, {}
+#
+# def assign_udo_policy(udo):
+#     def policy(_g, step, sL, s):
+#         s['udo_policies'][udo].updateX()
+#         return {udo: udoPipe(s['udo_policies'][udo])}
+#     return policy
+# policies_updates = {p: assign_udo_policy(udo) for p, udo in zip(['p1', 'p2'], ['udo_A', 'udo_B'])}
+
+def udo_policyA(_g, step, sL, s):
+    s['udo_policies']['udo_A'].updateX()
+    return {'udo_A': udoPipe(s['udo_policies']['udo_A'])}
+policies['a'] = udo_policyA
+
+def udo_policyB(_g, step, sL, s):
+    s['udo_policies']['udo_B'].updateX()
+    return {'udo_B': udoPipe(s['udo_policies']['udo_B'])}
+policies['b'] = udo_policyB
+
+
+# policies = {"p1": udo_policyA, "p2": udo_policyB}
+# policies = {"A": udo_policyA, "B": udo_policyB}
+
+# def increment_state_by_int(y: str, incr_by: int):
+#     return lambda _g, step, sL, s, _input: (y, s[y] + incr_by)
+state_updates['increment'] = add('increment', 1)
+
+@curried
+def perceive(s, self):
+    self.perception = self.ds[
+        (self.ds['run'] == s['run']) & (self.ds['substep'] == s['substep']) & (self.ds['timestep'] == s['timestep'])
+    ].drop(columns=['run', 'substep']).to_dict()
+    return self
+
+
+def state_udo_update(_g, step, sL, s, _input):
+    y = 'state_udo'
+    # s['hydra_state'].updateX().anon(perceive(s))
+    s['state_udo'].updateX().perceive(s)
+    x = udoPipe(s['state_udo'])
+    return y, x
+state_updates['state_udo'] = state_udo_update
+
+
+def track(destination, source):
+    return lambda _g, step, sL, s, _input: (destination, s[source].x)
+state_updates['state_udo_tracker'] = track('state_udo_tracker', 'state_udo')
+
+
+def track_state_udo_perception(destination, source):
+    def id(past_perception):
+        if len(past_perception) == 0:
+            return state_dict['state_udo_perception_tracker']
+        else:
+            return past_perception
+    return lambda _g, step, sL, s, _input: (destination, id(s[source].perception))
+state_updates['state_udo_perception_tracker'] = track_state_udo_perception('state_udo_perception_tracker', 'state_udo')
+
+
+def view_udo_policy(_g, step, sL, s, _input):
+    return 'udo_policies', _input
+state_updates['udo_policies'] = view_udo_policy
+
+
+def track_udo_policy(destination, source):
+    def val_switch(v):
+        if isinstance(v, pd.DataFrame) is True or isinstance(v, SilentDF) is True:
+            return SilentDF(v)
+        else:
+            return v.x
+    return lambda _g, step, sL, s, _input: (destination, tuple(val_switch(v) for _, v in s[source].items()))
+state_updates['udo_policy_tracker'] = track_udo_policy('udo_policy_tracker', 'udo_policies')
+
+def update_timestamp(_g, step, sL, s, _input):
+    y = 'timestamp'
+    return y, time_step(dt_str=s[y], dt_format='%Y-%m-%d %H:%M:%S', _timedelta=timedelta(days=0, minutes=0, seconds=1))
+
+
+system_substeps = 3
+# state_updates['timestamp'] = update_timestamp
+state_updates['timestamp'] = timestep_trigger(end_substep=system_substeps, y='timestamp', f=update_timestamp)
+# state_updates['timestamp'] = proc_trigger(y='timestamp', f=update_timestamp, conditions={'substep': [0, substeps]}, cond_op=lambda a, b: a and b)
+
+print()
+print("State Updates:")
+pp.pprint(state_updates)
+print()
+print("Policies:")
+pp.pprint(policies)
+print()
+
+filter_out = lambda remove_list, state_list: list(filter(lambda state: state not in remove_list, state_list))
+
+states = list(state_updates.keys())
+# states_noTS = filter_out(['timestamp'], states)
+# states_grid = [states,states_noTS,states_noTS]
+
+states_grid = [states] * system_substeps #[states,states,states]
+policy_grid = [['a', 'b'], ['a', 'b'], ['a', 'b']]
+
+
+PSUBS = genereate_psubs(policy_grid, states_grid, policies, state_updates)
+pp.pprint(PSUBS)
+# ToDo: Bug without specifying parameters
+append_configs(
+    sim_configs=sim_config,
+    initial_state=state_dict,
+    seeds={},
+    raw_exogenous_states={},
+    env_processes={},
+    partial_state_update_blocks=PSUBS,
+    # policy_ops=[lambda a, b: {**a, **b}]
+)
+
+# pp.pprint(partial_state_update_blocks)
+
+# PSUB = {
+#     'policies': policies,
+#     'states': state_updates
+# }
+# partial_state_update_blocks = [PSUB] * substeps
+
+

simulations/single_config_run.py

@@ -3,8 +3,7 @@ from tabulate import tabulate
 # The following imports NEED to be in the exact order
 from cadCAD.engine import ExecutionMode, ExecutionContext, Executor
 # from simulations.validation import policy_aggregation
-from simulations.validation import historical_state_access
+from simulations.validation import config1
-# from simulations.validation import config1
 # from simulations.validation import externalds
 # from simulations.validation import external_dataset
 from cadCAD import configs

simulations/udo_test.py

@@ -0,0 +1,45 @@
+import pandas as pd
+from tabulate import tabulate
+# The following imports NEED to be in the exact order
+from cadCAD.engine import ExecutionMode, ExecutionContext, Executor
+from simulations.regression_tests import udo
+from cadCAD import configs
+
+
+exec_mode = ExecutionMode()
+
+print("Simulation Execution: Single Configuration")
+print()
+
+
+first_config = configs # only contains config1
+single_proc_ctx = ExecutionContext(context=exec_mode.single_proc)
+run = Executor(exec_context=single_proc_ctx, configs=first_config)
+# cols = configs[0].initial_state.keys()
+cols = [
+    'increment',
+    'state_udo_tracker', 'state_udo', 'state_udo_perception_tracker',
+    'udo_policies', 'udo_policy_tracker',
+    'timestamp'
+]
+raw_result, tensor_field = run.main()
+result = pd.DataFrame(raw_result)[['run', 'substep', 'timestep'] + cols]
+# result = pd.concat([result.drop(['c'], axis=1), result['c'].apply(pd.Series)], axis=1)
+
+# print(list(result['c']))
+
+# print(tabulate(result['c'].apply(pd.Series), headers='keys', tablefmt='psql'))
+
+print()
+print("Tensor Field: config1")
+print(tabulate(tensor_field, headers='keys', tablefmt='psql'))
+print("Output:")
+print(tabulate(result, headers='keys', tablefmt='psql'))
+print()
+print(result.info(verbose=True))
+
+# def f(df, col):
+#     for k in df[col].iloc[0].keys():
+#         df[k] = None
+#     for index, row in df.iterrows():
+#         # df.apply(lambda row:, axis=1)

simulations/validation/config1.py

@@ -7,7 +7,9 @@ from datetime import timedelta
 from cadCAD.configuration.utils.policyAggregation import get_base_value
 
 from cadCAD.configuration import append_configs
-from cadCAD.configuration.utils import proc_trigger, bound_norm_random, ep_time_step, config_sim
+from cadCAD.configuration.utils import env_proc_trigger, bound_norm_random, ep_time_step, config_sim
+
+from cadCAD.configuration.utils import timestep_trigger
 
 seeds = {
     'z': np.random.RandomState(1),
@@ -23,8 +25,6 @@ def p1m1(_g, step, sL, s):
 def p2m1(_g, step, sL, s):
     return {'param1': 1, 'param2': 4}
 
-# []
-
 def p1m2(_g, step, sL, s):
     return {'param1': 'a', 'param2': 2}
 def p2m2(_g, step, sL, s):
@@ -113,20 +113,20 @@ genesis_states = {
 }
 
 
-raw_exogenous_states = {
+# raw_exogenous_states = {
-    "s3": es3p1,
+#     "s3": es3p1,
-    "s4": es4p2,
+#     "s4": es4p2,
-#     "timestep": es5p2
+# #     "timestep": es5p2
-}
+# }
 
 
 env_processes = {
     "s3": env_a,
-    "s4": proc_trigger(1, env_b)
+    "s4": env_proc_trigger(1, env_b)
 }
 
 
-partial_state_update_block = {
+partial_state_update_blocks = {
     "m1": {
         "policies": {
             "b1": p1m1,
@@ -134,7 +134,9 @@ partial_state_update_block = {
         },
         "variables": {
             "s1": s1m1,
-            "s2": s2m1
+            "s2": s2m1,
+            "s3": es3p1,
+            "s4": es4p2,
         }
     },
     "m2": {
@@ -144,7 +146,9 @@ partial_state_update_block = {
         },
         "variables": {
             "s1": s1m2,
-            "s2": s2m2
+            "s2": s2m2,
+            # "s3": timestep_trigger(3, 's3', es3p1),
+            # "s4": timestep_trigger(3, 's4', es4p2),
         }
     },
     "m3": {
@@ -154,7 +158,9 @@ partial_state_update_block = {
         },
         "variables": {
             "s1": s1m3,
-            "s2": s2m3
+            "s2": s2m3,
+            # "s3": timestep_trigger(3, 's3', es3p1),
+            # "s4": timestep_trigger(3, 's4', es4p2),
         }
     }
 }
@@ -171,8 +177,8 @@ append_configs(
     sim_configs=sim_config,
     initial_state=genesis_states,
     seeds=seeds,
-    raw_exogenous_states=raw_exogenous_states,
+    raw_exogenous_states={}, #raw_exogenous_states,
-    env_processes=env_processes,
+    env_processes={}, #env_processes,
-    partial_state_update_blocks=partial_state_update_block,
+    partial_state_update_blocks=partial_state_update_blocks,
     policy_ops=[lambda a, b: a + b]
 )

simulations/validation/config2.py

@@ -3,7 +3,7 @@ import numpy as np
 from datetime import timedelta
 
 from cadCAD.configuration import append_configs
-from cadCAD.configuration.utils import proc_trigger, bound_norm_random, ep_time_step, config_sim
+from cadCAD.configuration.utils import env_proc_trigger, bound_norm_random, ep_time_step, config_sim
 
 seeds = {
     'z': np.random.RandomState(1),
@@ -108,8 +108,8 @@ raw_exogenous_states = {
 
 
 env_processes = {
-    "s3": proc_trigger(1, env_a),
+    "s3": env_proc_trigger(1, env_a),
-    "s4": proc_trigger(1, env_b)
+    "s4": env_proc_trigger(1, env_b)
 }
 
 

simulations/validation/config4.py

@@ -3,7 +3,7 @@ import numpy as np
 from datetime import timedelta
 
 from cadCAD.configuration import append_configs
-from cadCAD.configuration.utils import proc_trigger, bound_norm_random, ep_time_step, config_sim
+from cadCAD.configuration.utils import env_proc_trigger, bound_norm_random, ep_time_step, config_sim
 
 
 seeds = {
@@ -113,7 +113,7 @@ raw_exogenous_states = {
 
 env_processes = {
     "s3": env_a,
-    "s4": proc_trigger('2018-10-01 15:16:25', env_b)
+    "s4": env_proc_trigger('2018-10-01 15:16:25', env_b)
 }
 
 

simulations/validation/historical_state_access.py

@@ -2,18 +2,18 @@ from cadCAD.configuration import append_configs
 from cadCAD.configuration.utils import config_sim
 
 # last_partial_state_update_block
-def last_update_block(_g, step, sH, s, _input):
+def last_update_block(_g, substep, sH, s, _input):
     return 'sh', sH[-1]
 
 
 # Policies per Mechanism
-def p(_g, step, sH, s):
+def p(_g, substep, sH, s):
     return {'last_update_block': sH[-1]}
 
 def add(y, x):
-    return lambda _g, step, sH, s, _input: (y, s[y] + x)
+    return lambda _g, substep, sH, s, _input: (y, s[y] + x)
 
-def policies(_g, step, sH, s, _input):
+def policies(_g, substep, sH, s, _input):
     y = 'policies'
     x = _input
     return (y, x)
@@ -22,7 +22,7 @@ policies = {"p1": p, "p2": p}
 
 genesis_states = {
     's': 0,
-    'sh': {}, # {[], {}}
+    'sh': [{}], # {[], {}}
     # 'policies': {},
 }
 

simulations/validation/new_sweep_config.py

@@ -0,0 +1,183 @@
+from decimal import Decimal
+import numpy as np
+from datetime import timedelta
+import pprint
+
+from cadCAD.configuration import append_configs
+from cadCAD.configuration.utils import env_proc_trigger, ep_time_step, config_sim
+
+from typing import Dict, List
+
+# from cadCAD.utils.sys_config import exo, exo_check
+
+pp = pprint.PrettyPrinter(indent=4)
+
+seeds = {
+    'z': np.random.RandomState(1),
+    'a': np.random.RandomState(2),
+    'b': np.random.RandomState(3),
+    'c': np.random.RandomState(3)
+}
+
+# Optional
+g: Dict[str, List[int]] = {
+    'alpha': [1],
+    'beta': [2, 5],
+    'gamma': [3, 4],
+    'omega': [7]
+}
+
+# Policies per Mechanism
+def p1m1(_g, step, sL, s):
+    return {'param1': 1}
+
+def p2m1(_g, step, sL, s):
+    return {'param2': 4}
+
+def p1m2(_g, step, sL, s):
+    return {'param1': 'a', 'param2': _g['beta']}
+
+def p2m2(_g, step, sL, s):
+    return {'param1': 'b', 'param2': 0}
+
+def p1m3(_g, step, sL, s):
+    return {'param1': np.array([10, 100])}
+
+def p2m3(_g, step, sL, s):
+    return {'param1': np.array([20, 200])}
+
+# Internal States per Mechanism
+def s1m1(_g, step, sL, s, _input):
+    return 's1', 0
+
+def s2m1(_g, step, sL, s, _input):
+    return 's2', _g['beta']
+
+def s1m2(_g, step, sL, s, _input):
+    return 's1', _input['param2']
+
+def s2m2(_g, step, sL, s, _input):
+    return 's2', _input['param2']
+
+def s1m3(_g, step, sL, s, _input):
+    return 's1', 0
+
+def s2m3(_g, step, sL, s, _input):
+    return 's2', 0
+
+
+# Exogenous States
+proc_one_coef_A = 0.7
+proc_one_coef_B = 1.3
+
+
+def es3p1(_g, step, sL, s, _input):
+    return 's3', _g['gamma']
+# @curried
+def es4p2(_g, step, sL, s, _input):
+    return 's4', _g['gamma']
+
+ts_format = '%Y-%m-%d %H:%M:%S'
+t_delta = timedelta(days=0, minutes=0, seconds=1)
+def es5p2(_g, step, sL, s, _input):
+    y = 'timestep'
+    x = ep_time_step(s, dt_str=s['timestep'], fromat_str=ts_format, _timedelta=t_delta)
+    return (y, x)
+
+
+# Environment States
+# @curried
+# def env_a(param, x):
+#     return x + param
+def env_a(x):
+    return x
+def env_b(x):
+    return 10
+
+
+# Genesis States
+genesis_states = {
+    's1': Decimal(0.0),
+    's2': Decimal(0.0),
+    's3': Decimal(1.0),
+    's4': Decimal(1.0),
+#     'timestep': '2018-10-01 15:16:24'
+}
+
+
+# remove `exo_update_per_ts` to update every ts
+raw_exogenous_states = {
+    "s3": es3p1,
+    "s4": es4p2,
+#     "timestep": es5p2
+}
+
+
+# ToDo: make env proc trigger field agnostic
+# ToDo: input json into function renaming __name__
+triggered_env_b = env_proc_trigger(1, env_b)
+env_processes = {
+    "s3": env_a, #sweep(beta, env_a),
+    "s4": triggered_env_b #rename('parameterized', triggered_env_b) #sweep(beta, triggered_env_b)
+}
+# parameterized_env_processes = parameterize_states(env_processes)
+#
+# pp.pprint(parameterized_env_processes)
+# exit()
+
+# ToDo: The number of values entered in sweep should be the # of config objs created,
+# not dependent on the # of times the sweep is applied
+# sweep exo_state func and point to exo-state in every other funtion
+# param sweep on genesis states
+
+partial_state_update_block = {
+    "m1": {
+        "policies": {
+            "b1": p1m1,
+            "b2": p2m1
+        },
+        "variables": {
+            "s1": s1m1,
+            "s2": s2m1
+        }
+    },
+    "m2": {
+        "policies": {
+            "b1": p1m2,
+            "b2": p2m2,
+        },
+        "variables": {
+            "s1": s1m2,
+            "s2": s2m2
+        }
+    },
+    "m3": {
+        "policies": {
+            "b1": p1m3,
+            "b2": p2m3
+        },
+        "variables": {
+            "s1": s1m3,
+            "s2": s2m3
+        }
+    }
+}
+
+# config_sim Necessary
+sim_config = config_sim(
+    {
+        "N": 2,
+        "T": range(5),
+        "M": g # Optional
+    }
+)
+
+# New Convention
+append_configs(
+    sim_configs=sim_config,
+    initial_state=genesis_states,
+    seeds=seeds,
+    raw_exogenous_states={}, #raw_exogenous_states,
+    env_processes={}, #env_processes,
+    partial_state_update_blocks=partial_state_update_block
+)

simulations/validation/policy_aggregation.py

@@ -96,11 +96,3 @@ append_configs(
     partial_state_update_blocks=partial_state_update_block,
     policy_ops=[lambda a, b: a + b, lambda y: y + 10, lambda y: y + 30]
 )
-
-
-# def p1m3(_g, step, sL, s):
-#     return {'param1': 1, 'param2': 2, 'param3': 3}
-# def p2m3(_g, step, sL, s):
-#     return {'param1': 1, 'param2': 2, 'param3': 3}
-#
-# xx = {'param1': [1,1], 'param2': [2,2], 'param3': [3,3]}

simulations/validation/sweep_config.py

@@ -4,7 +4,7 @@ from datetime import timedelta
 import pprint
 
 from cadCAD.configuration import append_configs
-from cadCAD.configuration.utils import proc_trigger, ep_time_step, config_sim
+from cadCAD.configuration.utils import env_proc_trigger, ep_time_step, config_sim
 
 from typing import Dict, List
 
@@ -46,34 +46,22 @@ def p2m3(_g, step, sL, s):
 
 # Internal States per Mechanism
 def s1m1(_g, step, sL, s, _input):
-    y = 's1'
+    return 's1', 0
-    x = 0
-    return (y, x)
 
 def s2m1(_g, step, sL, s, _input):
-    y = 's2'
+    return 's2', _g['beta']
-    x = _g['beta']
-    return (y, x)
 
 def s1m2(_g, step, sL, s, _input):
-    y = 's1'
+    return 's1', _input['param2']
-    x = _input['param2']
-    return (y, x)
 
 def s2m2(_g, step, sL, s, _input):
-    y = 's2'
+    return 's2', _input['param2']
-    x = _input['param2']
-    return (y, x)
 
 def s1m3(_g, step, sL, s, _input):
-    y = 's1'
+    return 's1', 0
-    x = 0
-    return (y, x)
 
 def s2m3(_g, step, sL, s, _input):
-    y = 's2'
+    return 's2', 0
-    x = 0
-    return (y, x)
 
 
 # Exogenous States
@@ -82,14 +70,10 @@ proc_one_coef_B = 1.3
 
 
 def es3p1(_g, step, sL, s, _input):
-    y = 's3'
+    return 's3', _g['gamma']
-    x = _g['gamma']
-    return (y, x)
 # @curried
 def es4p2(_g, step, sL, s, _input):
-    y = 's4'
+    return 's4', _g['gamma']
-    x = _g['gamma']
-    return (y, x)
 
 ts_format = '%Y-%m-%d %H:%M:%S'
 t_delta = timedelta(days=0, minutes=0, seconds=1)
@@ -129,7 +113,7 @@ raw_exogenous_states = {
 
 # ToDo: make env proc trigger field agnostic
 # ToDo: input json into function renaming __name__
-triggered_env_b = proc_trigger(1, env_b)
+triggered_env_b = env_proc_trigger(1, env_b)
 env_processes = {
     "s3": env_a, #sweep(beta, env_a),
     "s4": triggered_env_b #rename('parameterized', triggered_env_b) #sweep(beta, triggered_env_b)

simulations/validation/udo.py

@@ -1,7 +1,8 @@
+from copy import deepcopy
 from datetime import timedelta
 from cadCAD.utils import SilentDF #, val_switch
 from cadCAD.configuration import append_configs
-from cadCAD.configuration.utils import ep_time_step, config_sim
+from cadCAD.configuration.utils import time_step, ep_time_step, config_sim
 from cadCAD.configuration.utils.userDefinedObject import udoPipe, UDO
 import pandas as pd
 
@@ -46,14 +47,25 @@ state_udo = UDO(udo=udoExample(0, DF), masked_members=['obj', 'perception'])
 policy_udoA = UDO(udo=udoExample(0, DF), masked_members=['obj', 'perception'])
 policy_udoB = UDO(udo=udoExample(0, DF), masked_members=['obj', 'perception'])
 
+def udo_policyA(_g, step, sL, s):
+    s['udo_policies']['udo_A'].updateX()
+    return {'udo_A': udoPipe(s['udo_policies']['udo_A'])}
+
+def udo_policyB(_g, step, sL, s):
+    s['udo_policies']['udo_B'].updateX()
+    return {'udo_B': udoPipe(s['udo_policies']['udo_B'])}
+
+
+policies = {"p1": udo_policyA, "p2": udo_policyB}
+
 # ToDo: DataFrame Column order
 state_dict = {
     'increment': 0,
     'state_udo': state_udo, 'state_udo_tracker_a': 0, 'state_udo_tracker_b': 0,
     'state_udo_perception_tracker': {"ds1": None, "ds2": None, "ds3": None, "timestep": None},
     'udo_policies': {'udo_A': policy_udoA, 'udo_B': policy_udoB},
-    'udo_policy_tracker_a': (None, None), 'udo_policy_tracker_b': (None, None),
+    'udo_policy_tracker_a': (0, 0), 'udo_policy_tracker_b': (0, 0),
-    'timestamp': '2019-01-01 00:00:00',
+    'timestamp': '2019-01-01 00:00:00'
 }
 
 @curried
@@ -64,24 +76,14 @@ def perceive(s, self):
     return self
 
 def view_udo_policy(_g, step, sL, s, _input):
-    y = 'udo_policies'
+    return 'udo_policies', _input
-    x = _input
-    return (y, x)
-
-def update_timestamp(y, timedelta, format):
-    return lambda _g, step, sL, s, _input: (
-        y,
-        ep_time_step(s, dt_str=s[y], fromat_str=format, _timedelta=timedelta)
-    )
-time_model = update_timestamp('timestamp', timedelta(minutes=1), '%Y-%m-%d %H:%M:%S')
-
 
 def state_udo_update(_g, step, sL, s, _input):
     y = 'state_udo'
     # s['hydra_state'].updateX().anon(perceive(s))
     s['state_udo'].updateX().perceive(s)
     x = udoPipe(s['state_udo'])
-    return (y, x)
+    return y, x
 
 def increment(y, incr_by):
     return lambda _g, step, sL, s, _input: (y, s[y] + incr_by)
@@ -105,17 +107,17 @@ def track_state_udo_perception(destination, source):
             return past_perception
     return lambda _g, step, sL, s, _input: (destination, id(s[source].perception))
 
-def udo_policyA(_g, step, sL, s):
-    s['udo_policies']['udo_A'].updateX()
-    return {'udo_A': udoPipe(s['udo_policies']['udo_A'])}
 
-def udo_policyB(_g, step, sL, s):
+def time_model(y, substeps, time_delta, ts_format='%Y-%m-%d %H:%M:%S'):
-    s['udo_policies']['udo_B'].updateX()
+    def apply_incriment_condition(s):
-    return {'udo_B': udoPipe(s['udo_policies']['udo_B'])}
+        if s['substep'] == 0 or s['substep'] == substeps:
+            return y, time_step(dt_str=s[y], dt_format=ts_format, _timedelta=time_delta)
+        else:
+            return y, s[y]
+    return lambda _g, step, sL, s, _input: apply_incriment_condition(s)
 
-policies = {"p1": udo_policyA, "p2": udo_policyB}
 
-states_with_ts = {
+states = {
     'increment': increment('increment', 1),
     'state_udo_tracker_a': track('state_udo_tracker_a', 'state_udo'),
     'state_udo': state_udo_update,
@@ -123,39 +125,38 @@ states_with_ts = {
     'state_udo_tracker_b': track('state_udo_tracker_b', 'state_udo'),
     'udo_policy_tracker_a': track_udo_policy('udo_policy_tracker_a', 'udo_policies'),
     'udo_policies': view_udo_policy,
-    'udo_policy_tracker_b': track_udo_policy('udo_policy_tracker_b', 'udo_policies'),
+    'udo_policy_tracker_b': track_udo_policy('udo_policy_tracker_b', 'udo_policies')
-    'timestamp': time_model,
+}
+
+substeps=3
+update_timestamp = time_model(
+    'timestamp',
+    substeps=3,
+    time_delta=timedelta(days=0, minutes=0, seconds=1),
+    ts_format='%Y-%m-%d %H:%M:%S'
+)
+states['timestamp'] = update_timestamp
+
+PSUB = {
+    'policies': policies,
+    'states': states
 }
-del states_with_ts['timestamp']
-states_without_ts = states_with_ts
 
 # needs M1&2 need behaviors
-partial_state_update_blocks = {
+partial_state_update_blocks = [PSUB] * substeps
-    'PSUB1': {
-        'policies': policies,
-        'states': states_with_ts
-    },
-    'PSUB2': {
-        'policies': policies,
-        'states': states_without_ts
-    },
-    'PSUB3': {
-        'policies': policies,
-        'states': states_without_ts
-    }
-}
 
 sim_config = config_sim({
     "N": 2,
     "T": range(4)
 })
 
+# ToDo: Bug without specifying parameters
 append_configs(
-    sim_config,
+    sim_configs=sim_config,
-    state_dict,
+    initial_state=state_dict,
-    # seeds=seeds,
+    seeds={},
-    # raw_exogenous_states=raw_exogenous_states,
+    raw_exogenous_states={},
-    # env_processes=env_processes,
+    env_processes={},
-    partial_state_update_blocks,
+    partial_state_update_blocks=partial_state_update_blocks,
-    policy_ops=[lambda a, b: {**a, **b}]
+    # policy_ops=[lambda a, b: {**a, **b}]
 )