Merge pull request #13 from BlockScience/jj-dev

Bug Fix: Can't use environments without proc_trigger

README.md

@@ -46,9 +46,9 @@ exec_mode = ExecutionMode()
 
 print("Simulation Run 1")
 print()
-single_config = [configs[0]]
+first_config = [configs[0]] # from config1
 single_proc_ctx = ExecutionContext(context=exec_mode.single_proc)
-run1 = Executor(exec_context=single_proc_ctx, configs=single_config)
+run1 = Executor(exec_context=single_proc_ctx, configs=first_config)
 run1_raw_result = run1.main()
 result = pd.DataFrame(run1_raw_result)
 # result.to_csv('~/Projects/DiffyQ-SimCAD/results/config4csv', sep=',')

SimCAD/configuration/__init__.py

@@ -40,7 +40,6 @@ class Identity:
 
 
 class Processor:
-
     def __init__(self, id=Identity()):
         self.id = id
         self.b_identity = id.b_identity
@@ -65,7 +64,8 @@ class Processor:
     # Maybe Refactor to only use dictionary BUT I used dfs to fill NAs. Perhaps fill
     def generate_config(self, state_dict, mechanisms, exo_proc):
 
-        # include False / False case
+        # ToDo: include False / False case
+        # ToDo: Use Range multiplier instead for loop iterator
         def no_update_handler(bdf, sdf):
             if (bdf.empty == False) and (sdf.empty == True):
                 bdf_values = bdf.values.tolist()
@@ -88,7 +88,6 @@ class Processor:
             bdf_values = [[self.b_identity] * len(sdf_values)]
             return sdf_values, bdf_values
 
-        # zipped_list = []
         if len(mechanisms) != 0:
             bdf = self.create_matrix_field(mechanisms, 'behaviors')
             sdf = self.create_matrix_field(mechanisms, 'states')

SimCAD/configuration/utils/__init__.py

@@ -3,6 +3,7 @@ from decimal import Decimal
 from fn.func import curried
 import pandas as pd
 
+
 class TensorFieldReport:
     def __init__(self, config_proc):
         self.config_proc = config_proc

SimCAD/engine/__init__.py

@@ -28,7 +28,6 @@ class ExecutionContext:
             l = list(zip(fs, states_list, configs, env_processes, Ts, Ns))
             with Pool(len(configs)) as p:
                 results = p.map(lambda t: t[0](t[1], t[2], t[3], t[4], t[5]), l)
-
             return results
 
         if context == 'single_proc':

SimCAD/engine/simulation.py

@@ -1,44 +1,45 @@
 from copy import deepcopy
 from fn.op import foldr, call
-import pprint
+
-pp = pprint.PrettyPrinter(indent=4)
+from SimCAD.utils import rename
+from SimCAD.engine.utils import engine_exception
+
+
+id_exception = engine_exception(KeyError, KeyError, None)
 
 
 class Executor:
-    def __init__(self, behavior_ops):
+    def __init__(self, behavior_ops, behavior_update_exception=id_exception, state_update_exception=id_exception):
         self.behavior_ops = behavior_ops
+        self.state_update_exception = state_update_exception
+        self.behavior_update_exception = behavior_update_exception
 
     # Data Type reduction
-    def getBehaviorInput(self, step, sL, s, funcs):
+    def get_behavior_input(self, step, sL, s, funcs):
-
         ops = self.behavior_ops[::-1]
-        def getColResults(step, sL, s, funcs):
+
+        def get_col_results(step, sL, s, funcs):
             return list(map(lambda f: f(step, sL, s), funcs))
 
-        return foldr(call, getColResults(step, sL, s, funcs))(ops)
+        return foldr(call, get_col_results(step, sL, s, funcs))(ops)
 
     def apply_env_proc(self, env_processes, state_dict, step):
         for state in state_dict.keys():
             if state in list(env_processes.keys()):
-                state_dict[state] = env_processes[state](step)(state_dict[state])
+                env_state = env_processes[state]
+                if env_state.__name__ == '_curried': # might want to change
+                    state_dict[state] = env_state(step)(state_dict[state])
+                else:
+                    state_dict[state] = env_state(state_dict[state])
 
-    # remove / modify
-    def exception_handler(self, f, m_step, sL, last_mut_obj, _input):
-        try:
-            return f(m_step, sL, last_mut_obj, _input)
-        except KeyError:
-            print("Exception")
-            return f(m_step, sL, sL[-2], _input)
 
     def mech_step(self, m_step, sL, state_funcs, behavior_funcs, env_processes, t_step, run):
         last_in_obj = sL[-1]
 
-        _input = self.getBehaviorInput(m_step, sL, last_in_obj, behavior_funcs)
+        _input = self.state_update_exception(self.get_behavior_input(m_step, sL, last_in_obj, behavior_funcs))
 
-        # print(sL)
+        # ToDo: add env_proc generator to `last_in_copy` iterator as wrapper function
-
+        last_in_copy = dict([self.behavior_update_exception(f(m_step, sL, last_in_obj, _input)) for f in state_funcs])
-        # *** add env_proc value here as wrapper function ***
-        last_in_copy = dict([self.exception_handler(f, m_step, sL, last_in_obj, _input) for f in state_funcs])
 
         for k in last_in_obj:
             if k not in last_in_copy:
@@ -46,7 +47,7 @@ class Executor:
 
         del last_in_obj
 
-        #	make env proc trigger field agnostic
+        # make env proc trigger field agnostic
         self.apply_env_proc(env_processes, last_in_copy, last_in_copy['timestamp']) # mutating last_in_copy
 
         last_in_copy["mech_step"], last_in_copy["time_step"], last_in_copy['run'] = m_step, t_step, run
@@ -79,10 +80,7 @@ class Executor:
         time_seq = [x + 1 for x in time_seq]
         simulation_list = [states_list]
         for time_step in time_seq:
-            # print(run)
             pipe_run = self.mech_pipeline(simulation_list[-1], configs, env_processes, time_step, run)
-            # pp.pprint(pipe_run)
-            # exit()
             _, *pipe_run = pipe_run
             simulation_list.append(pipe_run)
 
@@ -93,7 +91,6 @@ class Executor:
         pipe_run = []
         for run in range(runs):
             run += 1
-            # print("Run: "+str(run))
             states_list_copy = deepcopy(states_list) # WHY ???
             head, *tail = self.block_pipeline(states_list_copy, configs, env_processes, time_seq, run)
             genesis = head.pop()

SimCAD/engine/utils.py

@@ -1,4 +1,5 @@
 from datetime import datetime
+from fn.func import curried
 
 
 def datetime_range(start, end, delta, dt_format='%Y-%m-%d %H:%M:%S'):
@@ -20,4 +21,21 @@ def last_index(l):
 
 
 def retrieve_state(l, offset):
-    return l[last_index(l) + offset + 1]
+    return l[last_index(l) + offset + 1]
+
+
+@curried
+def engine_exception(ErrorType, error_message, exception_function, try_function):
+    try:
+        return try_function
+    except ErrorType:
+        print(error_message)
+        return exception_function
+
+
+# def exception_handler(f, m_step, sL, last_mut_obj, _input):
+#     try:
+#         return f(m_step, sL, last_mut_obj, _input)
+#     except KeyError:
+#         print("Exception")
+#         return f(m_step, sL, sL[-2], _input)

SimCAD/utils/__init__.py

@@ -1,14 +1,32 @@
-def print_fwd(x):
+# from fn.func import curried
+
+def pipe(x):
+    return x
+
+
+def print_pipe(x):
     print(x)
     return x
 
 
-flatten = lambda l: [item for sublist in l for item in sublist]
+def flatten(l):
+    return [item for sublist in l for item in sublist]
 
 
 def flatmap(f, items):
-        return list(map(f, items))
+    return list(map(f, items))
 
 
 def key_filter(l, keyname):
-    return [v[keyname] for k, v in l.items()]
+    return [v[keyname] for k, v in l.items()]
+
+# @curried
+def rename(new_name, f):
+    f.__name__ = new_name
+    return f
+#
+# def rename(newname):
+#     def decorator(f):
+#         f.__name__ = newname
+#         return f
+#     return decorator

simulations/sim_test.py

@@ -4,6 +4,7 @@ from tabulate import tabulate
 # The following imports NEED to be in the exact same order
 from SimCAD.engine import ExecutionMode, ExecutionContext, Executor
 from simulations.validation import config1, config2
+# from simulations.validation import base_config1, base_config2
 # from simulations.barlin import config4
 # from simulations.zx import config_zx
 # from simulations.barlin import config6atemp #config6aworks,
@@ -14,18 +15,18 @@ from SimCAD import configs
 exec_mode = ExecutionMode()
 
 
-print("Simulation Run 1")
+print("Simulation Execution 1")
 print()
-single_config = [configs[0]]
+first_config = [configs[0]] # from config1
 single_proc_ctx = ExecutionContext(context=exec_mode.single_proc)
-run1 = Executor(exec_context=single_proc_ctx, configs=single_config)
+run1 = Executor(exec_context=single_proc_ctx, configs=first_config)
 run1_raw_result = run1.main()
 result = pd.DataFrame(run1_raw_result)
 # result.to_csv('~/Projects/DiffyQ-SimCAD/results/config4.csv', sep=',')
 print(tabulate(result, headers='keys', tablefmt='psql'))
 print()
 
-print("Simulation Run 2: Pairwise Execution")
+print("Simulation Execution 2: Pairwise Execution")
 print()
 multi_proc_ctx = ExecutionContext(context=exec_mode.multi_proc)
 run2 = Executor(exec_context=multi_proc_ctx, configs=configs)

simulations/validation/base_config1.py

@@ -0,0 +1,171 @@
+from decimal import Decimal
+import numpy as np
+from datetime import timedelta
+
+from SimCAD import configs
+from SimCAD.configuration import Configuration
+from SimCAD.configuration.utils import exo_update_per_ts, proc_trigger, bound_norm_random, \
+    ep_time_step
+
+seed = {
+    'z': np.random.RandomState(1),
+    'a': np.random.RandomState(2),
+    'b': np.random.RandomState(3),
+    'c': np.random.RandomState(3)
+}
+
+# Behaviors per Mechanism
+# Different return types per mechanism ?? *** No ***
+def b1m1(step, sL, s):
+    return {'param1': 1}
+def b2m1(step, sL, s):
+    return {'param1': 1}
+
+def b1m2(step, sL, s):
+    return {'param1': 1, 'param2': 2}
+def b2m2(step, sL, s):
+    return {'param1': 1, 'param2': 4}
+
+def b1m3(step, sL, s):
+    return {'param1': 1, 'param2': np.array([10, 100])}
+def b2m3(step, sL, s):
+    return {'param1': 1, 'param2': np.array([20, 200])}
+
+# deff not more than 2
+# Internal States per Mechanism
+def s1m1(step, sL, s, _input):
+    y = 's1'
+    x = s['s1'] + _input['param1']
+    return (y, x)
+def s2m1(step, sL, s, _input):
+    y = 's2'
+    x = s['s2'] + _input['param1']
+    return (y, x)
+
+def s1m2(step, sL, s, _input):
+    y = 's1'
+    x = s['s1'] + _input['param1']
+    return (y, x)
+def s2m2(step, sL, s, _input):
+    y = 's2'
+    x = s['s2'] + _input['param1']
+    return (y, x)
+
+def s1m3(step, sL, s, _input):
+    y = 's1'
+    x = s['s1'] + _input['param1']
+    return (y, x)
+def s2m3(step, sL, s, _input):
+    y = 's2'
+    x = s['s2'] + _input['param1']
+    return (y, x)
+
+# Exogenous States
+proc_one_coef_A = 0.7
+proc_one_coef_B = 1.3
+
+def es3p1(step, sL, s, _input):
+    y = 's3'
+    x = s['s3'] * bound_norm_random(seed['a'], proc_one_coef_A, proc_one_coef_B)
+    return (y, x)
+
+def es4p2(step, sL, s, _input):
+    y = 's4'
+    x = s['s4'] * bound_norm_random(seed['b'], proc_one_coef_A, proc_one_coef_B)
+    return (y, x)
+
+ts_format = '%Y-%m-%d %H:%M:%S'
+t_delta = timedelta(days=0, minutes=0, seconds=1)
+def es5p2(step, sL, s, _input):
+    y = 'timestamp'
+    x = ep_time_step(s, dt_str=s['timestamp'], fromat_str=ts_format, _timedelta=t_delta)
+    return (y, x)
+
+
+# Environment States
+def env_a(x):
+    return 10
+def env_b(x):
+    return 10
+# def what_ever(x):
+#     return x + 1
+
+# Genesis States
+genesis_states = {
+    's1': Decimal(0.0),
+    's2': Decimal(0.0),
+    's3': Decimal(1.0),
+    's4': Decimal(1.0),
+    'timestamp': '2018-10-01 15:16:24'
+}
+
+# remove `exo_update_per_ts` to update every ts
+exogenous_states = exo_update_per_ts(
+    {
+    "s3": es3p1,
+    "s4": es4p2,
+    "timestamp": es5p2
+    }
+)
+
+#	make env proc trigger field agnostic
+
+# ToDo: Bug - Can't use environments without proc_trigger. TypeError: 'int' object is not callable
+# "/Users/jjodesty/Projects/DiffyQ-SimCAD/SimCAD/engine/simulation.py"
+env_processes = {
+    # "s3": env_a,
+    # "s4": env_b
+    "s3": proc_trigger('2018-10-01 15:16:25', env_a),
+    "s4": proc_trigger('2018-10-01 15:16:25', env_b)
+}
+
+# need at least 1 behaviour and 1 state function for the 1st mech with behaviors
+# mechanisms = {}
+mechanisms = {
+    "m1": {
+        "behaviors": {
+            "b1": b1m1, # lambda step, sL, s: s['s1'] + 1,
+            "b2": b2m1
+        },
+        "states": { # exclude only. TypeError: reduce() of empty sequence with no initial value
+            "s1": s1m1,
+            "s2": s2m1
+        }
+    },
+    "m2": {
+        "behaviors": {
+            "b1": b1m2,
+            "b2": b2m2
+        },
+        "states": {
+            "s1": s1m2,
+            "s2": s2m2
+        }
+    },
+    "m3": {
+        "behaviors": {
+            "b1": b1m3,
+            "b2": b2m3
+        },
+        "states": {
+            "s1": s1m3,
+            "s2": s2m3
+        }
+    }
+}
+
+sim_config = {
+    "N": 2,
+    "T": range(5)
+}
+
+configs.append(
+    Configuration(
+        sim_config=sim_config,
+        state_dict=genesis_states,
+        seed=seed,
+        exogenous_states=exogenous_states,
+        env_processes=env_processes,
+        mechanisms=mechanisms
+    )
+)

simulations/validation/base_config2.py

@@ -0,0 +1,180 @@
+from decimal import Decimal
+import numpy as np
+from datetime import timedelta
+
+from SimCAD import configs
+from SimCAD.configuration import Configuration
+from SimCAD.configuration.utils import exo_update_per_ts, proc_trigger, bound_norm_random, \
+    ep_time_step
+
+
+seed = {
+    'z': np.random.RandomState(1),
+    'a': np.random.RandomState(2),
+    'b': np.random.RandomState(3),
+    'c': np.random.RandomState(3)
+}
+
+# Behaviors per Mechanism
+# Different return types per mechanism ?? *** No ***
+def b1m1(step, sL, s):
+    return {'param1': 1}
+def b2m1(step, sL, s):
+    return {'param2': 4}
+
+def b1m2(step, sL, s):
+    return {'param1': 'a', 'param2': 2}
+def b2m2(step, sL, s):
+    return {'param1': 'b', 'param2': 4}
+
+
+def b1m3(step, sL, s):
+    return {'param1': ['c'], 'param2': np.array([10, 100])}
+def b2m3(step, sL, s):
+    return {'param1': ['d'], 'param2': np.array([20, 200])}
+
+
+# Internal States per Mechanism
+def s1m1(step, sL, s, _input):
+    y = 's1'
+    x = _input['param1']
+    return (y, x)
+def s2m1(step, sL, s, _input):
+    y = 's2'
+    x = _input['param2']
+    return (y, x)
+
+def s1m2(step, sL, s, _input):
+    y = 's1'
+    x = _input['param1']
+    return (y, x)
+def s2m2(step, sL, s, _input):
+    y = 's2'
+    x = _input['param2']
+    return (y, x)
+
+def s1m3(step, sL, s, _input):
+    y = 's1'
+    x = _input['param1']
+    return (y, x)
+def s2m3(step, sL, s, _input):
+    y = 's2'
+    x = _input['param2']
+    return (y, x)
+
+# Exogenous States
+proc_one_coef_A = 0.7
+proc_one_coef_B = 1.3
+
+def es3p1(step, sL, s, _input):
+    y = 's3'
+    x = s['s3'] * bound_norm_random(seed['a'], proc_one_coef_A, proc_one_coef_B)
+    return (y, x)
+
+def es4p2(step, sL, s, _input):
+    y = 's4'
+    x = s['s4'] * bound_norm_random(seed['b'], proc_one_coef_A, proc_one_coef_B)
+    return (y, x)
+
+ts_format = '%Y-%m-%d %H:%M:%S'
+t_delta = timedelta(days=0, minutes=0, seconds=1)
+def es5p2(step, sL, s, _input):
+    y = 'timestamp'
+    x = ep_time_step(s, dt_str=s['timestamp'], fromat_str=ts_format, _timedelta=t_delta)
+    return (y, x)
+
+
+# Environment States
+def env_a(x):
+    return 10
+def env_b(x):
+    return 10
+# def what_ever(x):
+#     return x + 1
+
+# Genesis States
+genesis_states = {
+    's1': Decimal(0.0),
+    's2': Decimal(0.0),
+    's3': Decimal(1.0),
+    's4': Decimal(1.0),
+    'timestamp': '2018-10-01 15:16:24'
+}
+
+# remove `exo_update_per_ts` to update every ts
+# why `exo_update_per_ts` here instead of `env_processes`
+exogenous_states = exo_update_per_ts(
+    {
+    "s3": es3p1,
+    "s4": es4p2,
+    "timestamp": es5p2
+    }
+)
+
+#	make env proc trigger field agnostic
+env_processes = {
+    "s3": proc_trigger('2018-10-01 15:16:25', env_a),
+    "s4": proc_trigger('2018-10-01 15:16:25', env_b)
+}
+
+# lambdas
+# genesis Sites should always be there
+# [1, 2]
+# behavior_ops = [ foldr(_ + _), lambda x: x + 0 ]
+
+
+# [1, 2] = {'b1': ['a'], 'b2', [1]} =
+# behavior_ops = [behavior_to_dict, print_fwd, sum_dict_values]
+# behavior_ops = [foldr(dict_elemwise_sum())]
+# behavior_ops = []
+
+# need at least 1 behaviour and 1 state function for the 1st mech with behaviors
+# mechanisms = {}
+mechanisms = {
+    "m1": {
+        "behaviors": {
+            "b1": b1m1, # lambda step, sL, s: s['s1'] + 1,
+            # "b2": b2m1
+        },
+        "states": { # exclude only. TypeError: reduce() of empty sequence with no initial value
+            "s1": s1m1,
+            # "s2": s2m1
+        }
+    },
+    "m2": {
+        "behaviors": {
+            "b1": b1m2,
+            # "b2": b2m2
+        },
+        "states": {
+            "s1": s1m2,
+            # "s2": s2m2
+        }
+    },
+    "m3": {
+        "behaviors": {
+            "b1": b1m3,
+            "b2": b2m3
+        },
+        "states": {
+            "s1": s1m3,
+            "s2": s2m3
+        }
+    }
+}
+
+sim_config = {
+    "N": 2,
+    "T": range(5)
+}
+
+configs.append(
+    Configuration(
+        sim_config=sim_config,
+        state_dict=genesis_states,
+        seed=seed,
+        exogenous_states=exogenous_states,
+        env_processes=env_processes,
+        mechanisms=mechanisms
+    )
+)

simulations/validation/config1.py

@@ -91,7 +91,7 @@ def env_b(x):
 #     return x + 1
 
 # Genesis States
-state_dict = {
+genesis_states = {
     's1': Decimal(0.0),
     's2': Decimal(0.0),
     's3': Decimal(1.0),
@@ -167,7 +167,7 @@ sim_config = {
 configs.append(
     Configuration(
         sim_config=sim_config,
-        state_dict=state_dict,
+        state_dict=genesis_states,
         seed=seed,
         exogenous_states=exogenous_states,
         env_processes=env_processes,

simulations/validation/config2.py

@@ -93,7 +93,7 @@ def env_b(x):
 #     return x + 1
 
 # Genesis States
-state_dict = {
+genesis_states = {
     's1': Decimal(0.0),
     's2': Decimal(0.0),
     's3': Decimal(1.0),
@@ -171,7 +171,7 @@ sim_config = {
 configs.append(
     Configuration(
         sim_config=sim_config,
-        state_dict=state_dict,
+        state_dict=genesis_states,
         seed=seed,
         exogenous_states=exogenous_states,
         env_processes=env_processes,