init

.gitignore

@@ -3,6 +3,7 @@ jupyter notebook
 .ipynb_checkpoints
 .DS_Store
 .idea
+.pytest_cache
 notebooks
 *.egg-info
 __pycache__

cadCAD/configuration/__init__.py

@@ -1,3 +1,4 @@
+from copy import deepcopy
 from typing import Dict, Callable, List, Tuple
 from functools import reduce
 import pandas as pd
@@ -11,9 +12,9 @@ from cadCAD.configuration.utils.depreciationHandler import sanitize_partial_stat
 
 
 class Configuration(object):
-    def __init__(self, sim_config={}, initial_state={}, seeds={}, env_processes={},
+    def __init__(self, user_id, sim_config={}, initial_state={}, seeds={}, env_processes={},
                  exogenous_states={}, partial_state_update_blocks={}, policy_ops=[lambda a, b: a + b],
-                 **kwargs) -> None:
+                 session_id=0, simulation_id=0, run_id=1, **kwargs) -> None:
         # print(exogenous_states)
         self.sim_config = sim_config
         self.initial_state = initial_state
@@ -24,11 +25,18 @@ class Configuration(object):
         self.policy_ops = policy_ops
         self.kwargs = kwargs
 
+        self.user_id = user_id
+        self.session_id = session_id
+        self.simulation_id = simulation_id
+        self.run_id = run_id
+
         sanitize_config(self)
 
 
-def append_configs(sim_configs={}, initial_state={}, seeds={}, raw_exogenous_states={}, env_processes={},
+def append_configs(user_id='cadCAD_user', session_id=0, #ToDo: change to string
-                   partial_state_update_blocks={}, policy_ops=[lambda a, b: a + b], _exo_update_per_ts: bool = True) -> None:
+                   sim_configs={}, initial_state={}, seeds={}, raw_exogenous_states={}, env_processes={},
+                   partial_state_update_blocks={}, policy_ops=[lambda a, b: a + b], _exo_update_per_ts: bool = True
+                  ) -> None:
     if _exo_update_per_ts is True:
         exogenous_states = exo_update_per_ts(raw_exogenous_states)
     else:
@@ -37,7 +45,27 @@ def append_configs(sim_configs={}, initial_state={}, seeds={}, raw_exogenous_sta
     if isinstance(sim_configs, dict):
         sim_configs = [sim_configs]
 
-    for sim_config in sim_configs:
+    new_sim_configs = []
+    for t in list(zip(sim_configs, list(range(len(sim_configs))))):
+        sim_config, simulation_id = t[0], t[1]
+        N = sim_config['N']
+        if N > 1:
+            for n in range(N):
+                sim_config['simulation_id'] = simulation_id
+                sim_config['run_id'] = n
+                sim_config['N'] = 1
+                new_sim_configs.append(deepcopy(sim_config))
+            del sim_config
+        else:
+            sim_config['simulation_id'] = simulation_id
+            sim_config['run_id'] = 0
+            new_sim_configs.append(deepcopy(sim_config))
+
+    print(new_sim_configs)
+    print()
+
+    # for sim_config in sim_configs:
+    for sim_config in new_sim_configs:
         config = Configuration(
             sim_config=sim_config,
             initial_state=initial_state,
@@ -45,10 +73,15 @@ def append_configs(sim_configs={}, initial_state={}, seeds={}, raw_exogenous_sta
             exogenous_states=exogenous_states,
             env_processes=env_processes,
             partial_state_update_blocks=partial_state_update_blocks,
-            policy_ops=policy_ops
+            policy_ops=policy_ops,
+
+            user_id=user_id,
+            session_id=session_id,
+            simulation_id=sim_config['simulation_id'],
+            run_id=sim_config['run_id']
         )
-        print(sim_configs)
+        # print(sim_configs)
-        #for each sim config create new config
+        # for each sim config create new config
         configs.append(config)
 
 

cadCAD/engine/__init__.py

@@ -1,3 +1,4 @@
+from pprint import pprint
 from typing import Callable, Dict, List, Any, Tuple
 from pathos.multiprocessing import ProcessingPool as PPool
 from pandas.core.frame import DataFrame
@@ -25,11 +26,18 @@ def single_proc_exec(
         configs_structs: List[ConfigsType],
         env_processes_list: List[EnvProcessesType],
         Ts: List[range],
-        Ns: List[int]
+        Ns: List[int],
+        userIDs,
+        sessionIDs,
+        simulationIDs,
+        runIDs: List[int],
     ):
-    l = [simulation_execs, states_lists, configs_structs, env_processes_list, Ts, Ns]
+    params = [simulation_execs, states_lists, configs_structs, env_processes_list, Ts, Ns,
-    simulation_exec, states_list, config, env_processes, T, N = list(map(lambda x: x.pop(), l))
+              userIDs, sessionIDs, simulationIDs, runIDs]
-    result = simulation_exec(var_dict_list, states_list, config, env_processes, T, N)
+    simulation_exec, states_list, config, env_processes, T, N, user_id, session_id, simulation_id, run_id = \
+        list(map(lambda x: x.pop(), params))
+    result = simulation_exec(var_dict_list, states_list, config, env_processes, T, N,
+                             user_id, session_id, simulation_id, run_id)
     return flatten(result)
 
 
@@ -40,11 +48,16 @@ def parallelize_simulations(
         configs_structs: List[ConfigsType],
         env_processes_list: List[EnvProcessesType],
         Ts: List[range],
-        Ns: List[int]
+        Ns: List[int],
+        userIDs,
+        sessionIDs,
+        simulationIDs,
+        runIDs: List[int],
     ):
-    l = list(zip(simulation_execs, var_dict_list, states_lists, configs_structs, env_processes_list, Ts, Ns))
+    params = list(zip(simulation_execs, var_dict_list, states_lists, configs_structs, env_processes_list, Ts, Ns,
+                      userIDs, sessionIDs, simulationIDs, runIDs))
     with PPool(len(configs_structs)) as p:
-        results = p.map(lambda t: t[0](t[1], t[2], t[3], t[4], t[5], t[6]), l)
+        results = p.map(lambda t: t[0](t[1], t[2], t[3], t[4], t[5], t[6], t[7], t[8], t[9], t[10]), params)
     return results
 
 
@@ -70,7 +83,6 @@ class Executor:
         config_proc = Processor()
         create_tensor_field = TensorFieldReport(config_proc).create_tensor_field
 
-
         print(r'''
                             __________   ____ 
           ________ __ _____/ ____/   |  / __ \
@@ -82,19 +94,27 @@ class Executor:
         print(f'Execution Mode: {self.exec_context + ": " + str(self.configs)}')
         print(f'Configurations: {self.configs}')
 
-        var_dict_list, states_lists, Ts, Ns, eps, configs_structs, env_processes_list, partial_state_updates, simulation_execs = \
+        userIDs, sessionIDs, simulationIDs, runIDs, \
-            [], [], [], [], [], [], [], [], []
+        var_dict_list, states_lists, \
+        Ts, Ns, \
+        eps, configs_structs, env_processes_list, \
+        partial_state_updates, simulation_execs = \
+            [], [], [], [], [], [], [], [], [], [], [], [], []
         config_idx = 0
 
         for x in self.configs:
+            userIDs.append(x.user_id)
+            sessionIDs.append(x.session_id)
+            simulationIDs.append(x.simulation_id)
+            runIDs.append(x.run_id)
 
             Ts.append(x.sim_config['T'])
             Ns.append(x.sim_config['N'])
+
             var_dict_list.append(x.sim_config['M'])
             states_lists.append([x.initial_state])
             eps.append(list(x.exogenous_states.values()))
             configs_structs.append(config_proc.generate_config(x.initial_state, x.partial_state_updates, eps[config_idx]))
-            # print(env_processes_list)
             env_processes_list.append(x.env_processes)
             partial_state_updates.append(x.partial_state_updates)
             simulation_execs.append(SimExecutor(x.policy_ops).simulation)
@@ -105,11 +125,17 @@ class Executor:
 
         if self.exec_context == ExecutionMode.single_proc:
             tensor_field = create_tensor_field(partial_state_updates.pop(), eps.pop())
-            result = self.exec_method(simulation_execs, var_dict_list, states_lists, configs_structs, env_processes_list, Ts, Ns)
+            result = self.exec_method(
+                simulation_execs, var_dict_list, states_lists, configs_structs, env_processes_list, Ts, Ns,
+                userIDs, sessionIDs, simulationIDs, runIDs
+            )
             final_result = result, tensor_field
         elif self.exec_context == ExecutionMode.multi_proc:
             # if len(self.configs) > 1:
-            simulations = self.exec_method(simulation_execs, var_dict_list, states_lists, configs_structs, env_processes_list, Ts, Ns)
+            simulations = self.exec_method(
+                simulation_execs, var_dict_list, states_lists, configs_structs, env_processes_list, Ts, Ns,
+                userIDs, sessionIDs, simulationIDs, runIDs
+            )
             results = []
             for result, partial_state_updates, ep in list(zip(simulations, partial_state_updates, eps)):
                 results.append((flatten(result), create_tensor_field(partial_state_updates, ep)))

cadCAD/engine/simulation.py

@@ -202,7 +202,11 @@ class Executor:
             configs: List[Tuple[List[Callable], List[Callable]]],
             env_processes: Dict[str, Callable],
             time_seq: range,
-            runs: int
+            runs: int,
+            user_id,
+            session_id,
+            simulation_id,
+            run_id
         ) -> List[List[Dict[str, Any]]]:
 
         def execute_run(sweep_dict, states_list, configs, env_processes, time_seq, run) -> List[Dict[str, Any]]:
@@ -211,6 +215,7 @@ class Executor:
             def generate_init_sys_metrics(genesis_states_list):
                 for d in genesis_states_list:
                     d['run'], d['substep'], d['timestep'] = run, 0, 0
+                    d['user_id'], d['simulation_id'], d['session_id'], d['run_id'] = user_id, simulation_id, session_id, run_id
                     yield d
 
             states_list_copy: List[Dict[str, Any]] = list(generate_init_sys_metrics(deepcopy(states_list)))
@@ -222,6 +227,8 @@ class Executor:
 
             return first_timestep_per_run
 
+        # print(type(run_id))
+        # print(runs)
         tp = TPool(runs)
         pipe_run: List[List[Dict[str, Any]]] = flatten(
             tp.map(

cadCAD/utils/__init__.py

@@ -17,6 +17,14 @@ def append_dict(dict, new_dict):
     return dict
 
 
+def arrange_cols(df, reverse):
+    session_metrics = ['user_id', 'session_id', 'simulation_id', 'run_id']
+    sys_metrics = ['run', 'timestep', 'substep']
+    result_cols = list(set(df.columns) - set(session_metrics) - set(sys_metrics))
+    result_cols.sort(reverse=reverse)
+    return df[session_metrics + sys_metrics + result_cols]
+
+
 class IndexCounter:
     def __init__(self):
         self.i = 0

simulations/regression_tests/config1.py

@@ -152,6 +152,7 @@ sim_config = config_sim(
 )
 
 append_configs(
+    user_id='user_a',
     sim_configs=sim_config,
     initial_state=genesis_states,
     env_processes=env_processes,

simulations/regression_tests/config2.py

@@ -140,6 +140,7 @@ sim_config = config_sim(
 )
 
 append_configs(
+    user_id='user_b',
     sim_configs=sim_config,
     initial_state=genesis_states,
     env_processes=env_processes,

simulations/regression_tests/sweep_config.py

@@ -144,6 +144,7 @@ sim_config = config_sim(
 # New Convention
 partial_state_update_blocks = psub_list(psu_block, psu_steps)
 append_configs(
+    user_id='user_a',
     sim_configs=sim_config,
     initial_state=genesis_states,
     seeds=seeds,

simulations/test_executions/config1_test.py

@@ -3,6 +3,7 @@ from typing import List
 from tabulate import tabulate
 # The following imports NEED to be in the exact order
 from cadCAD.engine import ExecutionMode, ExecutionContext, Executor
+from cadCAD.utils import arrange_cols
 from simulations.regression_tests import config1
 from cadCAD import configs
 
@@ -14,8 +15,11 @@ first_config = configs # only contains config1
 single_proc_ctx = ExecutionContext(context=exec_mode.single_proc)
 run = Executor(exec_context=single_proc_ctx, configs=first_config)
 
+# print(set(result.columns) - set(['user_id', 'session_id', 'simulation_id', 'run_id']) - set(['run', 'timestep', 'substep']))
+# print(['run', 'timestep', 'substep'])
+
 raw_result, tensor_field = run.execute()
-result = pd.DataFrame(raw_result)
+result = arrange_cols(pd.DataFrame(raw_result), False)
 print()
 print("Tensor Field: config1")
 # print(raw_result)

simulations/test_executions/config2_test.py

@@ -2,6 +2,7 @@ import pandas as pd
 from tabulate import tabulate
 # The following imports NEED to be in the exact order
 from cadCAD.engine import ExecutionMode, ExecutionContext, Executor
+from cadCAD.utils import arrange_cols
 from simulations.regression_tests import config2
 from cadCAD import configs
 
@@ -9,15 +10,15 @@ exec_mode = ExecutionMode()
 
 print("Simulation Execution: Single Configuration")
 print()
-first_config = configs # only contains config2
+single_proc_ctx = ExecutionContext(context=exec_mode.multi_proc)
-single_proc_ctx = ExecutionContext(context=exec_mode.single_proc)
+run = Executor(exec_context=single_proc_ctx, configs=configs)
-run = Executor(exec_context=single_proc_ctx, configs=first_config)
 
-raw_result, tensor_field = run.execute()
+for raw_result, tensor_field in run.execute():
-result = pd.DataFrame(raw_result)
+    result = arrange_cols(pd.DataFrame(raw_result), False)
-print()
+    print()
-print("Tensor Field: config1")
+    # print("Tensor Field: " + config_names[i])
-print(tabulate(tensor_field, headers='keys', tablefmt='psql'))
+    print(tabulate(tensor_field, headers='keys', tablefmt='psql'))
-print("Output:")
+    print("Output:")
-print(tabulate(result, headers='keys', tablefmt='psql'))
+    print(tabulate(result, headers='keys', tablefmt='psql'))
-print()
+    print()
+    # i += 1

simulations/test_executions/multi_config_test.py

@@ -2,6 +2,7 @@ import pandas as pd
 from tabulate import tabulate
 # The following imports NEED to be in the exact order
 from cadCAD.engine import ExecutionMode, ExecutionContext, Executor
+from cadCAD.utils import arrange_cols
 from simulations.regression_tests import config1, config2
 from cadCAD import configs
 
@@ -15,9 +16,9 @@ run = Executor(exec_context=multi_proc_ctx, configs=configs)
 i = 0
 config_names = ['config1', 'config2']
 for raw_result, tensor_field in run.execute():
-    result = pd.DataFrame(raw_result)
+    result = arrange_cols(pd.DataFrame(raw_result), False)
     print()
-    print(f"Tensor Field: {config_names[i]}")
+    # print(f"Tensor Field: {config_names[i]}")
     print(tabulate(tensor_field, headers='keys', tablefmt='psql'))
     print("Output:")
     print(tabulate(result, headers='keys', tablefmt='psql'))

simulations/test_executions/param_sweep_test.py

@@ -14,11 +14,11 @@ multi_proc_ctx = ExecutionContext(context=exec_mode.multi_proc)
 run = Executor(exec_context=multi_proc_ctx, configs=configs)
 
 i = 0
-config_names = ['sweep_config_A', 'sweep_config_B']
+# config_names = ['sweep_config_A', 'sweep_config_B']
 for raw_result, tensor_field in run.execute():
     result = pd.DataFrame(raw_result)
     print()
-    print("Tensor Field: " + config_names[i])
+    # print("Tensor Field: " + config_names[i])
     print(tabulate(tensor_field, headers='keys', tablefmt='psql'))
     print("Output:")
     print(tabulate(result, headers='keys', tablefmt='psql'))