This commit is contained in:
Joshua E. Jodesty 2018-12-11 10:25:25 -05:00
parent a0266641f7
commit e2752161c3
2 changed files with 590 additions and 6 deletions

View File

@ -74,19 +74,27 @@ class Executor:
# make env proc trigger field agnostic
self.apply_env_proc(env_processes, last_in_copy, last_in_copy['timestamp']) # mutating last_in_copy
print()
print(last_in_copy)
print()
def set_sys_metrics(m_step, t_step, run):
last_in_copy["mech_step"], last_in_copy["time_step"], last_in_copy['run'] = m_step, t_step, run
def set_sys_metrics(state_dict, m_step, t_step, run):
state_dict["mech_step"], state_dict["time_step"], state_dict['run'] = m_step, t_step, run
if any(isinstance(x, list) for x in last_in_copy.values()):
last_in_copies = flatten(last_in_copy)
for last_in_copy in last_in_copies:
set_sys_metrics(m_step, t_step, run)
set_sys_metrics(last_in_copy, m_step, t_step, run)
sL.append(last_in_copies)
else:
set_sys_metrics(m_step, t_step, run)
set_sys_metrics(last_in_copy, m_step, t_step, run)
sL.append(last_in_copy)
print()
pp.pprint(last_in_copies)
print()
del last_in_copy
return sL
@ -118,7 +126,7 @@ class Executor:
last_states
)
print()
pp.pprint(configs)
# pp.pprint(configs)
else:
states_lists = self.mech_step(m_step, states_list, s_conf, b_conf, env_processes, t_step, run)
@ -135,7 +143,7 @@ class Executor:
def block_pipeline(self, states_list, configs, env_processes, time_seq, run):
time_seq = [x + 1 for x in time_seq]
simulation_list = [states_list]
print(len(configs))
# print(len(configs))
for time_step in time_seq:
# print(simulation_list)
if len(simulation_list) == 1:

View File

@ -0,0 +1,576 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"import _thread\n",
"import time\n",
"from fn.func import curried"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Define a function for the thread\n",
"def f(threadName, delay):\n",
" count = 0\n",
" print(count)\n",
" # while count < 5:\n",
" # time.sleep(delay)\n",
" # count += 1\n",
" # print(count)\n",
" \n",
"def pipe(x):\n",
" print(x)\n",
" return x"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"00\n",
"\n"
]
}
],
"source": [
"# Create two threads as follows\n",
"try:\n",
" _thread.start_new_thread( f, (\"Thread-1\", 2, ) )\n",
" _thread.start_new_thread( f, (\"Thread-2\", 4, ) )\n",
"except:\n",
" print (\"Error: unable to start thread\")\n",
"\n",
"while 1:\n",
" pass\n"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[1, 2]\n",
"('s2', <function fit_param.<locals>.<lambda> at 0x1099efae8>)\n",
"('s2', <function fit_param.<locals>.<lambda> at 0x1099ef9d8>)\n"
]
}
],
"source": [
"from SimCAD.engine.utils import sweep\n",
"from SimCAD.utils import rename\n",
"from SimCAD.configuration.utils import s_update\n",
"\n",
"# @curried\n",
"def fit_param(param):\n",
" return lambda x: x + param\n",
"\n",
"# xf = lambda param: lambda x: x + param\n",
"\n",
"def sweep(params, y, xf):\n",
" op = [rename('sweep', s_update(y, xf(param))) for param in params]\n",
" print(params)\n",
" # print()\n",
" return op\n",
"\n",
"for f in sweep([1,2], 's2', fit_param):\n",
" print(f(1,2,3,4))\n",
"# sweep([1,2], 's2', xf)"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[1, 64, 2187, 65536]\n"
]
}
],
"source": [
"# instantiate and configure the worker pool\n",
"from pathos.threading import ThreadPool\n",
"pool = ThreadPool(nodes=4)\n",
"\n",
"# do a blocking map on the chosen function\n",
"print(pool.map(pow, [1,2,3,4], [5,6,7,8]))"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"ename": "SyntaxError",
"evalue": "invalid syntax (<ipython-input-2-6e999d313015>, line 3)",
"traceback": [
"\u001b[0;36m File \u001b[0;32m\"<ipython-input-2-6e999d313015>\"\u001b[0;36m, line \u001b[0;32m3\u001b[0m\n\u001b[0;31m [for f in fs]\u001b[0m\n\u001b[0m ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n"
],
"output_type": "error"
}
],
"source": [
"with Pool(len(configs)) as p:\n",
" results = p.map(lambda t: t[0](t[1], t[2], t[3], t[4], t[5]), l)\n",
" \n",
"\n",
"def state_multithreading(self, fs, m_step, sL, last_in_obj, _input):\n",
" if type(fs) == 'list':\n",
" pool.map(f(m_step, sL, last_in_obj, _input), fs)\n",
" else:\n",
" f(m_step, sL, last_in_obj, _input)"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[('s2', [11, 23])]"
]
},
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"from itertools import groupby\n",
"l = [('s2', 11), ('s2', 23)]\n",
"l.sort(key = lambda i : i[0])\n",
"[(key, [i[1] for i in values]) for key, values in groupby(l, lambda i: i[0])]"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {},
"outputs": [],
"source": [
"def groupByKV(l):\n",
" l.sort(key = lambda i : i[0])\n",
" return [(key, [i[1] for i in values]) for key, values in groupby(l, lambda i: i[0])]"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[('s2', [11, 23])]"
]
},
"execution_count": 18,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"groupByKV(l)"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {},
"outputs": [
{
"ename": "SyntaxError",
"evalue": "invalid syntax (<ipython-input-20-fada0ccd8d2a>, line 2)",
"traceback": [
"\u001b[0;36m File \u001b[0;32m\"<ipython-input-20-fada0ccd8d2a>\"\u001b[0;36m, line \u001b[0;32m2\u001b[0m\n\u001b[0;31m collect = lambda tuplist: reduce(lambda acc, (k,v): acc[k].append(v) or acc,tuplist, defaultdict(list))\u001b[0m\n\u001b[0m ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n"
],
"output_type": "error"
}
],
"source": [
"from collections import defaultdict \n",
"collect = lambda tuplist: reduce(lambda acc, (k,v): acc[k].append(v) or acc,tuplist, defaultdict(list))"
]
},
{
"cell_type": "code",
"execution_count": 23,
"metadata": {},
"outputs": [],
"source": [
"from collections import defaultdict\n",
"d = defaultdict(list)\n",
"for key, value in [('s2', 11), ('s2', 23)]:\n",
" d[key].append(value)"
]
},
{
"cell_type": "code",
"execution_count": 25,
"metadata": {
"collapsed": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Help on defaultdict object:\n",
"\n",
"class defaultdict(builtins.dict)\n",
" | defaultdict(default_factory[, ...]) --> dict with default factory\n",
" | \n",
" | The default factory is called without arguments to produce\n",
" | a new value when a key is not present, in __getitem__ only.\n",
" | A defaultdict compares equal to a dict with the same items.\n",
" | All remaining arguments are treated the same as if they were\n",
" | passed to the dict constructor, including keyword arguments.\n",
" | \n",
" | Method resolution order:\n",
" | defaultdict\n",
" | builtins.dict\n",
" | builtins.object\n",
" | \n",
" | Methods defined here:\n",
" | \n",
" | __copy__(...)\n",
" | D.copy() -> a shallow copy of D.\n",
" | \n",
" | __getattribute__(self, name, /)\n",
" | Return getattr(self, name).\n",
" | \n",
" | __init__(self, /, *args, **kwargs)\n",
" | Initialize self. See help(type(self)) for accurate signature.\n",
" | \n",
" | __missing__(...)\n",
" | __missing__(key) # Called by __getitem__ for missing key; pseudo-code:\n",
" | if self.default_factory is None: raise KeyError((key,))\n",
" | self[key] = value = self.default_factory()\n",
" | return value\n",
" | \n",
" | __reduce__(...)\n",
" | Return state information for pickling.\n",
" | \n",
" | __repr__(self, /)\n",
" | Return repr(self).\n",
" | \n",
" | copy(...)\n",
" | D.copy() -> a shallow copy of D.\n",
" | \n",
" | ----------------------------------------------------------------------\n",
" | Data descriptors defined here:\n",
" | \n",
" | default_factory\n",
" | Factory for default value called by __missing__().\n",
" | \n",
" | ----------------------------------------------------------------------\n",
" | Methods inherited from builtins.dict:\n",
" | \n",
" | __contains__(self, key, /)\n",
" | True if D has a key k, else False.\n",
" | \n",
" | __delitem__(self, key, /)\n",
" | Delete self[key].\n",
" | \n",
" | __eq__(self, value, /)\n",
" | Return self==value.\n",
" | \n",
" | __ge__(self, value, /)\n",
" | Return self>=value.\n",
" | \n",
" | __getitem__(...)\n",
" | x.__getitem__(y) <==> x[y]\n",
" | \n",
" | __gt__(self, value, /)\n",
" | Return self>value.\n",
" | \n",
" | __iter__(self, /)\n",
" | Implement iter(self).\n",
" | \n",
" | __le__(self, value, /)\n",
" | Return self<=value.\n",
" | \n",
" | __len__(self, /)\n",
" | Return len(self).\n",
" | \n",
" | __lt__(self, value, /)\n",
" | Return self<value.\n",
" | \n",
" | __ne__(self, value, /)\n",
" | Return self!=value.\n",
" | \n",
" | __new__(*args, **kwargs) from builtins.type\n",
" | Create and return a new object. See help(type) for accurate signature.\n",
" | \n",
" | __setitem__(self, key, value, /)\n",
" | Set self[key] to value.\n",
" | \n",
" | __sizeof__(...)\n",
" | D.__sizeof__() -> size of D in memory, in bytes\n",
" | \n",
" | clear(...)\n",
" | D.clear() -> None. Remove all items from D.\n",
" | \n",
" | fromkeys(iterable, value=None, /) from builtins.type\n",
" | Returns a new dict with keys from iterable and values equal to value.\n",
" | \n",
" | get(...)\n",
" | D.get(k[,d]) -> D[k] if k in D, else d. d defaults to None.\n",
" | \n",
" | items(...)\n",
" | D.items() -> a set-like object providing a view on D's items\n",
" | \n",
" | keys(...)\n",
" | D.keys() -> a set-like object providing a view on D's keys\n",
" | \n",
" | pop(...)\n",
" | D.pop(k[,d]) -> v, remove specified key and return the corresponding value.\n",
" | If key is not found, d is returned if given, otherwise KeyError is raised\n",
" | \n",
" | popitem(...)\n",
" | D.popitem() -> (k, v), remove and return some (key, value) pair as a\n",
" | 2-tuple; but raise KeyError if D is empty.\n",
" | \n",
" | setdefault(...)\n",
" | D.setdefault(k[,d]) -> D.get(k,d), also set D[k]=d if k not in D\n",
" | \n",
" | update(...)\n",
" | D.update([E, ]**F) -> None. Update D from dict/iterable E and F.\n",
" | If E is present and has a .keys() method, then does: for k in E: D[k] = E[k]\n",
" | If E is present and lacks a .keys() method, then does: for k, v in E: D[k] = v\n",
" | In either case, this is followed by: for k in F: D[k] = F[k]\n",
" | \n",
" | values(...)\n",
" | D.values() -> an object providing a view on D's values\n",
" | \n",
" | ----------------------------------------------------------------------\n",
" | Data and other attributes inherited from builtins.dict:\n",
" | \n",
" | __hash__ = None\n",
"\n"
]
}
],
"source": [
"help(d)"
]
},
{
"cell_type": "code",
"execution_count": 27,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'s2': [11, 23]}"
]
},
"execution_count": 27,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"dict(d)"
]
},
{
"cell_type": "code",
"execution_count": 55,
"metadata": {},
"outputs": [],
"source": [
"def groupByKey(l):\n",
" d = defaultdict(list)\n",
" for key, value in l:\n",
" d[key].append(value)\n",
" return list(dict(d).items()).pop()"
]
},
{
"cell_type": "code",
"execution_count": 56,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"('s2', [11, 23])"
]
},
"execution_count": 56,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"r = groupByKey([('s2', 11), ('s2', 23)])\n",
"r"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# xf = lambda param: 1.0 + param\n",
"# def xf(y, param, s):\n",
"# return s[y] + param\n",
"\n",
"# def fit_param(param):\n",
"# y = 's2'\n",
"# x = 1 + param\n",
"# return lambda step, sL, s, _input: (y, x)\n",
"#\n",
"# def fit_param(param):\n",
"# return lambda step, sL, s, _input: (\n",
"# 's2',\n",
"# s['s2'] + param\n",
"# )\n",
"#\n",
"# s2m1 = sweep(\n",
"# params = [Decimal(11.0), Decimal(22.0)],\n",
"# sweep_f = fit_param\n",
"# )"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [],
"source": [
"from decimal import Decimal\n",
"from itertools import product\n",
"\n",
"# def \n",
"\n",
"l = {\n",
" 's1': 1, \n",
" 's2': [Decimal('11'), Decimal('22')], \n",
" 's3': [Decimal('12'), Decimal('23')], \n",
" 's4': 10, \n",
" 'timestamp': '2018-10-01 15:16:25', \n",
" 'mech_step': 0, \n",
" 'time_step': 1\n",
"}\n",
"\n",
"def flattenDict(l):\n",
" def tupalize(k, vs):\n",
" l = []\n",
" if isinstance(vs, list):\n",
" for v in vs:\n",
" l.append((k, v))\n",
" else:\n",
" l.append((k, vs))\n",
" return l\n",
"\n",
" flat_list = [tupalize(k, vs) for k, vs in l.items()]\n",
" flat_dict = [dict(items) for items in product(*flat_list)]\n",
" return flat_dict"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[{'s1': 1,\n",
" 's2': Decimal('11'),\n",
" 's3': Decimal('12'),\n",
" 's4': 10,\n",
" 'timestamp': '2018-10-01 15:16:25',\n",
" 'mech_step': 0,\n",
" 'time_step': 1},\n",
" {'s1': 1,\n",
" 's2': Decimal('11'),\n",
" 's3': Decimal('23'),\n",
" 's4': 10,\n",
" 'timestamp': '2018-10-01 15:16:25',\n",
" 'mech_step': 0,\n",
" 'time_step': 1},\n",
" {'s1': 1,\n",
" 's2': Decimal('22'),\n",
" 's3': Decimal('12'),\n",
" 's4': 10,\n",
" 'timestamp': '2018-10-01 15:16:25',\n",
" 'mech_step': 0,\n",
" 'time_step': 1},\n",
" {'s1': 1,\n",
" 's2': Decimal('22'),\n",
" 's3': Decimal('23'),\n",
" 's4': 10,\n",
" 'timestamp': '2018-10-01 15:16:25',\n",
" 'mech_step': 0,\n",
" 'time_step': 1}]"
]
},
"execution_count": 15,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"flattenDict(l)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.5"
}
},
"nbformat": 4,
"nbformat_minor": 1
}