diff --git a/SimCAD/engine/simulation.py b/SimCAD/engine/simulation.py index 9c41d50..e5bf89c 100644 --- a/SimCAD/engine/simulation.py +++ b/SimCAD/engine/simulation.py @@ -74,19 +74,27 @@ class Executor: # make env proc trigger field agnostic self.apply_env_proc(env_processes, last_in_copy, last_in_copy['timestamp']) # mutating last_in_copy + print() + print(last_in_copy) + print() - def set_sys_metrics(m_step, t_step, run): - last_in_copy["mech_step"], last_in_copy["time_step"], last_in_copy['run'] = m_step, t_step, run + + def set_sys_metrics(state_dict, m_step, t_step, run): + state_dict["mech_step"], state_dict["time_step"], state_dict['run'] = m_step, t_step, run if any(isinstance(x, list) for x in last_in_copy.values()): last_in_copies = flatten(last_in_copy) for last_in_copy in last_in_copies: - set_sys_metrics(m_step, t_step, run) + set_sys_metrics(last_in_copy, m_step, t_step, run) sL.append(last_in_copies) else: - set_sys_metrics(m_step, t_step, run) + set_sys_metrics(last_in_copy, m_step, t_step, run) sL.append(last_in_copy) + print() + pp.pprint(last_in_copies) + print() + del last_in_copy return sL @@ -118,7 +126,7 @@ class Executor: last_states ) print() - pp.pprint(configs) + # pp.pprint(configs) else: states_lists = self.mech_step(m_step, states_list, s_conf, b_conf, env_processes, t_step, run) @@ -135,7 +143,7 @@ class Executor: def block_pipeline(self, states_list, configs, env_processes, time_seq, run): time_seq = [x + 1 for x in time_seq] simulation_list = [states_list] - print(len(configs)) + # print(len(configs)) for time_step in time_seq: # print(simulation_list) if len(simulation_list) == 1: diff --git a/notebooks/jjodesty/multithreading.ipynb b/notebooks/jjodesty/multithreading.ipynb new file mode 100644 index 0000000..c3a41ad --- /dev/null +++ b/notebooks/jjodesty/multithreading.ipynb @@ -0,0 +1,576 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "import _thread\n", + "import time\n", + "from fn.func import curried" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Define a function for the thread\n", + "def f(threadName, delay):\n", + " count = 0\n", + " print(count)\n", + " # while count < 5:\n", + " # time.sleep(delay)\n", + " # count += 1\n", + " # print(count)\n", + " \n", + "def pipe(x):\n", + " print(x)\n", + " return x" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "00\n", + "\n" + ] + } + ], + "source": [ + "# Create two threads as follows\n", + "try:\n", + " _thread.start_new_thread( f, (\"Thread-1\", 2, ) )\n", + " _thread.start_new_thread( f, (\"Thread-2\", 4, ) )\n", + "except:\n", + " print (\"Error: unable to start thread\")\n", + "\n", + "while 1:\n", + " pass\n" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[1, 2]\n", + "('s2', . at 0x1099efae8>)\n", + "('s2', . at 0x1099ef9d8>)\n" + ] + } + ], + "source": [ + "from SimCAD.engine.utils import sweep\n", + "from SimCAD.utils import rename\n", + "from SimCAD.configuration.utils import s_update\n", + "\n", + "# @curried\n", + "def fit_param(param):\n", + " return lambda x: x + param\n", + "\n", + "# xf = lambda param: lambda x: x + param\n", + "\n", + "def sweep(params, y, xf):\n", + " op = [rename('sweep', s_update(y, xf(param))) for param in params]\n", + " print(params)\n", + " # print()\n", + " return op\n", + "\n", + "for f in sweep([1,2], 's2', fit_param):\n", + " print(f(1,2,3,4))\n", + "# sweep([1,2], 's2', xf)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[1, 64, 2187, 65536]\n" + ] + } + ], + "source": [ + "# instantiate and configure the worker pool\n", + "from pathos.threading import ThreadPool\n", + "pool = ThreadPool(nodes=4)\n", + "\n", + "# do a blocking map on the chosen function\n", + "print(pool.map(pow, [1,2,3,4], [5,6,7,8]))" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "ename": "SyntaxError", + "evalue": "invalid syntax (, line 3)", + "traceback": [ + "\u001b[0;36m File \u001b[0;32m\"\"\u001b[0;36m, line \u001b[0;32m3\u001b[0m\n\u001b[0;31m [for f in fs]\u001b[0m\n\u001b[0m ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n" + ], + "output_type": "error" + } + ], + "source": [ + "with Pool(len(configs)) as p:\n", + " results = p.map(lambda t: t[0](t[1], t[2], t[3], t[4], t[5]), l)\n", + " \n", + "\n", + "def state_multithreading(self, fs, m_step, sL, last_in_obj, _input):\n", + " if type(fs) == 'list':\n", + " pool.map(f(m_step, sL, last_in_obj, _input), fs)\n", + " else:\n", + " f(m_step, sL, last_in_obj, _input)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[('s2', [11, 23])]" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from itertools import groupby\n", + "l = [('s2', 11), ('s2', 23)]\n", + "l.sort(key = lambda i : i[0])\n", + "[(key, [i[1] for i in values]) for key, values in groupby(l, lambda i: i[0])]" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "def groupByKV(l):\n", + " l.sort(key = lambda i : i[0])\n", + " return [(key, [i[1] for i in values]) for key, values in groupby(l, lambda i: i[0])]" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[('s2', [11, 23])]" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "groupByKV(l)" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "ename": "SyntaxError", + "evalue": "invalid syntax (, line 2)", + "traceback": [ + "\u001b[0;36m File \u001b[0;32m\"\"\u001b[0;36m, line \u001b[0;32m2\u001b[0m\n\u001b[0;31m collect = lambda tuplist: reduce(lambda acc, (k,v): acc[k].append(v) or acc,tuplist, defaultdict(list))\u001b[0m\n\u001b[0m ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n" + ], + "output_type": "error" + } + ], + "source": [ + "from collections import defaultdict \n", + "collect = lambda tuplist: reduce(lambda acc, (k,v): acc[k].append(v) or acc,tuplist, defaultdict(list))" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [], + "source": [ + "from collections import defaultdict\n", + "d = defaultdict(list)\n", + "for key, value in [('s2', 11), ('s2', 23)]:\n", + " d[key].append(value)" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": { + "collapsed": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Help on defaultdict object:\n", + "\n", + "class defaultdict(builtins.dict)\n", + " | defaultdict(default_factory[, ...]) --> dict with default factory\n", + " | \n", + " | The default factory is called without arguments to produce\n", + " | a new value when a key is not present, in __getitem__ only.\n", + " | A defaultdict compares equal to a dict with the same items.\n", + " | All remaining arguments are treated the same as if they were\n", + " | passed to the dict constructor, including keyword arguments.\n", + " | \n", + " | Method resolution order:\n", + " | defaultdict\n", + " | builtins.dict\n", + " | builtins.object\n", + " | \n", + " | Methods defined here:\n", + " | \n", + " | __copy__(...)\n", + " | D.copy() -> a shallow copy of D.\n", + " | \n", + " | __getattribute__(self, name, /)\n", + " | Return getattr(self, name).\n", + " | \n", + " | __init__(self, /, *args, **kwargs)\n", + " | Initialize self. See help(type(self)) for accurate signature.\n", + " | \n", + " | __missing__(...)\n", + " | __missing__(key) # Called by __getitem__ for missing key; pseudo-code:\n", + " | if self.default_factory is None: raise KeyError((key,))\n", + " | self[key] = value = self.default_factory()\n", + " | return value\n", + " | \n", + " | __reduce__(...)\n", + " | Return state information for pickling.\n", + " | \n", + " | __repr__(self, /)\n", + " | Return repr(self).\n", + " | \n", + " | copy(...)\n", + " | D.copy() -> a shallow copy of D.\n", + " | \n", + " | ----------------------------------------------------------------------\n", + " | Data descriptors defined here:\n", + " | \n", + " | default_factory\n", + " | Factory for default value called by __missing__().\n", + " | \n", + " | ----------------------------------------------------------------------\n", + " | Methods inherited from builtins.dict:\n", + " | \n", + " | __contains__(self, key, /)\n", + " | True if D has a key k, else False.\n", + " | \n", + " | __delitem__(self, key, /)\n", + " | Delete self[key].\n", + " | \n", + " | __eq__(self, value, /)\n", + " | Return self==value.\n", + " | \n", + " | __ge__(self, value, /)\n", + " | Return self>=value.\n", + " | \n", + " | __getitem__(...)\n", + " | x.__getitem__(y) <==> x[y]\n", + " | \n", + " | __gt__(self, value, /)\n", + " | Return self>value.\n", + " | \n", + " | __iter__(self, /)\n", + " | Implement iter(self).\n", + " | \n", + " | __le__(self, value, /)\n", + " | Return self<=value.\n", + " | \n", + " | __len__(self, /)\n", + " | Return len(self).\n", + " | \n", + " | __lt__(self, value, /)\n", + " | Return self size of D in memory, in bytes\n", + " | \n", + " | clear(...)\n", + " | D.clear() -> None. Remove all items from D.\n", + " | \n", + " | fromkeys(iterable, value=None, /) from builtins.type\n", + " | Returns a new dict with keys from iterable and values equal to value.\n", + " | \n", + " | get(...)\n", + " | D.get(k[,d]) -> D[k] if k in D, else d. d defaults to None.\n", + " | \n", + " | items(...)\n", + " | D.items() -> a set-like object providing a view on D's items\n", + " | \n", + " | keys(...)\n", + " | D.keys() -> a set-like object providing a view on D's keys\n", + " | \n", + " | pop(...)\n", + " | D.pop(k[,d]) -> v, remove specified key and return the corresponding value.\n", + " | If key is not found, d is returned if given, otherwise KeyError is raised\n", + " | \n", + " | popitem(...)\n", + " | D.popitem() -> (k, v), remove and return some (key, value) pair as a\n", + " | 2-tuple; but raise KeyError if D is empty.\n", + " | \n", + " | setdefault(...)\n", + " | D.setdefault(k[,d]) -> D.get(k,d), also set D[k]=d if k not in D\n", + " | \n", + " | update(...)\n", + " | D.update([E, ]**F) -> None. Update D from dict/iterable E and F.\n", + " | If E is present and has a .keys() method, then does: for k in E: D[k] = E[k]\n", + " | If E is present and lacks a .keys() method, then does: for k, v in E: D[k] = v\n", + " | In either case, this is followed by: for k in F: D[k] = F[k]\n", + " | \n", + " | values(...)\n", + " | D.values() -> an object providing a view on D's values\n", + " | \n", + " | ----------------------------------------------------------------------\n", + " | Data and other attributes inherited from builtins.dict:\n", + " | \n", + " | __hash__ = None\n", + "\n" + ] + } + ], + "source": [ + "help(d)" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'s2': [11, 23]}" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dict(d)" + ] + }, + { + "cell_type": "code", + "execution_count": 55, + "metadata": {}, + "outputs": [], + "source": [ + "def groupByKey(l):\n", + " d = defaultdict(list)\n", + " for key, value in l:\n", + " d[key].append(value)\n", + " return list(dict(d).items()).pop()" + ] + }, + { + "cell_type": "code", + "execution_count": 56, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "('s2', [11, 23])" + ] + }, + "execution_count": 56, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "r = groupByKey([('s2', 11), ('s2', 23)])\n", + "r" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# xf = lambda param: 1.0 + param\n", + "# def xf(y, param, s):\n", + "# return s[y] + param\n", + "\n", + "# def fit_param(param):\n", + "# y = 's2'\n", + "# x = 1 + param\n", + "# return lambda step, sL, s, _input: (y, x)\n", + "#\n", + "# def fit_param(param):\n", + "# return lambda step, sL, s, _input: (\n", + "# 's2',\n", + "# s['s2'] + param\n", + "# )\n", + "#\n", + "# s2m1 = sweep(\n", + "# params = [Decimal(11.0), Decimal(22.0)],\n", + "# sweep_f = fit_param\n", + "# )" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "from decimal import Decimal\n", + "from itertools import product\n", + "\n", + "# def \n", + "\n", + "l = {\n", + " 's1': 1, \n", + " 's2': [Decimal('11'), Decimal('22')], \n", + " 's3': [Decimal('12'), Decimal('23')], \n", + " 's4': 10, \n", + " 'timestamp': '2018-10-01 15:16:25', \n", + " 'mech_step': 0, \n", + " 'time_step': 1\n", + "}\n", + "\n", + "def flattenDict(l):\n", + " def tupalize(k, vs):\n", + " l = []\n", + " if isinstance(vs, list):\n", + " for v in vs:\n", + " l.append((k, v))\n", + " else:\n", + " l.append((k, vs))\n", + " return l\n", + "\n", + " flat_list = [tupalize(k, vs) for k, vs in l.items()]\n", + " flat_dict = [dict(items) for items in product(*flat_list)]\n", + " return flat_dict" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'s1': 1,\n", + " 's2': Decimal('11'),\n", + " 's3': Decimal('12'),\n", + " 's4': 10,\n", + " 'timestamp': '2018-10-01 15:16:25',\n", + " 'mech_step': 0,\n", + " 'time_step': 1},\n", + " {'s1': 1,\n", + " 's2': Decimal('11'),\n", + " 's3': Decimal('23'),\n", + " 's4': 10,\n", + " 'timestamp': '2018-10-01 15:16:25',\n", + " 'mech_step': 0,\n", + " 'time_step': 1},\n", + " {'s1': 1,\n", + " 's2': Decimal('22'),\n", + " 's3': Decimal('12'),\n", + " 's4': 10,\n", + " 'timestamp': '2018-10-01 15:16:25',\n", + " 'mech_step': 0,\n", + " 'time_step': 1},\n", + " {'s1': 1,\n", + " 's2': Decimal('22'),\n", + " 's3': Decimal('23'),\n", + " 's4': 10,\n", + " 'timestamp': '2018-10-01 15:16:25',\n", + " 'mech_step': 0,\n", + " 'time_step': 1}]" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "flattenDict(l)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.5" + } + }, + "nbformat": 4, + "nbformat_minor": 1 +}