577 lines
16 KiB
Plaintext
577 lines
16 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {
|
|
"collapsed": true
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"import _thread\n",
|
|
"import time\n",
|
|
"from fn.func import curried"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Define a function for the thread\n",
|
|
"def f(threadName, delay):\n",
|
|
" count = 0\n",
|
|
" print(count)\n",
|
|
" # while count < 5:\n",
|
|
" # time.sleep(delay)\n",
|
|
" # count += 1\n",
|
|
" # print(count)\n",
|
|
" \n",
|
|
"def pipe(x):\n",
|
|
" print(x)\n",
|
|
" return x"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 3,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"00\n",
|
|
"\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"# Create two threads as follows\n",
|
|
"try:\n",
|
|
" _thread.start_new_thread( f, (\"Thread-1\", 2, ) )\n",
|
|
" _thread.start_new_thread( f, (\"Thread-2\", 4, ) )\n",
|
|
"except:\n",
|
|
" print (\"Error: unable to start thread\")\n",
|
|
"\n",
|
|
"while 1:\n",
|
|
" pass\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 11,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"[1, 2]\n",
|
|
"('s2', <function fit_param.<locals>.<lambda> at 0x1099efae8>)\n",
|
|
"('s2', <function fit_param.<locals>.<lambda> at 0x1099ef9d8>)\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"from SimCAD.engine.utils import sweep\n",
|
|
"from SimCAD.utils import rename\n",
|
|
"from SimCAD.configuration.utils import s_update\n",
|
|
"\n",
|
|
"# @curried\n",
|
|
"def fit_param(param):\n",
|
|
" return lambda x: x + param\n",
|
|
"\n",
|
|
"# xf = lambda param: lambda x: x + param\n",
|
|
"\n",
|
|
"def sweep(params, y, xf):\n",
|
|
" op = [rename('sweep', s_update(y, xf(param))) for param in params]\n",
|
|
" print(params)\n",
|
|
" # print()\n",
|
|
" return op\n",
|
|
"\n",
|
|
"for f in sweep([1,2], 's2', fit_param):\n",
|
|
" print(f(1,2,3,4))\n",
|
|
"# sweep([1,2], 's2', xf)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 5,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"[1, 64, 2187, 65536]\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"# instantiate and configure the worker pool\n",
|
|
"from pathos.threading import ThreadPool\n",
|
|
"pool = ThreadPool(nodes=4)\n",
|
|
"\n",
|
|
"# do a blocking map on the chosen function\n",
|
|
"print(pool.map(pow, [1,2,3,4], [5,6,7,8]))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 2,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"ename": "SyntaxError",
|
|
"evalue": "invalid syntax (<ipython-input-2-6e999d313015>, line 3)",
|
|
"traceback": [
|
|
"\u001b[0;36m File \u001b[0;32m\"<ipython-input-2-6e999d313015>\"\u001b[0;36m, line \u001b[0;32m3\u001b[0m\n\u001b[0;31m [for f in fs]\u001b[0m\n\u001b[0m ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n"
|
|
],
|
|
"output_type": "error"
|
|
}
|
|
],
|
|
"source": [
|
|
"with Pool(len(configs)) as p:\n",
|
|
" results = p.map(lambda t: t[0](t[1], t[2], t[3], t[4], t[5]), l)\n",
|
|
" \n",
|
|
"\n",
|
|
"def state_multithreading(self, fs, m_step, sL, last_in_obj, _input):\n",
|
|
" if type(fs) == 'list':\n",
|
|
" pool.map(f(m_step, sL, last_in_obj, _input), fs)\n",
|
|
" else:\n",
|
|
" f(m_step, sL, last_in_obj, _input)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 12,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"[('s2', [11, 23])]"
|
|
]
|
|
},
|
|
"execution_count": 12,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"from itertools import groupby\n",
|
|
"l = [('s2', 11), ('s2', 23)]\n",
|
|
"l.sort(key = lambda i : i[0])\n",
|
|
"[(key, [i[1] for i in values]) for key, values in groupby(l, lambda i: i[0])]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 17,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def groupByKV(l):\n",
|
|
" l.sort(key = lambda i : i[0])\n",
|
|
" return [(key, [i[1] for i in values]) for key, values in groupby(l, lambda i: i[0])]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 18,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"[('s2', [11, 23])]"
|
|
]
|
|
},
|
|
"execution_count": 18,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"groupByKV(l)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 20,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"ename": "SyntaxError",
|
|
"evalue": "invalid syntax (<ipython-input-20-fada0ccd8d2a>, line 2)",
|
|
"traceback": [
|
|
"\u001b[0;36m File \u001b[0;32m\"<ipython-input-20-fada0ccd8d2a>\"\u001b[0;36m, line \u001b[0;32m2\u001b[0m\n\u001b[0;31m collect = lambda tuplist: reduce(lambda acc, (k,v): acc[k].append(v) or acc,tuplist, defaultdict(list))\u001b[0m\n\u001b[0m ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n"
|
|
],
|
|
"output_type": "error"
|
|
}
|
|
],
|
|
"source": [
|
|
"from collections import defaultdict \n",
|
|
"collect = lambda tuplist: reduce(lambda acc, (k,v): acc[k].append(v) or acc,tuplist, defaultdict(list))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 23,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from collections import defaultdict\n",
|
|
"d = defaultdict(list)\n",
|
|
"for key, value in [('s2', 11), ('s2', 23)]:\n",
|
|
" d[key].append(value)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 25,
|
|
"metadata": {
|
|
"collapsed": true
|
|
},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Help on defaultdict object:\n",
|
|
"\n",
|
|
"class defaultdict(builtins.dict)\n",
|
|
" | defaultdict(default_factory[, ...]) --> dict with default factory\n",
|
|
" | \n",
|
|
" | The default factory is called without arguments to produce\n",
|
|
" | a new value when a key is not present, in __getitem__ only.\n",
|
|
" | A defaultdict compares equal to a dict with the same items.\n",
|
|
" | All remaining arguments are treated the same as if they were\n",
|
|
" | passed to the dict constructor, including keyword arguments.\n",
|
|
" | \n",
|
|
" | Method resolution order:\n",
|
|
" | defaultdict\n",
|
|
" | builtins.dict\n",
|
|
" | builtins.object\n",
|
|
" | \n",
|
|
" | Methods defined here:\n",
|
|
" | \n",
|
|
" | __copy__(...)\n",
|
|
" | D.copy() -> a shallow copy of D.\n",
|
|
" | \n",
|
|
" | __getattribute__(self, name, /)\n",
|
|
" | Return getattr(self, name).\n",
|
|
" | \n",
|
|
" | __init__(self, /, *args, **kwargs)\n",
|
|
" | Initialize self. See help(type(self)) for accurate signature.\n",
|
|
" | \n",
|
|
" | __missing__(...)\n",
|
|
" | __missing__(key) # Called by __getitem__ for missing key; pseudo-code:\n",
|
|
" | if self.default_factory is None: raise KeyError((key,))\n",
|
|
" | self[key] = value = self.default_factory()\n",
|
|
" | return value\n",
|
|
" | \n",
|
|
" | __reduce__(...)\n",
|
|
" | Return state information for pickling.\n",
|
|
" | \n",
|
|
" | __repr__(self, /)\n",
|
|
" | Return repr(self).\n",
|
|
" | \n",
|
|
" | copy(...)\n",
|
|
" | D.copy() -> a shallow copy of D.\n",
|
|
" | \n",
|
|
" | ----------------------------------------------------------------------\n",
|
|
" | Data descriptors defined here:\n",
|
|
" | \n",
|
|
" | default_factory\n",
|
|
" | Factory for default value called by __missing__().\n",
|
|
" | \n",
|
|
" | ----------------------------------------------------------------------\n",
|
|
" | Methods inherited from builtins.dict:\n",
|
|
" | \n",
|
|
" | __contains__(self, key, /)\n",
|
|
" | True if D has a key k, else False.\n",
|
|
" | \n",
|
|
" | __delitem__(self, key, /)\n",
|
|
" | Delete self[key].\n",
|
|
" | \n",
|
|
" | __eq__(self, value, /)\n",
|
|
" | Return self==value.\n",
|
|
" | \n",
|
|
" | __ge__(self, value, /)\n",
|
|
" | Return self>=value.\n",
|
|
" | \n",
|
|
" | __getitem__(...)\n",
|
|
" | x.__getitem__(y) <==> x[y]\n",
|
|
" | \n",
|
|
" | __gt__(self, value, /)\n",
|
|
" | Return self>value.\n",
|
|
" | \n",
|
|
" | __iter__(self, /)\n",
|
|
" | Implement iter(self).\n",
|
|
" | \n",
|
|
" | __le__(self, value, /)\n",
|
|
" | Return self<=value.\n",
|
|
" | \n",
|
|
" | __len__(self, /)\n",
|
|
" | Return len(self).\n",
|
|
" | \n",
|
|
" | __lt__(self, value, /)\n",
|
|
" | Return self<value.\n",
|
|
" | \n",
|
|
" | __ne__(self, value, /)\n",
|
|
" | Return self!=value.\n",
|
|
" | \n",
|
|
" | __new__(*args, **kwargs) from builtins.type\n",
|
|
" | Create and return a new object. See help(type) for accurate signature.\n",
|
|
" | \n",
|
|
" | __setitem__(self, key, value, /)\n",
|
|
" | Set self[key] to value.\n",
|
|
" | \n",
|
|
" | __sizeof__(...)\n",
|
|
" | D.__sizeof__() -> size of D in memory, in bytes\n",
|
|
" | \n",
|
|
" | clear(...)\n",
|
|
" | D.clear() -> None. Remove all items from D.\n",
|
|
" | \n",
|
|
" | fromkeys(iterable, value=None, /) from builtins.type\n",
|
|
" | Returns a new dict with keys from iterable and values equal to value.\n",
|
|
" | \n",
|
|
" | get(...)\n",
|
|
" | D.get(k[,d]) -> D[k] if k in D, else d. d defaults to None.\n",
|
|
" | \n",
|
|
" | items(...)\n",
|
|
" | D.items() -> a set-like object providing a view on D's items\n",
|
|
" | \n",
|
|
" | keys(...)\n",
|
|
" | D.keys() -> a set-like object providing a view on D's keys\n",
|
|
" | \n",
|
|
" | pop(...)\n",
|
|
" | D.pop(k[,d]) -> v, remove specified key and return the corresponding value.\n",
|
|
" | If key is not found, d is returned if given, otherwise KeyError is raised\n",
|
|
" | \n",
|
|
" | popitem(...)\n",
|
|
" | D.popitem() -> (k, v), remove and return some (key, value) pair as a\n",
|
|
" | 2-tuple; but raise KeyError if D is empty.\n",
|
|
" | \n",
|
|
" | setdefault(...)\n",
|
|
" | D.setdefault(k[,d]) -> D.get(k,d), also set D[k]=d if k not in D\n",
|
|
" | \n",
|
|
" | update(...)\n",
|
|
" | D.update([E, ]**F) -> None. Update D from dict/iterable E and F.\n",
|
|
" | If E is present and has a .keys() method, then does: for k in E: D[k] = E[k]\n",
|
|
" | If E is present and lacks a .keys() method, then does: for k, v in E: D[k] = v\n",
|
|
" | In either case, this is followed by: for k in F: D[k] = F[k]\n",
|
|
" | \n",
|
|
" | values(...)\n",
|
|
" | D.values() -> an object providing a view on D's values\n",
|
|
" | \n",
|
|
" | ----------------------------------------------------------------------\n",
|
|
" | Data and other attributes inherited from builtins.dict:\n",
|
|
" | \n",
|
|
" | __hash__ = None\n",
|
|
"\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"help(d)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 27,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"{'s2': [11, 23]}"
|
|
]
|
|
},
|
|
"execution_count": 27,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"dict(d)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 55,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def groupByKey(l):\n",
|
|
" d = defaultdict(list)\n",
|
|
" for key, value in l:\n",
|
|
" d[key].append(value)\n",
|
|
" return list(dict(d).items()).pop()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 56,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"('s2', [11, 23])"
|
|
]
|
|
},
|
|
"execution_count": 56,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"r = groupByKey([('s2', 11), ('s2', 23)])\n",
|
|
"r"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# xf = lambda param: 1.0 + param\n",
|
|
"# def xf(y, param, s):\n",
|
|
"# return s[y] + param\n",
|
|
"\n",
|
|
"# def fit_param(param):\n",
|
|
"# y = 's2'\n",
|
|
"# x = 1 + param\n",
|
|
"# return lambda step, sL, s, _input: (y, x)\n",
|
|
"#\n",
|
|
"# def fit_param(param):\n",
|
|
"# return lambda step, sL, s, _input: (\n",
|
|
"# 's2',\n",
|
|
"# s['s2'] + param\n",
|
|
"# )\n",
|
|
"#\n",
|
|
"# s2m1 = sweep(\n",
|
|
"# params = [Decimal(11.0), Decimal(22.0)],\n",
|
|
"# sweep_f = fit_param\n",
|
|
"# )"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 14,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from decimal import Decimal\n",
|
|
"from itertools import product\n",
|
|
"\n",
|
|
"# def \n",
|
|
"\n",
|
|
"l = {\n",
|
|
" 's1': 1, \n",
|
|
" 's2': [Decimal('11'), Decimal('22')], \n",
|
|
" 's3': [Decimal('12'), Decimal('23')], \n",
|
|
" 's4': 10, \n",
|
|
" 'timestamp': '2018-10-01 15:16:25', \n",
|
|
" 'mech_step': 0, \n",
|
|
" 'time_step': 1\n",
|
|
"}\n",
|
|
"\n",
|
|
"def flattenDict(l):\n",
|
|
" def tupalize(k, vs):\n",
|
|
" l = []\n",
|
|
" if isinstance(vs, list):\n",
|
|
" for v in vs:\n",
|
|
" l.append((k, v))\n",
|
|
" else:\n",
|
|
" l.append((k, vs))\n",
|
|
" return l\n",
|
|
"\n",
|
|
" flat_list = [tupalize(k, vs) for k, vs in l.items()]\n",
|
|
" flat_dict = [dict(items) for items in product(*flat_list)]\n",
|
|
" return flat_dict"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 15,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"[{'s1': 1,\n",
|
|
" 's2': Decimal('11'),\n",
|
|
" 's3': Decimal('12'),\n",
|
|
" 's4': 10,\n",
|
|
" 'timestamp': '2018-10-01 15:16:25',\n",
|
|
" 'mech_step': 0,\n",
|
|
" 'time_step': 1},\n",
|
|
" {'s1': 1,\n",
|
|
" 's2': Decimal('11'),\n",
|
|
" 's3': Decimal('23'),\n",
|
|
" 's4': 10,\n",
|
|
" 'timestamp': '2018-10-01 15:16:25',\n",
|
|
" 'mech_step': 0,\n",
|
|
" 'time_step': 1},\n",
|
|
" {'s1': 1,\n",
|
|
" 's2': Decimal('22'),\n",
|
|
" 's3': Decimal('12'),\n",
|
|
" 's4': 10,\n",
|
|
" 'timestamp': '2018-10-01 15:16:25',\n",
|
|
" 'mech_step': 0,\n",
|
|
" 'time_step': 1},\n",
|
|
" {'s1': 1,\n",
|
|
" 's2': Decimal('22'),\n",
|
|
" 's3': Decimal('23'),\n",
|
|
" 's4': 10,\n",
|
|
" 'timestamp': '2018-10-01 15:16:25',\n",
|
|
" 'mech_step': 0,\n",
|
|
" 'time_step': 1}]"
|
|
]
|
|
},
|
|
"execution_count": 15,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"flattenDict(l)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": []
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.6.5"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 1
|
|
}
|