From 8cf3b095822568f3130e1d55d9a22e8161460f3e Mon Sep 17 00:00:00 2001 From: "Joshua E. Jodesty" Date: Thu, 3 Oct 2019 11:28:58 -0400 Subject: [PATCH] readme pt. 2: new build --- .gitignore | 3 +- distroduce/README.md | 88 ++++++++++++++++++ distroduce/action_policies.py | 30 +++++- distroduce/configuration/cluster.py | 46 --------- .../configuration/launch_local_kafka.sh | 4 + distroduce/dist/distroduce.zip | Bin 20561 -> 31821 bytes distroduce/emr/launch.py | 39 ++++++++ distroduce/local_messaging_sim.py | 14 +-- distroduce/messaging_sim.py | 10 +- distroduce/simulation.py | 77 +++++++++++++++ distroduce/state_updates.py | 28 +++++- 11 files changed, 276 insertions(+), 63 deletions(-) diff --git a/.gitignore b/.gitignore index c15c07f..40ae22d 100644 --- a/.gitignore +++ b/.gitignore @@ -30,4 +30,5 @@ Simulation.md monkeytype.sqlite3 distributed_produce/bash/ -distributed_produce/notes.txt \ No newline at end of file +distributed_produce/notes.txt +notes.txt \ No newline at end of file diff --git a/distroduce/README.md b/distroduce/README.md index e69de29..a057d01 100644 --- a/distroduce/README.md +++ b/distroduce/README.md @@ -0,0 +1,88 @@ +``` + ___ _ __ _ __ __ __ + / _ \ (_)___ / /_ ____ (_)/ / __ __ / /_ ___ ___/ / + / // // /(_-6S~ySqbhm*DR1PH=bE!QBVf;7)J|mJnQnyGGF99uoK`+1+QeoA2F! zxX#7VyZh9suCBVPdgdj>@>r`VlIuulvk_4z0iWyGjA^p$u*q|f=*ygWLdeHEQ5)vwLPKom%19(ZG`+mBS z23&@brwSB1=L`J0W zY55H8MW0T`=QloiCpY!YIpn31W`hJy{>$kdE}!?V(y^?a@+uDrhH1q=G-h;mmit%l z)tuJY0XJ7K}#}Q zczrlWu$0(17$`ju(I{#?p-#{2iW3EJX3&-Ls2b_pTsF~P#TfAE4BqI}wq6niw!-jW zs!ydlc%6J(mgSVE`C61}RxlEt3GjWMh+q*;YyEuP$_Lvc$covCqHS6S2yw!9Ih=X? zs#UWy&@Ts}YSQ-sDLsE|VBZTf?&Eu9zih%bsT+&xgj84k`nEy%Le8%q21Ma=x$oiA zg|qV4_FXV50x`&D;DYEh?Vu8_Tk{Z!MygWAzPhn<#OS<39p8EJ6gotHrqJ;suFx&$ z9jVZpGdET^h%N`>>fI0iSHOFg{(NVaZ|iuQEF40+ht#*R_YO}EZw%4S?%C_Re9r+o z5_l`!@NPGd7uz(4-iyGIdMbvyU18}CU)McpXjZDAm+-`sQPDUEiTpfldz@8{Y4UX+ zDf~?HvebCAxG9U2slz2hHLJxs{bIXXeR!zsR42InnGA|hcBy0Quz@?B2CR7n0&ffq z{F{ucqZTV}b*dTEHy4qV{7#s}U!I~bp^M=hPKT46xpJ9~ukecY#B|Bsd8ewr1e8L< zuuTBeL$6FO)*Rlld%iN~RkHdZht_wM>LASB=68IEt0N3?VMf9Bd{g4>>PYtIuOTpX z#ubyRe5COoKRWQ^qc{VtW2&`*@<(VdZZ=LRfRBwsDF*d?tFRUPuV!`04of8gg=JkKZ`p)%8O zE~+w5U+~4=Tuo${vC_Q90;wWvL4gM@s1i8~HB(7Kg;@^jh7A$`5QYIMqtF7C!6Tm; zY!&`BuIn_FoY%(-{B-tR z+7;S7j7Y#DhN|-5EtXL4&6_9|=aUIt9J%n8QJbu5v@oXL!qqU@ya#$1??2#{%YyiH*WW$Q*#sOXV-uqm!vNRVGK%9zjl{wPW`5 zX(h-W^&QRcbbW+`u)>K(Qnk#=9bq5@J_k2Kts-ivp3cp!)W7lc`}#^b-z)iIb-V5= z!Iw|0LgifqJL>Jn^{0L}MDt0qmpMoYCi$_JB9Kq;Q*~H%Qe`}{Y>%Nq9GtxH-<<;M zW-|4GmzNAydCX+U>l07~%FicIuBzG`q=W7*(G3NQK(}^ak$rPU>stBh)JO{#K|FfJu9Os07Ws7e@AoKacV=DpY z!qMD@`A^?=lx{f?NCydk`mc6pzGSO)ff zHWO!%Mk4q}Vic&Rg)L&mr7~OB_+^(tscj`r-R#!w(ahD2ebg!PeGpSkB0vTNjwo{o zOyPcZIZO)43+{bS>98kep{u630#X?R=)F;xEL`cR3}pLR2gnJ+x#l);OjL^09CrGr zUGf#HfU57~!6&MSQTj24FK=x6p9-hq7#J~MazB4QP)lb(Fs^UDRL+-{{&`eX*QhJW zz3fG4CCl=l%c(Nai=?U1Ic}g=o64EG;tRFD9e{>5dF$$3VyZo(DYx{4a*_|tz-2wC z%BstlcBVHs`NcDfk>rHPWcVJ?T2sdfnj1d+sM5$b*37~Db2p7Iq|uDGVJ zQ0of`dNbE-_Eh3P_8G5a?Su;yaKqYteqpHL#7%@%s$)!|%~dgnU%@qaA6tYJ&X=>H z*l)HxyS9>-AqgmaR;R#U)S=;j4V^5th>`Z>vr;L)_TY-LVyLefd&p{I)YQsWSUv zXW!r%!?y<7yJlos)|7Y`$*1@ZRo?@eL9Jn%&Atg6^T~vx2M5O4fhFJH*c;4G)e24p=iv&W%!2nHSQ~ar>4Zo@@s24{Q z$_6$AgoC^MkMoFltN%ELz}NqAU$ZejA;d2ilpx>THE$AxLH=?M78FEm^7H0x;w;!d zFA###sgnP9lL#FmXjKjsq)9FEv-J$>`Tt!5yJVm!8ZpR`^5!ub3I?yh*dJHotMjqsQKgwk2kh2x{8psCmhG>Wb0_kbbXkqD&KGmduhJdnE| zX?zxR-qZ(zn(SeQkBqA3Pp8|FugG;dD%^v;NV_33_CDd zk1yip3-d6$1~Lgpf)e?}nxCIV;VfHY?)ti{>yl4L$ON|!8#`}vkpu?si7CVPD6Dya z3f6C!%TSuS{H4-Skec5JT&$(4ZVhe}to!3{_HSqAs84?Fsll|bIoKSwD|*$|$1U6g z(_sCH6OD~QuQ)@Gq#zcU3B8TlHmVS7m;gCy0k7zaMt=&02{7!x;Gkm&W|e!G*^?BR zxQk=my<+$66u|O)cEFP@#U_5D=Mbi!H;Y8OLX+QmxKCO8vC|U)_VHi{E$%z_8wr*> z8iEo)vK4!UEW6;j&_3o@j6ggUydWcWeWlAU;bbc_VXW6m1J+nTwYYAe$I>NYrApj6b_dgrgdJrnQi1u!nql^BldaoN53^y)Al>o? zfc)xE8Ez~!^R((x_5NKD(U7prK97(&`#x3V6H2F8J9$As%N;zEG53<0o?J55l)^x( z3+%?{A>Ll3uOLPsJNNg4q*pomz>?)K`yD4U4U+tCbYU_zuvyzD>*sui1c6eJ2{lA+B|z$1!sPaRufhEGk5oIuTXvP@J;0ITy)xs z=lx&+glP6*{s4_C)-A5mc8@he4K@~j2p}7dXFz&~bR$Oa_OB_qM@!6kjUO5838Hr` z_ZgZGM*)PeH`Xls`%zExL*J@7tm4=yL{d95$3EVsq<&eo&Bg6MgQkX;x}cH@vLGz+ zuZ)nO^M_tRTkX(OCS2T%-PKmCe#I2P+eymEC6`)in-f5w^|@3Vxa|}kM)8apzCpc} zB=k%lNqN0E^rMiibR2AWwHQA~EyD(y_y*|tl}(*ZqiibB86A1GkqT7UbhI5U8YxrN zP>$q{<;r5`@GalQAgu1Jic3Rz1z7m1*MZ#jGaBLv7fp-{z6V{+gF4y`;9ZM((pZSo zwrb4RuZQkxwhJR(KrYf+^#0&#mi{s!?9po*WPi3BuNbe_N0Pu=m}ufe1Nk#5;#)Wq znlU%Oyvb_iR3QBkar|N+sePIR{Q_GjF%yiWgnn%@2XE$+O+>CEqz7{6%rl=e$in#z zG@Ci*FOU<$Ky@VwJ%OzYUtPIP9t)|AxGVLkq<&<3)hSrxnwArHxIaK-$I^AIh$8Y5vGRIj^JQr znfPVox(oDJ6ELLc$>=E^TWl{g^Efm$l@;(4?>`U)Ua)B`6{yW5t2i4VWvU43g_H{$ z=SW=cgg^;K0->1%8y?Rkkh>t)?t{_{nR8DN{BEo6KD~b&NbHT-DRG;rmq?eIlq~wb zJ4&CL%$!l`uvyPbhdls4p+RqDcQ=z^x03M5fj70LA^nBE0y(oxH9=J*l?_AE2+JurGr|7iP!kAA9@lEb#0~`+)+t4zpCxl?k8In`B>)L|q!b zsJL>hB&hmQ>WS<J0J9{v6h3r{n>zLDj5ox+Vpfpe@&ep!h<%cDFm>#i=F&92xb z&6I&eW}RjY6kgn_BVtAVzI@!kyH%^)TjlaspBbrM=Z#8PZ#ANvmOk`Y$|gSTv6Q;h zXvuOUe9PAI@q&C@?-g81|6KeH=7)FjgA;)Vkh~UG;e=-*+62Afq6W%Yr5hc8`)yzy zXDJ}Q6GfM-F~^0#usU$7-$sF*!SaCsJ`2S^YkogvZ=a`5DF>|+yBM}3HoGCmGdBnE zqJPc}zy}~Gc9o_g&`BsY>snZTk$`oRwX(^k^lqD9QH0-#kA3f(3t=fvxW(eH`^2~UbKeyqf0p3bMg^`jdbiw? zSU6sVNBP;VjY?FzuwL|=BCOA2k0R1X-&n)%OqocOq$LDlLkTTkT`zIh(Y6vprrtF- zn8aLqyo|FUfeydJ#`~W%npe4d;edaMrc_x%1=U-{^f}n@K!WKid;l@P!q(lx&Dp}! z+>+JZ*3r|!)Wg=4lafj$=j2_giuj*3fVxA) zW%cc4zc*$A_9YE7b;T~}>w;zA1kJtBLYVaF*n&*n;uA= zq}e_Tt!CR##$C)MB2>^st)GRB+g;^JN62TGF`{RqujffsMzKu|*z9|SixG4iy$zud z>u%M+c?bPcPpz-Ab$EsY9-9nDj!<`(Rd2VaY?qKzvr5R#A)!C|db*a*dE%W#cRDpp zW;s%7$F6RJSmL>4VX$@X-Ix~sW*cx4qFw>6C9@mDW@b;sKDQ%_Ic#V!h4La~d0&C)!4_`D^!Nuj_=Wg5TZCA(pJoE_vYqrktJ`z9%M25~jJvoc4ok$$5@kxDmcP z_-5LWYy|khHYG^q>cT@IDe0J*ByjRbQsNl28lK4%VLB{xF(})+o@4JHS5^+pYCT<4 zf$XsA?tqKc-WAHuep%VIO1r5sk3x3I6!{|3{An?vdP}6ABr}Aj)RVl18Ss)?K-W}% z_)`jY!Y)UIaA$samFvuWQwto8j`h<;5Oz+`0mYIikeXRwT#W8HL`Mdpx?jf_4JB) zIDMmf&gZ(FHh;Ajwzd+h@C{3@n6O-M*BK4^P)CcCpl_LY;?6Uo4ScI zo_-TxUixe#mK`mkae;@zM>qv9Lf1!WlNk3WQxsl32AEF`H3Tz7cA_>w4GvTCPqOXi z?rvpIkq*--P(vQS}uV9N}GD zDYsr{6G{fP^)7x=s2lVJVsK7nphY^BL#PZBm&1N-_A_!Em1z5PoMPRP#2?Yh>N zbY88+S1a})Gt#$iHM@yViZ&qqcR^S+Y zZ*yxwz#nS&N0I^o9IpOIR{ZrN_@~;LJ3Cp~T6?g<9&bJ{(NQ}Z- z_hp#BsdxFh=*~(KC6+>#2X~eyk$3;qT8Gx3J;jOA@_kVx?4cFRf*(@uDvX>ZF+PJ# zjB)Bgl}LO4vl!Rj=`6Ir2iJI(xQY?PdS)kha4mob7sGFFiTL+xf*!f?VVVEv88Ht& z@K&eeBt!w%{4xc*3Qk-#Rvj~3jny%H@#aOFo`2nU@}lUcO}(`M>SyYSLcZJu?VP7Q z+21c#;&e}>H0y+CtUrs==IcT(h?+U2!5(~jl!RwSoHDD<9!Yms;k>)arxnM(&aNxs zkAO*6p@IAmtXfAC7Z3ZHtL=ou!69+Y8~6m@%8?cihjjs+8_IED-0{)SHfNYE-E6c& ziAc#RZiY>F{cS2?y)|SBo|8>6@fBYmv=*sc&BBT3`%~$<;mfB z^%UXDvgN$(>RIn7HD)_QohoD~Iuz^uT}EcueMDU6LfdziuF^;dF)@8L@(ucw zsjES%^CoLy4ictr$2VQFlGta^49wpnXitIOPY=W{5v*^{kr(${3Zsq`tqnnJJ} zETS5Sjy%5Od(-3faJ*ftoa2mWuK?7CK5-(`&`5s3T@PyTUnj|D(y9^bk6PIu_{5*% znZuelHvd)s{gA4xOSHK{UdrW@h8swXk8Dy{@>LEa^?p~^@Yup#nu_Hi`ga{V=B}|5 zoREM7-knfva&D+k9fOI+2WscS{rw6QrNU4K(x=)A)vX$PmEP>m8XNJw4amTSf)GsU zA+~LsdD7xC2=DgH5JKkYSbA$;dFqo+^IKr>Oao#;FBA$#2l!*qNNQ4MWda0@WsjTA9WP!L{~!ucplos zMr;QlBH|TuD8o3i%;_skiJl1*4q2#aCTL#Q0;191KaY7KwWKHuXKl~XOEw`B*B8J} z7VaIl({J(BoQq6keMvA=_P=$V)y>oCuhA&&8}oBCF0=T` zDT+&|uocdw?SuWp7svnr{eNLtTRNGTIXioR-A7ZGztGV4(v5`n(m}U;bfj+)lFbZ= z_*No|8Ikq*4f5zyXsY~V>x?tCjhE$*MWm?AS|C&Vo%lX@~gzaGatac424SAwM$uphK^R#EB3eJT+m7$yH1 z-|~4ttp0<H_sm+2y5)>doeoqfy;N{nf(l%PQPPa#7 zOb#C_3de;Gm+d802U-`w+n}U}Q7tzIr4BQg)Hd#{ca%iW+_anh(ICBZ&wFwt! zbkpy6``Re>A-bdkmMUS=>MDTd$7#6PGy+57@AmP4=4E-Fkw@w7vTN*t{@;zemd+7j zTleT1`e_B-$7*~U=@w%|^?Xoz`-(hCeFaLf7ljaIBY4l0k1%)E%5QqRi~@+3guQF1 zBP*78P)14)93Zb*u}b85hN>zV)Rcs-&PQL-;nAU$pJjVjaj*nJbhSPZ)4&=#V~9V8 z8MFxm!ZUU15Bovz(5Vky3FP2K8%yn4{t+Qoc)liR5d9_bAcD zOl+a91+Q&ROG_F+NiEGYmvjq4nd)qPv1odbR6F+=Qr04W@M3?JLc!elqOAH(kT@eq zy%G~N#B`6@aYAwA;m;J9EsRkRW#4=?SQO;|e0p*juxjr&zDrELDTtm->v8b!pc`?5 zS$h7Zsio(1S4!YjS@}sx`Gs?9Fs@;x`9f6L63LSIF59^f{m1+5ef&-2?{Xbic<&)g z@rj&p-_lJLYQ}nh_I(me7&}4+C130c(SEAnr>jAiZ^?KN<|ffoQ{TH2|F-z(-y`74 z0JL3wH~}KY>vYd!I)-NIzMjFasg;#ysb;QvT7$1Fj`Z~8T**_n>_JUmh(~H8%?z!N zN;P1}hvCI>;0d<3+n7N$0t&F;dzv8Kju@1oJ3ZHfixcbUFIf~|4NJO$a$!qe*SGty zvnjOS-#1C^eil%Ng={~E%woJ}bO{$I40OssyR)aQZf#EZ;8nV6fo@qgpHt>P+>jg%7oDxNg@SGKXDHQZW#q3jAN_ ztk@*3a_uO`=LnuK$>9M2s{b1Q<_@0j9+qx@g^JGl=|-;L?O&ndq?VEMDjRm-`&Xht zu#h3M46~%qBuT1m)Iu4wv?3JU5J+Qy`S&GZTHkI5DH=)+i>R)msH99Dw8 zA_{^B8%2fj_>Z$?{9>z+fDeJpEI$5q4I zkC9@4WrZlLjW(fsnSt&5iB55UgSp9Ho6|b`6mz4Pt#P{7S5A)|D(DMxQ{8G>pyhRI zi+ZKUp_`Y98+VD!k{EYZfWS$6JH{@>ypNC>F=1U~y1~Y`xT+16EdA&6z(JOyO7G5s z4$V(*ybWX)TuBPu1NpCZ6CaX#$s$z_1KQ6~7I}mF$0_$b>h2O4k!8sM*w~YZk4f#(|rEPblt){t;iCkJl z?o&Z6As;NELxiJ0AqK^50p+J7_|3X$T)!e4p)0zQMXK>LCrY+n&lx=Y5?uZ8a&tVD zM|QyO=_t5phUNdQ&cHw1TnAH6CvzKm8zZaBkmjtE zJ|c!V;3iC>Y@S=GtIr~YbKHlZ<>}yIyVxkRu*Omktrx`CrrMl^=@nqIX1VbI9PiS# z4=B@8;Tz5~FNk9N+n0`R5LcnaejDu$mSqrld-ng93<7NYO)bE zlnotrJHLXj#C`{Xn3%W|!(4ZVq1CXZ?DZS!#k;f4=RtGrTb|15d-w`-CSP1S3uM;F zDlDbsh=m9g$ov{%6;U{j`d_%KArfE_3^HmhUTM_tvZ$ctz(_6mcHQ4FXBWfq8Ob72 zQMbLG-FXW5nH&Fu8(cIHSK(m5$Cd}bfyw{RTd>nnUOsVqV&7vOKnl~#+ZkSW~xA%Kml2s8jd z`yY~H{hN*Zt3_y~|EC=5G=MX9Yhc5AtFu1SBrGQns_^P^Y)3PvvSbIonclJJ^`M}vuW&K`9^d2nKxD|5~>10t~_4VU0l^g#G_(w zOi(|6R`m!jHG6MZ3Kd-uh)Q>I?|pw6Iir2N`I@_Un0&cLsf!J-QQU?*_k%8Zpi6i1 z&f8vLk6V$km?=`>4e6;|t~L~L50i-$4930ay`t|SI8%j5J70qWxjP8maH3L;4h1n^ zE60=oN7;xJ%Q@>M1-#Gz=Wjvl)FMAM{Ex_hV%H$v6Jn+YJ z>u-?By}(-94~~`pMGgOW?hga`*L?p2AnNzprPd#{OJHjFK?nZ04;B5nLD;eC%cvJf z4Q7tKEw3HJ$j4hsv@?(cOZP*+-Kw>LhE}}R0-6SaGQAZtvrV=M*8?5&z8=yla;Dwc zbm`nE=d5+}+$lnzr1=Y} zj(40zzon|m-_r-w)FCjF!j~}W!Ho;N{lr0W=b>0`)i;JQh@O}PLsm#Nq>F-(>eY%-;d*B+AG}y0Urrwb5y5!)h8z0J0@;Z1n7Ycb2%wQZG z3whA>?L*+B6|{KCGr|bZnWHaBZTX8E-XS+eKzs#$+Zo%F4%3hGAgcn4i)|FoP)1^A zy9CIl?jCC-!N7gA6`4vO)koo>L{79Y$tW?xxkn!BiV0OdYOVsOctjLcIv}L4K-)~n zhHT3x@-NP2qL<31SxwmA^N9}xp{(Sol|S6|wMpNx^iFb6;Up1mOsaN>cNnmG{3MQkpo^pDroI45VJN;I;RX_}Rji>*x0O*BDHhB?_Z^kD! zP^%rPp~zNmf!k=>ibdxctudB2pd|Dmy(M>Ez3`6anC6;`Tx^v*fs5LdzJ_M9MM~c` zw`C6Xpp?i0ifYsBkr=8T%IOky>qQ@N^N@e0!$ojJ?WjJ`-CwMAkLReneFWFJ0cD$4 z56ObX#o(={9+Ag}dP &J*%b9v)rfBN-V_po5Ue6khH z5Hg4gDDJ=q&dLt1NVaRR(7=3;&Ft*k8nElTCMJ=0_bVzjP}5eNPfhQXBOuBkWKe#M zS|Se5X5t!@HV|3FXji%xDw7?&2d_L%WHJevKGqv}MJm+YRg5}%u?tezQww+idJGn3d>bwE_v>>m)E<`$$BVFq z=m4N8x}E?COX&S(b9U&pRW(oROzJXGsiAI}j`#F1W!Rxh9%Ey(5;0L-@piGl=zkZW zX5wavG*7{~Kb8H`w^(u6r?MSMEC?)MXIU8BhD#CJOGyojZi1t;?pKV5WAYbDK@?AC zDTTrXLKQYret;sVpgq{O7I|}rO-$2(V`&?-&ijgo#YdL-E#`Bi_CvtqY#k6HgmZg# zK4*2p?8bm=Xu@~Ttn_TYAotPTdMa0>Vt?#w@YFO&B&gBqWJ06Ot$w}g0fN)nsgdwf zFCV71?_Fbik`?#oNepR?sMoAa=kmVQJlBeoz~-B-*~1pE&&^YCf|nJ-_Ft&p?g)2U zE+$Zw3bxs{jJdbg@UMmXFDmy3c{^@BJqeF|&rKcl=f9nv@LhQ#P=_B~6Q7~3K^yqrB&D98tjrLD9p4Ci#Mx2ue4xjah9JIk+# zl@nMgq^1}zx78A(DY(R~7muuV6Tgq&Vp?$6y^WX3_c|aun77qBkE2q;%u4oMVfkSR zJ3JJ24qKG9JgHIS79IoZJa0Q$$c4-tK5(g+%r@@|(lq>Cxbo5+5h)Ivsp5hyqjhJH zG)~~NXYtVO#Cq>o$cncsy(XaiJU{3Ra}^Idz{-xrqqzj4;McQ$lx8OKBynPApvnY)IBnfw-a|G)I>?F^#g^c zAx_?Pf(!L;8akZiKZvefh-**zKhI}l7|h&_oE>klTMsJwAeq5Vu@O;u(THw|mWe?P zxAc9(WTN4-4@NjqwYN)baAZ1X%v@X+Z`ta(Y2zD zrYESsT~_cK1{Dsrgtzjr0Dvhv0D%2ZCqi9HOj1dT#nIxY;ry}E;R?L{D>`b`adcjj zMt-a}>7(d3LrQ4@wmqnue@~{hO{php;jR?q?x}iu_?h>H!Y$Ov&rPE6Vc@0hz z#?<;#Bn8|N%dz6d0RXdHWprnBj&FR6$&gKKgWV8Le9%B1@)roy^z`z!kZ??FS$i5T z=+p5}xur9&DyfuKW!@onSr){~>d4E$AkeE|e(gWKeI!u_7DW5*!Vhz83E3bE2&}Y) zP;7Pwd7VmlmG?M0;1e`CZwftcLcBe{4CH5XaM%=j{TbKTw#ybeZ}2Wd;i&D_D{zDE zZZ!}o9_ZrY!r*A7d7M^nC59haF~(n#kpOybh@a1!J)oBrzT>H^Ab4py`$nbT1#v7i zSY4lS5wn&7m@A$vjjo02WbVQ<~n2;=a&13f$ z3c8qP4KTjs&MXU`Ip*npQYeZgXmJmebIIRt%0M;h&~SpK_lWz z15g&DbsU|=w**b#L!ZgnYXOz=?BMr!j( zD**AX7uBs>!vmogDhb|b09UxF=nV2viTec+Si;4L0F_3eDF=0A)G_#^Sn`0)hEvA z8`$b@p(RRd!5=^Hj$RD8WnaA?NoJK{iw`CXag|z#m<=ilT=|I1gii3SqEUr0%L}22 zC0jJpqe1Cl{ec* z;YF-4x23Aq^Cv*Y5bLK-m9EJxLP{Qe zB(!*!HG3Lfn%f&LlA%#CS7%1MUr!}wn?^G3yi*){r2_l5-^x6q4)PQJOKokdB#oU_RCs}`vPD!Hsr-{1!V?&_-2+0P z`aB=I(;*bq5_Y1s%kr`;C0?;il$gJwz9~i#P1CTqUX^q2CsVdIe-7;Dkgu-yiSYT3 z#~)Z-#|gWGjRbfaHMaWR=hyCFG>)3-%xW%o`CFNfPFTMA%)X>|;$;5+bIeYfm}t&p}@XA_yJ;2gQ&;f3+R-jRZWWc4AHC_wCTSfFEBamcJ6yDuP5e*D7$A;}|x zn&iYGr6~VAG>6~?vB_bBYH8p>aq{92v40-nfNteQArGj3;wUO;L(0G3g z6#pophXVk(|0kp$lBA#mfY|Bzen|38prB?(l;0R4gE9 zK@h5fpaxlJ&`V)h@!#w5epQ|^LBM}h=Ycy0uFm_>mcs8vM8CpliT(rT4}ROx(%s$E z+SbY11RNFrq;$Y|;1Z)hRTNg@_X?3;#k?&356piSduVBbvA~@9&%XU$Ch}|F7$yD# z=kKBsFdn#QC2;vq=>2i~6q|6h5lAJr&eU?=9Ez#wiVbOy42r~sYJ{|yGR zQexEjJzf55U$*|^an|3H=ish@ljlF)QR(+I+^@I{ivM@FSpRytAN8AHL~v5>2cqup znUG%*uay4-@&8DRfWiO##OlAN27Uz>QvDBbcNbGPdlPq0Ge=vGpM*CU7MwWv(P#DF zx%yvWG1dM9_BZAp3=3xNf558#PRIWW+xuUz|03tXpkQ(y4Eoa)J^!6~{R8+9!$ha~ zkN*C_!-5gOH0qCTiT}>O{@N{<-=F@4jr|*j;y32&A4Zf<=Z6u+NBo-!1D%Uv|1}+2 t{2uoG+J`X1f4spjK_IyOA3@;10zg#-7;yLqUh4|}BPIaPjOIU%{y*(;U>pDd delta 5078 zcmZu#2UHZxww(w_78pQ=oCZ`R=bSTx2&l}E1j(6^BsgRcP(lkRB6%buNstUmkR(AQ zDToXrDnZE-KX|YA|NGwWUcGwPuIf{@PS>g0rv`VyBROCyV|^%$9CUik5mb|@xJV*5 z0L!dx;1d{e<}aSKNHy7npX7x5o38>Gp-?+(RiF$iMAGa3H_cIV1T}+^4V$W>yYK9w z01TN1e#suNj6bn=D}Vw7hHZgCAQ;0*PXx0z&-YpktsK0X`nAh)9@xU-_j$T@LgPsU zb(`AfO>njNJTgvBTBljvB*VD%8Np;|V?33FPPgAc63DV~d#K_0l`kGo_GXi-CplH{ z8r?1XtFL!9)d#F;Y0rOQai<272y+;3CVU0nI#-Ip#0UruTJ&-&IOj;`l~eGLZ#2llD01s_r_N19S&qk%S2H7ASUK0MJDBk% zNfbdwi^@#PNeaQOL#L#|P#?2zN1=A5?7d%(*>?2*!tLPdr*~`* z$+l+{+Gv`>Hr%rJ;eomo#|S6ipv_Qhpv2su*(5sOR3<`7MM_tC0tnCp8v`~0OwX8Brn6pvz?1;-5;zr za-`bK&F^5Ir+$9Kex2syaa0i%+0GZ~nYH1j+rgO>#e>&(TvIvvJC{7@;vbzk|n@HVwy z8*3(try~o`l$P|Dd@4bP3+kxW8;ei-K9C<6ogC$<7Z{Xe;BQXvo=T40`55(%gIUkr zc5cqlJt6@XRjmpz-w=2}Ww8+Mp3D`2{+VSd-f;IVm`qA%eX4VE5i!R0RN;U(F)9>N z363qfmuc~I4O8-tMak_+=PO{B6Q51llRwc=XPMcNw>goVLLXX^_%Tb|ZxA#DxLb9= z9k-P9n2R3ADy!5)EUHWBU_x+RI9-3lI#z3@qN*jz&ABVhoQa&8E$mlVyCgVfO$>K% zl*X9G81a3mU7<^SAe&AGorL{lSt%x~PIX~u3SL?nb8S~xAssh-#l4^bA5H={$+o#O zi}Y{!UW*roVzpbpuM!w42tPtrR!#dR<*0H>vzKK8OTt%Y^f!2owvJ}=Q2lQR-?r5z z;eLBsx4t}<3{*b+xV|3Me3@@ef3IYjemcT@t|A2Wi-;e$=?yGyzZAaHt}vzRpw}U^ zB1csIV(H&{r5{P$krVnZ^9fJpzm}SCHM&Ok#=640n+Rzp0ltgJXCyT z#k!6;+fk_|P2F~-FTqD|x_YQfF_#!zbg$6eohuZID4&x&e>X4g9fN48XhOMZa&xJT zLlQNyJ&pozGK-y|ei<%}sROQzgrcO4Uf$(W=~IxSg%yiqV(k$g@Wb!!Rky?$;I>$^6*&p`6( zL*Q`K`SVY=bZfU67!2xk36f`~?7l`e;lv&<~tRg>nwB4&^PUxGSL4h>J}zq;clCcI4i1j9%iU_KOndVx~ZX5&@C z%LlZ96Jx07y8-r^$e*>3E;9AMVwE^10NX^HW#_{Og-^B{*h*zZ^N37L`LGkO{qJHKI4<*;)V8pd2?E3 zGJV?bj#Rn=yW(TO)9%%GlV^CEQ>Q{Szcus576`=HooP}1jUF`iR@lH3)OeRzt`+!|kqpWgN9Wudzlb?#uLrBk$KvO0iicCqu24*v~=BQa^UyqX+3e9k^=W(~@tXjrN0)VJO5Ef*2) zTlLF6`++a#iarrG{F2|?j*O`}vuIr>i6T}^eoImv7|$v;LmpAZ7FOwxWTnvcU5IC2 z0$_}K*{ts^ZeG4{T#;w5@ho;Mx3otKB5&Ng=XCL$u1EdF!?kzmh$e5YVe)yM3~6z+rxr_XM$Xj@ zWURCKG(C-FDaVb-AnBt7~X%F?MvEU0G0?0r@~y;nk;MS9X4qgixQW^4fBN zxt&Iom1v+3Nx@xRW(g1@Qp@Cux;1w_-KCyQA>(JuG@?br4c0AJ-5T!AYdiR8QUGuhS%#UoZb9H?)~H% zA`E~UKZ9@iZy_J}BnT0{FQz{bThJ49<4e^eTE~QR7pz)hkpjM9>P2$G$7|=&YGBo+ z>++c6P`lWwzhn<&40O=R{WUufKqnm|aGi-6P@(tud%^(y^rq00xdU)BO#Y#m5YvAk zHKWC8ZH+4<3-}M0$D|F3*sl4`6ao1|#j{xdZnnbmkmN580c3Ke{i#NCU;f=kKKIxk z>H&}HsgVgDtAFg8@Lm6Hq=GL5Om!m3(fe7bfh{Nqga>3$s`yiJ{{QIja??Ri9!ZA! z9y`Yri5ZierWHtuIT^E322xF+By^i3F3FrsaMVSDMc%ft*PznU4`XOlK33Lwk9f(% zcfoo+mHrrln%yXHoMS=Ev> zMnxYbfpd3R6Tb2?!+tb#srL$h4b|mBrrXo@=}6fwRP((TA@5~Rd9gw2kWd_N?~*!* zBF~byE_~JVwZ8E|;iJ(*gNqcz$;f@Oc;!%IIFChgoVhB(y!l2N_AOjuGfy;6Ea0pP zcsvnlWKYB*#-kc9qKpdgj|Tf+p2;UKu8ptlGd-eL_}Tn0Yg+YzbE-wTJ*$kz(9^yb zPI(Es+I^j2)H&65n!k7%BOIQ{_7WCF%BwU{blKX{Ui{%dkRy*p4t`bcKe;@OLmTN8 zCn$b@c@AyrU?{Esc_h6@w^-Zqx{@HJDH_#3RWb0XjWi!qpJmYRjnDer$7(QyLyFnC z6VFko@jK~2zkLy}i@bzgkFF1dzC<+SUF}HH1$`6CsEP}$6l;x^+&-4`*kujp^OYlY z`_kc?6KIqLSWhXum=nKruCo2+BoCu@tNx{;?6Zm{-a7A>Xkx1dj{9i_b=Q@#BRXP<${tNL<@C94~U9ObbDt*m<-dSTXJhN7pG*E zpT9yYy~V^1;ay`bUhQjp23V<17!`3t<(KNcpW5j^zE7WDU7e)mNxYxdz2RhU)OOdF z^wo@og4Db5(jL({>o#KV4uNzrulvcBOZNbpw96kBt+xK z`vMG(bpBUUe`5Q(neTV;=4)kV3L|W{B!}0Y@#0PW2_G@feZCOrc04~f-Yc)& zB0cUMzjq76aa3B~(N4+0ZND@)(9+U%Vcah_C(V0&(M7}9tm3x7Ir{~2=IIXpBk2qE6V)NLi}tz7orbk-3^l|f`GVxAy`&}F&;xvp zNAQNHwiokdPl(^SQuQF;`&5k`7LPA8xF5n#cfb@M=9;_<-7xz5YO3gUwE9Ij1O&<@ z0fE>+tiX{kCvb^_6w(?{BhA4Ap*hXi$w0V!s_21)VfhJQS)3Q2@EWi@ef}e{UwJ4> z{!)Y?`uM6m3@awLNIuIMo7GO?Bs9+{6MLI~=<|qM?=wcX7^Ao&j!2r(^a;P-ehLr< z`%QKZEMrfMvtsQ*YW9pT77aDo-I8M~R~|9GPlu?JAr4y}XMT{b;Q`3GB57~X;>}H^ z50@RuxLT9^@MO4xD|J5uRnwnHW>KGF(TW`X>5<=9N?OQn!#234N?jC9DtCYHCf8B4 zZkp%!WAcc@Hrn}^DxU_cOF!JmW}RH!ow}n%>w8{?MW`eTQ#uB8!R1B;F)?5+r4F`J zpZ(4Zf6ZbvYwO*3}J`K?#vEII^h2Iu%ybt1hBk$4oGq&0k|D;|Cw_YI7<9&8`yDVhw#w> zOirp?|JE%a5E8_EdX57+kujixupl}RAVkX!hl8LeYUX4d0LFzD=yjro)-Zs84g>;N zb5etX5r1J7dMd!&Sp{nGce%pZ2wM9OrUW=$u7k^fDpxij-9-%wVmkeb4fy4v4^97# z1pzNt73j_h=3-Oxcgd00l?A&QPBrwbb2h82npz;5d>44X6Do`5MKlP+QI}14= z?XCvJ{fpxPI^0zt-#G!OhXq812e9@qgT(Uybsjd5PdvbFPYyuZQw<`+djbO>=lOtE zFC;+gr3MK(g=~Pgmp9ZdiCe;A7a zhrYZ3r^oNV%=u3|o%9JiX~*!dF_cLFHaH~YB*Z$L*?Q_TY6O Y`j!F%{3KvOpi3t=fer*pQ#>vG4`8MzfdBvi diff --git a/distroduce/emr/launch.py b/distroduce/emr/launch.py index e69de29..8257b54 100644 --- a/distroduce/emr/launch.py +++ b/distroduce/emr/launch.py @@ -0,0 +1,39 @@ +import os, json, boto3 +from typing import List + + +def launch_cluster(name, region, ec2_attributes, bootstrap_actions, instance_groups, configurations): + def log_uri(name, region): + return f's3n://{name}-{region}/elasticmapreduce/' + + os.system(f""" + aws emr create-cluster \ + --applications Name=Hadoop Name=Hive Name=Spark \ + --ec2-attributes '{json.dumps(ec2_attributes)}' \ + --release-label emr-5.26.0 \ + --log-uri '{str(log_uri(name, region))}' \ + --instance-groups '{json.dumps(instance_groups)}' \ + --configurations '{json.dumps(configurations)}' \ + --auto-scaling-role EMR_AutoScaling_DefaultRole \ + --bootstrap-actions '{json.dumps(bootstrap_actions)}' \ + --ebs-root-volume-size 10 \ + --service-role EMR_DefaultRole \ + --enable-debugging \ + --name '{name}' \ + --scale-down-behavior TERMINATE_AT_TASK_COMPLETION \ + --region {region} + """) + + +# def benchmark(names: List[str], region, ec2_attributes, bootstrap_actions, instance_groups, configurations): +# current_dir = os.path.dirname(__file__) +# s3 = boto3.client('s3') +# bucket = 'insightde' +# +# file = 'distroduce.sh' +# abs_path = os.path.join(current_dir, file) +# key = f'emr/bootstraps/{file}' +# +# s3.upload_file(abs_path, bucket, key) +# for name in names: +# launch_cluster(name, region, ec2_attributes, bootstrap_actions, instance_groups, configurations) \ No newline at end of file diff --git a/distroduce/local_messaging_sim.py b/distroduce/local_messaging_sim.py index de7bd59..5c29896 100644 --- a/distroduce/local_messaging_sim.py +++ b/distroduce/local_messaging_sim.py @@ -9,7 +9,8 @@ from distroduce.spark.session import sc from distroduce.executor.spark import distributed_produce if __name__ == "__main__": - intitial_conditions = { + # Initial States + initial_conditions = { 'record_creation': datetime.now(), 'client_a': {'users': [], 'messages': [], 'msg_count': 0, 'send_time': 0.0}, 'client_b': {'users': [], 'messages': [], 'msg_count': 0, 'send_time': 0.0}, @@ -17,20 +18,21 @@ if __name__ == "__main__": 'total_send_time': 0.000000 } - # N = 5 = 10000 / (500 x 4) - # T = 500 + ''' + Simulation Configuration: + N = Simulation Runs + T = Timesteps for each Partial State Update Block + ''' sim_config = config_sim( { "N": 1, "T": range(10), - # "T": range(5000), } ) exec_mode = ExecutionMode() kafkaConfig = {'send_topic': 'test', 'producer_config': {'bootstrap_servers': f'localhost:9092', 'acks': 'all'}} - # kafkaConfig = {'send_topic': 'test', 'producer_config': {'bootstrap_servers': f'{sys.argv[1]}:9092', 'acks': 'all'}} dist_proc_ctx = ExecutionContext(context=exec_mode.dist_proc, method=distributed_produce, kafka_config=kafkaConfig) run = Executor(exec_context=dist_proc_ctx, configs=configs, spark_context=sc) - main(run, sim_config, intitial_conditions, sim_composition) + main(run, sim_config, initial_conditions, sim_composition) diff --git a/distroduce/messaging_sim.py b/distroduce/messaging_sim.py index 5d80ca7..3a90dfa 100644 --- a/distroduce/messaging_sim.py +++ b/distroduce/messaging_sim.py @@ -9,7 +9,8 @@ from distroduce.spark.session import sc from distroduce.executor.spark import distributed_produce if __name__ == "__main__": - intitial_conditions = { + # Initial States + initial_conditions = { 'record_creation': datetime.now(), 'client_a': {'users': [], 'messages': [], 'msg_count': 0, 'send_time': 0.0}, 'client_b': {'users': [], 'messages': [], 'msg_count': 0, 'send_time': 0.0}, @@ -17,6 +18,11 @@ if __name__ == "__main__": 'total_send_time': 0.000000 } + ''' + Simulation Configuration: + N = Simulation Runs + T = Timesteps for each Partial State Update Block + ''' sim_config = config_sim( { "N": 1, @@ -29,4 +35,4 @@ if __name__ == "__main__": dist_proc_ctx = ExecutionContext(context=exec_mode.dist_proc, method=distributed_produce, kafka_config=kafkaConfig) run = Executor(exec_context=dist_proc_ctx, configs=configs, spark_context=sc) - main(run, sim_config, intitial_conditions, sim_composition) + main(run, sim_config, initial_conditions, sim_composition) diff --git a/distroduce/simulation.py b/distroduce/simulation.py index e69de29..6bd3695 100644 --- a/distroduce/simulation.py +++ b/distroduce/simulation.py @@ -0,0 +1,77 @@ +import sys +from pprint import pprint + +import pandas as pd +from tabulate import tabulate + +from cadCAD.utils import arrange_cols +from cadCAD.configuration import append_configs + +from distroduce.action_policies import enter_action, message_actions, exit_action +from distroduce.state_updates import send_message, count_messages, add_send_time, current_time + +# State Updates +variables = { + 'client_a': send_message('client_a'), + 'client_b': send_message('client_b'), + 'total_msg_count': count_messages, + 'total_send_time': add_send_time, + 'record_creation': current_time('record_creation') +} + +# Action Policies +policy_group_1 = { + "action_1": enter_action('server', 'room_1', 'A'), + "action_2": enter_action('server', 'room_1', 'B') +} + +policy_group_2 = { + "action_1": message_actions('client_A', 'room_1', "Hi B", 'A', 'B'), + "action_2": message_actions('client_B', 'room_1', "Hi A", 'B', 'A') +} + +policy_group_3 = { + "action_1": message_actions('client_A', 'room_1', "Bye B", 'A', 'B'), + "action_2": message_actions('client_B', 'room_1', "Bye A", 'B', 'A') +} + +policy_group_4 = { + "action_1": exit_action('server', 'room_1', 'A'), + "action_2": exit_action('server', 'room_1', 'B') +} + +policy_groups = [policy_group_1, policy_group_2, policy_group_3, policy_group_4] +# Partial State update Block +sim_composition = [{'policies': policy_group, 'variables': variables} for policy_group in policy_groups] + + +def main(executor, sim_config, intitial_conditions, sim_composition): + append_configs( + user_id='Joshua', + sim_configs=sim_config, + initial_state=intitial_conditions, + partial_state_update_blocks=sim_composition, + policy_ops=[lambda a, b: a + b] + ) + pprint(sim_composition) + + i = 0 + for raw_result, tensor_field in executor.execute(): + result = arrange_cols(pd.DataFrame(raw_result), False)[ + [ + 'run_id', 'timestep', 'substep', + 'record_creation', 'total_msg_count', 'total_send_time' + ] + ] + print() + if i == 0: + print(tabulate(tensor_field, headers='keys', tablefmt='psql')) + last = result.tail(1) + last['msg_per_sec'] = last['total_msg_count'] / last['total_send_time'] + print("Output: Head") + print(tabulate(result.head(5), headers='keys', tablefmt='psql')) + print("Output: Tail") + print(tabulate(result.tail(5), headers='keys', tablefmt='psql')) + print(tabulate(last, headers='keys', tablefmt='psql')) + print() + i += 1 diff --git a/distroduce/state_updates.py b/distroduce/state_updates.py index 0e71932..3022b21 100644 --- a/distroduce/state_updates.py +++ b/distroduce/state_updates.py @@ -2,7 +2,12 @@ from copy import deepcopy from datetime import datetime from functools import reduce -# State Updates +add = lambda a, b: a + b + + +''' +Function that maintains the state of users in a chat room +''' def update_users(users, actions, action_types=['send','enter','exit']): users = deepcopy(users) for action_type in action_types: @@ -18,13 +23,17 @@ def update_users(users, actions, action_types=['send','enter','exit']): users.remove(user) # remove_exited return users -add = lambda a, b: a + b + +''' +State Update used to counts sent messages +''' def count_messages(_g, step, sL, s, actions, kafkaConfig): return 'total_msg_count', s['total_msg_count'] + reduce(add, actions['msg_counts']) -def add_send_time(_g, step, sL, s, actions, kafkaConfig): - return 'total_send_time', s['total_send_time'] + reduce(add, actions['send_times']) +''' +State Update used to create to log message events +''' def send_message(state): return lambda _g, step, sL, s, actions, kafkaConfig: ( state, @@ -36,5 +45,16 @@ def send_message(state): } ) + +''' +State Update used to sum the time taken for the Kafka Producer to send messages between users +''' +def add_send_time(_g, step, sL, s, actions, kafkaConfig): + return 'total_send_time', s['total_send_time'] + reduce(add, actions['send_times']) + + +''' +State Update used to record the event record creation time +''' def current_time(state): return lambda _g, step, sL, s, actions, kafkaConfig: (state, datetime.now()) \ No newline at end of file