From 0265e6faa7366ce8d8366b0129f68f5c4239fe3d Mon Sep 17 00:00:00 2001 From: HaoYizhi Date: Sun, 12 Mar 2023 12:02:01 +0800 Subject: [PATCH] data collection --- .vscode/launch.json | 2 +- __pycache__/computation.cpython-38.pyc | Bin 0 -> 1542 bytes __pycache__/controller_db.cpython-38.pyc | Bin 0 -> 7233 bytes __pycache__/model.cpython-38.pyc | Bin 0 -> 7094 bytes __pycache__/orm.cpython-38.pyc | Bin 0 -> 5069 bytes computation.py | 52 +++++ conf_db.yaml | 16 ++ conf_db_prefix.yaml | 1 + conf_experiment.yaml | 38 ++++ controller_db.py | 237 +++++++++++++++++++++++ main.py | 48 ++--- model.py | 4 +- orm.py | 151 +++++++++++++++ requirements.txt | 8 +- 14 files changed, 529 insertions(+), 28 deletions(-) create mode 100644 __pycache__/computation.cpython-38.pyc create mode 100644 __pycache__/controller_db.cpython-38.pyc create mode 100644 __pycache__/model.cpython-38.pyc create mode 100644 __pycache__/orm.cpython-38.pyc create mode 100644 computation.py create mode 100644 conf_db.yaml create mode 100644 conf_db_prefix.yaml create mode 100644 conf_experiment.yaml create mode 100644 controller_db.py create mode 100644 orm.py diff --git a/.vscode/launch.json b/.vscode/launch.json index fa990d4..3f1ca37 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -8,7 +8,7 @@ "name": "Python: Current File", "type": "python", "request": "launch", - // "program": "C:\\Users\\ASUS\\OneDrive\\Project\\ScrAbm\\Dissertation\\IIabm\\model.py", + "program": "C:\\Users\\ASUS\\OneDrive\\Project\\ScrAbm\\Dissertation\\IIabm\\main.py", "console": "integratedTerminal", "justMyCode": true } diff --git a/__pycache__/computation.cpython-38.pyc b/__pycache__/computation.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e6acd9c972b02e83a95e9ed984169eea3ca80cd7 GIT binary patch literal 1542 zcmZuxOK&7K5Vqa!$Mp0}cm(khXapP>4$&S73ehf;B}9lJQZ_Hm%;Wh}8L0uVD7ll+?A7xJe=JJ5K4O_nhL?V|7BLWNZ7zei&Gb#e`8Qb=tE+!*D@o*=-}N6;Q~ z43wUuu@7J01moZ$T%bqj_vjQfe?W&=1R^|(KqDHr#<6RIEp&=cytyZ$55WiAB%<}0 z%)MX92{oNLgTAYyTukQDBT22bt zo6%lsdKrc^u9hpCtI3P7^=m+K$mFXRPVgKX=NZ1IM5lFf1)_uU>M2MItrQKsMH^>TlF@AY(1gO?psB&MnR?J(9;-z3l#Quvrgc?1##= z2aB{(JcGexRP$UwJelEu4_k4EXJAZHm0G`a<7-1D^|Ua035|ol!AHFTr{`how}2p= z;0RMM!S9h6Czk;U2*LD{67o0oBDkHc{0Fo?QC;|P*!NxIS83^jk2wdg(VrbA%(3kOWhlc-_|lOW>R=!v6qts&zO3 literal 0 HcmV?d00001 diff --git a/__pycache__/controller_db.cpython-38.pyc b/__pycache__/controller_db.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fb534294b3a5deeb45b5ecc6acdeec2da5d2ac34 GIT binary patch literal 7233 zcmaJ`&2JpZb?@r#>FH?>heL{#D6Kx4+7EaZGNX?yyjj!h-PMOJd(Dldm9=hY(QZ!F zP&4fLV0CjXPMWb}DTs3rvI&A*q6LOXE(e_)VsNY0?DJXTF!0DEQa@P%9?ziGf*9$zZ+wn{=gRyp84XS*f4QhO!3+BFK z$dYs(8Pa)Z26Z(r%Uh=YRMn;XPy`D~jE>35z8Ed;8^Mw~u9hAOtZCmi8s1av%xKuT zCpW@GB}u#A%N^C*Z1+@dzZWGcx7xjg23&tIP`ce!y)<`jN8LfJ^75bSzT9c1xqYLp zyA6@oKfstM#UfFBUF*JHZ_@gQe>NKL;7WdtLK%UOMqu87Yyu0HtxC#~<{Guj%0=B; zGgJk48+Q-)()jLR%z5d`$|C`}cyi{E5meQza%J_2kh8M($i}zV=@+Eotk%8Gu@)dbTp2gGSSowrJhn0P~ z!YeOJ0r|a}SFiSasqV+I(%0Tw&&%y3OjVNRH3+m9byYafs?~mwds-zb4O?-vnb+ea z4U^~-CBr0a%kWCRaHQu_cv%Yws7Wd``8lXma9zcf{2WCJF+CK2SKqTzF)}l2U+md; zjW>)R8n=v(j7&&#%gl_>$^=R~Gdh-D%WP?$GjdPo15V*a5T1cbPRcQnT zQ!@9Sk8PNG0iU6dp~y>+5+0YOYDGJ7+Nk7qf1rA~%?nmqJ8>Mw{YdI6O_`-Bv$C0n zgGfWnsnWVe59(CRp=da{ov65#JH-j`y?3phU{K07aPQ{L2wix^ zSV2O$u{zkz-4Khn(=c4Z!U?Tz2v02Ha>Q|5&NJI|49774*RdQiTso{_tCP7+Qjf`x z>LxDMj73y3V`S_bg6@Lud|G0?VdhKt_W00wg=w?_Q=aGUgzdWM`dsK!Sc$*BD9)gn z&+yiHQnG@Mgbdmct{5&)?eowu07E57b*Q7>rkdI}&2@3{zTZbBDDIKHk`KrMFkfTxr9l{5G@q zyv#)ZWgv%z``7jyzMsX+3U*M!$eB%%d08oQvhu$34P&pe{l)gbZa+yKSf%?Ov1uT^3$2cm6B07yuP`oeB^FBe%EIZj#p{DG!_6rMxuIq+c^^V!1~ORD5l~ zp5VyW@$+Jr`U;i}YSRPqV_*4%@Y0JqF9+4Od=Ms(4kXpjOFi0FP{UphN$H4=>vMSJ z>$sBFP#ESi|J6iIEa1;}UDFn>>Efyh6EI{xtD3{}Q;PpW%mMm;6YJ?U6wNX%J^m4? zBEP_v@GDd{4QLl>-9PNPKnRh-20A9$1+bu~!#)n`u#;rYud6VqqpbpzOsIz5^ z*I~OD*V!sEE3-+Lu?EPL^d8T_8pu2f%*nN_GZ-Af+cm24gC9zIfaMp&~s7ePD{k^~o}I(qgH)QBQ_xKaJwC zUSo!CHx+;g6f8e>RM*?R&D_PC3-adl!Zy5SV-B1JIsrYNUK=1$~Fsd5ek{y|hf$L)-r@Eh*Gd^?eiF-@}z$L}8G!3R_h9&lK)6 z(=6&#n&Us`4^F)be+;iSjo?QOc?!Wvhsm1}Rwe(2B8B&zXzITBTjOtx7Ce-+9=WNN z5+5^(dp(3-yM}TP-fiTh(4~>PZ|s@SdW*GwjZhoo?M|f_3)C&89@;ZmNtRB)A~I%g zvi#U4JRVguhqd#g?HgEgmY5;yXN~o9PP%EGJSI@M@)_=9e&DS$9h>?lI;IjXqG-u+ z6r1o#AJ_K8FX3$@{LbH-+Z0HMr#>FS&(6Tf0gUBFJ54}0%+-sz2tSAa)xFI_G{RdW z*Nd|ryscE~u!>YJ!&QaJs6D*lWL;x=#*M5qFnxjwd|e?UzJ_hJu)3%rZoTqK3CZlj@z zcxfL+6IXH;g(2#~2Lh8aFA!zKpZPzIT`QFK>>n56jV67^IyKzO1y!KX=I|~)GPYmK ztWSmhU0RjycmKOKf2Z})e7XPwJC-hV7V(_+Uh2KcBWAWttabjY zV=#6pJw7_IZ(zKM@ou^-s}IF5V0NRE`ygh;d$JN-)RR4jS2(pVIxl5T=d_%CEa1M% z2P1|4RaVBxbvQHH&l%p&na4+ZXk8fad0xLN=ci@Iglu8`|Hpw>pfQXCr@=RJ816UO31SSE=2X&ura0UMU;&=>qR)hp@5m&8*MlsvZbN(Aw@sL>)MGgbsxS- zPH5d*{?J-kSs7kh0mwtV)ZE*7ZcfG{jTo!HYZ~c z`U47pEUH>Az;Om!$Zyee|q?O;9C`C+e~e+8q0j;8qp1AHG;x-Yeg{q7LhcUY zCtPyVP>W~&Ds0!=>fx9DBOi@2!Z0;4p;C-J* z2e=1At%rAj579RTvjn?^7{E0GmjVMi&7h1BgJWQPU4qO5T$M_KnQgs0Az!%|6iUeI z$YHR{=Z=6r!RwNB6oz>mOv1*WhwC_lb4@s=C(fF6(-8}J>WO8rj0LEzW4RdNBI>8| zG+$MKxGR=11H5B8>Yu!A$|mV_Nz_+y9qI>5D znHeI+6j`5)#yAB;5Zp_O0IARYjPgmC1E^XBdt<<;Tbcq>?^*u=TXCW;pwrVjqM$Sj@rRVxKbz+}gP;CQib`a|!a?+TQWOd~I= z2dYUnz_}|Y0gZ`tK@T`-#k_$sR|Q)r72Nj$I)dtiwTDOyLO=UI)tfYsa!1eMF+YY( z7t&!WBx*F?{l8#HvWUVEzF7xtScG=bUk%~f(4PX&7fhd0ID!OHI##%!phD>yZ~_6} zTBjrBG~^^n0hYl2+#Z!u(9uyjbvkZZp)}5xiMmJWN%&LniqQ;vQTXc2q_FO`u?=#` zDI7jRhD(OjM)f&-={B~YV=AVxNB5|Y5racNXVB-C=(GO+d{VL!b-_Hjjf8lm05UR^dmk!=DE)q%&xK*sLk^v@$w?R%$YCLuT0|oa+A@8=K2MXXpjm0V z30if(FN<{4IeJKOx27nVV+cK@+Fw$^=lct)ksHyEsCX^`u5su=sY$vLT8X?H#mb!W zvAn8lo1$*JE1dW_>DIlAZp{>)=Pj5*(=7V*ZiIAtNI8C;q{HiDsJ;5Zo%Ih1_5Z}b za^>A9PNgWCrL)KMf9{8h(M-;rFmTbpS+`O;4 zyQ}I!lNpUZL64aWm5?AF?CRI4gEfWruKyZEZpsb)7u3qAg?JF@l>)e5Ag$b8D~|VV X(k)i}Wp?`o?7CL%XU6wjcK`ne`ezgx literal 0 HcmV?d00001 diff --git a/__pycache__/model.cpython-38.pyc b/__pycache__/model.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4a346dc22fb1622cb39253e244ea1115178ad1a6 GIT binary patch literal 7094 zcmb7JO^h7Jb*}2}>FMeD-PwQsiJ>LRV@qqUtRG9J1<|CX$dPD4q>!MnOrv+Ichs}< zx79rqIp{$Gtpt*Y2`~tdOKf2FW&%F=m|TJ!e8?dOAAIVoec?3$3`9}1`Cj$xa)y+H zWCvY8uU=KX_v+RAsr{&2b{YPDd}z!6e4Vl1QD^qgM&~Vj(N9rGCV7Vmu4_Y>y0%14 z*S4^Aofi&jqf_X*!sRr_>=e5tQKGge)-NX75bg@jBLhUCyZh*?1izXf^Pq2;O&4cZvb+dSZMB#mVXOh z^fHP-OH>$=OXD*JshZM4ZONRpQRie{I;d?~kS^-HEQ0$|ntQ)5gHDrUg`>ZdXq51o z_@Yx5V+m`sHcyzxV`F0Ko^?QdBhIyri9G|!$Byn5_G#Rm_KLb!(j#TSD{-}5BSNKl z-x%|Rw-=-)^uEEF9?MV1jKpZy(FPNGY_Yur$a1_mSvg?R!E7^RV=FP+t5?{Km2nPh zttGj2Hd#MliIwCovb|pb`$S?U)`4DqQj>)PKDHBkc4hR|pk(ZG1C+S3C`&DC$5?0M ze2a~pnNg_T=LU^TP98Aqd2Y)X7shVlOim?k;>yY{#@eSRXAal_=j;LBJC!&|{&@B3 zj&YGC`3X3t@r;vu>@J%;li1Q)*R#M=@h;Omt$`=ZKIW-NJYm&@xVp=dqULIGvU(RY zLC@chOEfbntut9&Ckp?1T#nBs=HyS3@&Vr`oc6c^j!q|bQUR8xnCKl>@30*%YjWW; zC#hax`~04H#{{HKkh>;9#xiYm+`Zc=Mr#vVBq}EXT8MKz2~vw^X+f8S8>YA_ldp>km=$*9ybz}EVjROmrECP zuG}}rON2Z5cCyss$zqEW?+=W<0oHo~xU|lTdd&^A&(E#-)9ISu0TrC&oUMwosN0Yi z=4?zZ+~MGORm%YV_9k(Cz~$O~d%T=1w_gH%-x{v~H$V9<`Y+4%lkBLLCsv8W<@PJd z3Qoz~BWaxny~}g-{`#2S^HX|X#fh&1w=#K6Yt4sXX|kHE>%A3PxN9@3zKvDa$0w2#;Ney9u#&8|U)r@Fn9-N<_vEQ$rM>wN zOr8c5_0O$+{wYWMb!hgAK2LM+x8Uht6E=C{fRQ1F2|06hRQh4RdlhCR+Gs{M6`7I- z7lp7VgIiv+-x1bj3|$eAi_hHZq`@OXIVQ@3(4Fm7JeksE;|W3rjc@jIFc3HTEnp#nl23;0JO&;;Uh##>R!T5DlA6ZI3ZxzvB^+57`%& zF2Aw$Q52|X>(aH4u5JCK7hF-{&0q^Iq#ZQlt!qtnX}i01C5(U+`!Q(WdiPx)pe`2g zYz%Iv?nN^D&35LiQYux2>U4x$=hfS*U-=$ z*2uudxO)5iBEdfbq;OX!;fS21XX`CM^#Rv zs&4=&iZQ8-nySnUUunh9oGtlQxO!Cpx^hrFMe(uP5J_67B@*ggc0_GP4biHeMFAp9 zT8u|a>ycv-J_O3K2uIEo>FlU{w%3o(hP|`o^3OKDub!a=Dfmcj<@aR2o7x}hS``Us z^ai3RgB^d^LA20o38y!h;+bJDR^-G}7cDP~HdL7a)eLYPcBl$L*Ns*f-Q)cKL;%QkL#uX#J=*G1`lkQZJF-%hv&<0BNqJ z$);E+&2{|kpd(z`*rz}-N((wbe2?g{Gmu!Gs#V;N{f-XYpj?q6^MfET=)fdo>iIlD z9Gphbg7Hx2(G?4H3`B$1^ap-3jBls)x%G(J2(pL+RFzBGq`;v32(nV?%DhHTBS@K+ zr1C#GVySVGV$jZxaE{Eja37mvk$cD6?sr8Mg{S$A^eJhQg+(S$=tx-~mdKo&71zp`Vr>!s< zmFU+f7;$*?IerS5E_Z-IV=EY|7bL zylz|A)8_6WFb^$*aB7xq)`<#U2V}_{6=(d*oY^C1N-BF9UvvQl!U#6u(`3)96)vlI9|Z#&rOtH6uHZs0|6Fi3f|{uD{E#iAEHvW> zoW;mODaio+sP}US#aWhwe5N^^m=uWZ^L>Pv#BtWYaS9<6A0sV62vr%I37;VOpx|il zhkLKaIbhm4#J)AQupV-*#FF`aORtfiI7mvEmj_lJ^G}R>F*yn7sRXGCB?>gI)2X#RNCxL$ zp=?>NGIRbq(5+rgxjS0=!Poqe$_4|iy>m)OUV~x>9n+Ei(&5~wPgRH`>@?c^?{vti z6&-;NnHOzqADUTQX6^>J)7(wJGYp=*4DLnppE_#%H%yC2ziq=lwAkp(^j?FknXvH~ zedIpo0!EQuAM|BS*QXw2I5_fLM^0ppF|rKAoMxU{$J1qxBco6xT3 zBZ`ZLgF)xEI6+~63igHp9)rkv=@3V!9%45K`?ZrQlkE}nu$`UPdo<-0c5PbuW+JRdX%0BXE2?c zT00$onwmTgu0O<{|AH^Ni-Ngz@{za^&R5o9@ zFH1g~Mr`1f;D_OD;nhp)$E4+1kfk$N@^kwK1b?F_mA;WAzK;HxrHeQ%;Teb5q*IIo{n`` zmyhA6BrIMVe(xfS&9>Fn9;=<7KEIh`BQ_`Wgl7YmKQQnJQ^@A~y(j(XBF;y@d0%5f7Y#Uj1mAWWlvMkLbtTTd#TbFt%hx26B) zD_G<|fXYVC{;yXjF@EZOhm)DW)2mbn*!vR}*W-DQ1-Av^Nw+ zZxi23>TS$c7pc&<|FH=mseSKQYP@u4gT+S_{i$6P;R_gR7$UzDwzNetQ%=^Tmxpuhm!<3tc}Rboy~8+{SATwNiUpJF5AC zul#PLHZf5Ydbf}|Qc5YRy1E&TkY?TP4`cNP5Y+bxYq9loMvSKwN|s$SBQu#WqIHh6M4MP7 zQ97jo?w59M+Wv3b8r7dxwyO;%bv%z=mpm_ZJ+F%w9JoX;EqUJF(8r?$ji^5(vOb`K zJcHV%g6Nu2n#Y<`#lSgy5h;tqoxftMPTtveOhqG&CCW76k_NZ6^TPq_w+?jygLoz! zb}{g6v{eHIUShHbg46}aM(eF6=okqXHOdn{-`Dgzv!h?$P_5NQ+ na_U^n7=IH2XVD8NK=<^QIWIURhf+uUzO*5DNH_camtFWDaBSyf literal 0 HcmV?d00001 diff --git a/__pycache__/orm.cpython-38.pyc b/__pycache__/orm.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..84a240df86e7749f34b5277a62bb0b5c1e802e5c GIT binary patch literal 5069 zcmcIoTW=J}74GgC&!y+;8Q+b~V6KLZ2_%~b>j-XbC+<%PeJpHPp<6MrBNyd>YLp20nMEx}6hsE^M%b?NS^^L^)3->cPZ z1AnV#k$*C882=_C`&WjshAaMuX&Bt#rf&o$GXslRCfSys3-TzF2y&ynp_sfqRp7fizxRHPC(F^0A z=qAgiYI4!_UFjy?x5C+UW3gOPh3k>u4?np-MM!m%CmdRQ|4bLi6Xh`e7GOP?f^RO`gl#i!4%JbkvNC3IHq$3p6!s?>Ai_;Dn;D!QCYJe11!BAKw_V6NTz zjcXk*?hFUxfy+HSEY>WJ$p}Q^4vqY#7xFGGdG>D%#u~182CxwZGr7Smk^36}^ul6! zF(wM!!Y}tc$BNp@^DzWZDWjJ&x}DK08NCX<#tXc-XYq7~`XNu_kf)iMVLYR^GJ5;a zsEI?K$wQv0L!N2Cvc${W<`ul_EU)s~cK|7`BUBs8zWJmlBw{B_RK*c*jk~WbRq3I@ zl{?NuL|)D9Cy^H-q^a+%SBRkyG30Lg;x#eYYYPG~>jphvDEnY0RpdJe5U-0F>zi)( zq0@_ceT1dJw)^hxP8U1$_K)-&uIsqsIs`X=YdkQpZ`O+(o;Y`Bx3+rl%vU;Z%&T}7 z%j1ywSomA740#db-u(Rfm5qCdxp?E#^?U0ZUxwm_1l%^hlF| zB^o6gw{N>}MKW0F?WmIDNYRsybCsTjUe7Rh+Xwev8Mfb9o~OECy0387mlN< zj`OJR`e}=tzznM9I0-Gn3U`ZQk-^;9yIENA1`8U>ZpqrJ7rQ;&2hTA$O7}aedVy&nAXRuPYp) zrAW244>&{DZ-pQAh>A*~6Q>(o2puoc4-q@U_X01(hVsEonNjdnd^gzSu9MJ1w-Z-- zTU+#0UBr>QEu2f==e#Uw^Hm|eoxDUVMBADXKPD3?(>R4IJ_AuQZL@Awb5%3_wM_a! zhbPrrR^8HL4iQEXc1}0zAE40jupsp&&g9`#p{b zeN7mYCn0=dnLm8;uixj=Ad)v>zp~^x{8L3FE8mL~RRmiQx+H~{JPYxmY*k%NDY4`6 zD;;^Brnx}kMhG;Ix%QWsMBz{Ulzt6{{jWUTW!KVsa;4(K+ zdEt0m*+eb_Kv7vqdS%U zlc?t;npRl3d&zk$TtB%F_3}`yz4bbLJ7KlAE!w0pDIw@F1C_=MXks`fW7=p99iq9I z*_dvqx3#|)#h$LJ(hhCiVbm$B^d`knB&b0ivf4m#P>=}88bfo)nZ!f#v>q~`)T7ym z0`wZL_-7EBjm$+W<)WrJWmc^L9}RhFz*AMTNk6L&_R>LqG&}tV6TeS6^WN+Bn8TAmOjd~`*YA8)GM-_<2`GVJAfe~QM z@FpJz{%3g$_;1TQczI;n!8W7>AU}gs-hxQsFPF$BgzJ@*Uz7F)gv#GS%=~a5*H#By z@V-d?9d0^Aa&s^?1Le67g0lQlzd{Tgvy#sAsJT;(@+zX(>Gy^!6C;X|VGoLcm^I-T zg+?WGT;ETDlp=T<#}vB4{7}8Tf)QBMnp z+5>n@033AyVv;uih^G8CUWkq9JCdx%3s5m5oP0}K%9QK(3N{i>pGfgNp&=-5`7Usf#<3c zm#9fVG0Gl%ej&O!D73Uj)s`x)rsuS)wD|EFL=*Y=94`7_%Go$gl+3(oTP=(9R<3Cl a4E+9^w 0 + if force_drop: + while is_exist: + a_table = random.choice(lst_table_obj) + try: + Base.metadata.drop_all(bind=engine, tables=[a_table]) + except KeyError: + pass + except OperationalError: + pass + else: + lst_table_obj.remove(a_table) + print(f"Table {a_table.name} is dropped for exp: {self.db_name_prefix}!!!") + finally: + is_exist = len(lst_table_obj) > 0 + + if is_exist: + print(f"All tables exist. No need to reset for exp: {self.db_name_prefix}.") + # change the is_done_flag from 0 to -1, to rerun the in-finished tasks + if self.reset_flag > 0: + if self.reset_flag == 2: + result = db_session.query(Sample).filter(Sample.is_done_flag == 0) + elif self.reset_flag == 1: + result = db_session.query(Sample).filter(Sample.is_done_flag == 0, + Sample.computer_name == platform.node()) + else: + raise ValueError('Wrong reset flag') + if result.count() > 0: + for res in result: + qry_product = db_session.query(Product).filter_by(s_id=res.id) + if qry_product.count() > 0: + for p in qry_product: + db_session.query(Firm).filter(Firm.p_id == p.id).delete() + db_session.commit() + db_session.query(Product).filter(Product.id == p.id).delete() + db_session.commit() + res.is_done_flag = -1 + db_session.commit() + print(f"Reset the task id {res.id} flag from 0 to -1") + else: + Base.metadata.create_all() + self.init_tables() + print(f"All tables are just created and initialized for exp: {self.db_name_prefix}.") + + def prepare_list_sample(self): + res = db_session.execute(f'''SELECT count(*) FROM {self.db_name_prefix}_sample s, + {self.db_name_prefix}_experiment e WHERE s.e_id=e.id and e.idx_exp < 3''').scalar() + n_sample_1_2 = 0 if res is None else res + print(f'There are {n_sample_1_2} sample for exp 1 and 2.') + res = db_session.execute(f'SELECT id FROM {self.db_name_prefix}_sample WHERE is_done_flag = -1') + for row in res: + s_id = row[0] + if s_id <= n_sample_1_2: + self.lst_saved_s_id_1_2.append(s_id) + else: + self.lst_saved_s_id_3.append(s_id) + print(f'Left: {len(self.lst_saved_s_id_1_2)} for exp 1 and 2; {len(self.lst_saved_s_id_3)} for exp 3') + + @staticmethod + def select_random_sample(lst_s_id): + while 1: + if len(lst_s_id) == 0: + return None + s_id = random.choice(lst_s_id) + lst_s_id.remove(s_id) + res = db_session.query(Sample).filter(Sample.id == int(s_id), Sample.is_done_flag == -1) + if res.count() == 1: + return res[0] + + def fetch_a_sample(self, s_id=None): + if s_id is not None: + res = db_session.query(Sample).filter(Sample.id == int(s_id)) + if res.count() == 0: + return None + else: + return res[0] + + sample = self.select_random_sample(self.lst_saved_s_id_1_2) + if sample is not None: + return sample + + sample = self.select_random_sample(self.lst_saved_s_id_3) + if sample is not None: + return sample + + return None + + @staticmethod + def lock_the_sample(sample: Sample): + sample.is_done_flag, sample.computer_name = 0, platform.node() + db_session.commit() + + +if __name__ == '__main__': + # pprint.pprint(dct_exp_config) + # pprint.pprint(dct_conf_problem) + db = ControllerDB('first') + ratio = db_session.execute('SELECT COUNT(*) / 332750 FROM first_sample s WHERE s.is_done_flag = 1').scalar() + print(ratio) + # db.fill_experiment_table() + # print(db.dct_parameter) + # db.init_tables() + # db.fill_sample_table() + # pprint.pprint(dct_conf_exp) + # db.update_bi() + # db.reset_db(force_drop=True) + # db.prepare_list_sample() + # + # for i in range(1000): + # if i % 10 == 0: + # print(i) + # print(len(db.lst_saved_s_id_1_2), len(db.lst_saved_s_id_3)) + # r = db.fetch_a_sample() + # if i % 10 == 0: + # print(len(db.lst_saved_s_id_1_2), len(db.lst_saved_s_id_3)) + # print(r, r.experiment.idx_exp) + # if i == 400: + # print() + # pass diff --git a/main.py b/main.py index 178df55..3645ffd 100644 --- a/main.py +++ b/main.py @@ -19,34 +19,34 @@ def do_computation(c_db): break -# if __name__ == '__main__': - # parser = argparse.ArgumentParser(description='setting') - # parser.add_argument('--exp', type=str, default='test') - # parser.add_argument('--job', type=int, default='3') - # parser.add_argument('--reset', type=int, default='0') +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='setting') + parser.add_argument('--exp', type=str, default='test') + parser.add_argument('--job', type=int, default='3') + parser.add_argument('--reset', type=int, default='0') - # args = parser.parse_args() - # assert args.job >= 1, 'Number of jobs should >= 1' + args = parser.parse_args() + assert args.job >= 1, 'Number of jobs should >= 1' - # prefix_file_name = 'conf_db_prefix.yaml' - # if os.path.exists(prefix_file_name): - # os.remove(prefix_file_name) - # with open(prefix_file_name, 'w', encoding='utf-8') as file: - # yaml.dump({'db_name_prefix': args.exp}, file) + prefix_file_name = 'conf_db_prefix.yaml' + if os.path.exists(prefix_file_name): + os.remove(prefix_file_name) + with open(prefix_file_name, 'w', encoding='utf-8') as file: + yaml.dump({'db_name_prefix': args.exp}, file) - # from controller_db import ControllerDB - # controller_db = ControllerDB(args.exp, reset_flag=args.reset) - # controller_db.reset_db() + from controller_db import ControllerDB + controller_db = ControllerDB(args.exp, reset_flag=args.reset) + controller_db.reset_db() - # controller_db.prepare_list_sample() + controller_db.prepare_list_sample() - # close_all_sessions() + close_all_sessions() - # process_list = [] - # for i in range(int(args.job)): - # p = Process(target=do_computation, args=(controller_db,)) - # p.start() - # process_list.append(p) + process_list = [] + for i in range(int(args.job)): + p = Process(target=do_computation, args=(controller_db,)) + p.start() + process_list.append(p) - # for i in process_list: - # i.join() + for i in process_list: + i.join() diff --git a/model.py b/model.py index e59a18f..4a50423 100644 --- a/model.py +++ b/model.py @@ -324,5 +324,5 @@ class Model(ap.Model): plt.savefig("network.png") -model = Model(dct_sample_para) -model.run() +# model = Model(dct_sample_para) +# model.run() diff --git a/orm.py b/orm.py new file mode 100644 index 0000000..dd683c0 --- /dev/null +++ b/orm.py @@ -0,0 +1,151 @@ +# -*- coding: utf-8 -*- +from sqlalchemy import create_engine, inspect +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy import Column, Integer, String, ForeignKey, BigInteger, DECIMAL, DateTime, Text +from sqlalchemy.sql import func +from sqlalchemy.orm import relationship, Session +from sqlalchemy.pool import NullPool +import yaml + + +with open('conf_db.yaml') as file: + dct_conf_db_all = yaml.full_load(file) + is_local_db = dct_conf_db_all['is_local_db'] + if is_local_db: + dct_conf_db = dct_conf_db_all['local'] + else: + dct_conf_db = dct_conf_db_all['remote'] + +with open('conf_db_prefix.yaml') as file: + dct_conf_db_prefix = yaml.full_load(file) + db_name_prefix = dct_conf_db_prefix['db_name_prefix'] + + +str_login = 'mysql://{}:{}@{}:{}/{}'.format(dct_conf_db['user_name'], dct_conf_db['password'], + dct_conf_db['address'], dct_conf_db['port'], dct_conf_db['db_name']) +print('DB is {}:{}/{}'.format(dct_conf_db['address'], dct_conf_db['port'], dct_conf_db['db_name'])) + +engine = create_engine(str_login, poolclass=NullPool) # must be null pool to avoid connection lost error +ins = inspect(engine) + +Base = declarative_base(constructor=engine) + +db_session = Session(bind=engine) + + +class Experiment(Base): + __tablename__ = f"{db_name_prefix}_experiment" + id = Column(Integer, primary_key=True, autoincrement=True) + + idx_exp = Column(Integer, nullable=False) + + # fixed parameters + int_n_country = Column(Integer, nullable=False) + max_int_n_supplier = Column(Integer, nullable=False) # uni(1, max), random parameter 1 of firm + int_n_product = Column(Integer, nullable=False) + int_n_firm_per_product_per_country = Column(Integer, nullable=False) + flt_demand_total = Column(DECIMAL(10, 2), nullable=False) # tri(0, total_demand, mean), to compute random para a + flt_bm_price_ratio = Column(DECIMAL(10, 2), nullable=False) # benchmark value of b, same for both countries + flt_beta_developing = Column(DECIMAL(10, 2), nullable=False) # benchmark value of c(beta), for developing countries + n_sample = Column(Integer, nullable=False) + n_iter = Column(Integer, nullable=False) + + # variables + is_eliminated = Column(Integer, nullable=False) + flt_beta_developed = Column(DECIMAL(10, 2), nullable=False) # larger, for developed countries + lambda_tier = Column(DECIMAL(10, 2), nullable=False) + tariff_percentage_1 = Column(DECIMAL(10, 2), nullable=False) + tariff_percentage_2 = Column(DECIMAL(10, 2), nullable=False) + + sample = relationship('Sample', back_populates='experiment', lazy='dynamic') + + def __repr__(self): + return f'' + + +class Sample(Base): + __tablename__ = f"{db_name_prefix}_sample" + id = Column(Integer, primary_key=True, autoincrement=True) + e_id = Column(Integer, ForeignKey('{}.id'.format(f"{db_name_prefix}_experiment")), nullable=False) + + idx_sample = Column(Integer, nullable=False) + seed = Column(BigInteger, nullable=False) + is_done_flag = Column(Integer, nullable=False) # -1, waiting; 0, running; 1, done + computer_name = Column(String(64), nullable=True) + ts_done = Column(DateTime(timezone=True), onupdate=func.now()) + stop_t = Column(Integer, nullable=True) + + c1_wealth = Column(DECIMAL(20, 2), nullable=True) # country 1, developing countries + c2_wealth = Column(DECIMAL(20, 2), nullable=True) # country 2, developed countries + c1_wealth_dgt = Column(Integer, nullable=True) + c2_wealth_dgt = Column(Integer, nullable=True) + c1_tariff = Column(DECIMAL(20, 2), nullable=True) # country 1, developing countries + c2_tariff = Column(DECIMAL(20, 2), nullable=True) # country 2, developed countries + c1_tariff_dgt = Column(Integer, nullable=True) + c2_tariff_dgt = Column(Integer, nullable=True) + c1_n_firms = Column(Integer, nullable=True) + c2_n_firms = Column(Integer, nullable=True) + c1_n_positive_firms = Column(Integer, nullable=True) + c2_n_positive_firms = Column(Integer, nullable=True) + network = Column(Text(4294000000), nullable=True) + network_order = Column(Text(4294000000), nullable=True) + network_country = Column(Text(4294000000), nullable=True) + + experiment = relationship('Experiment', back_populates='sample', uselist=False) + product = relationship('Product', back_populates='sample', lazy='dynamic') + + def __repr__(self): + return f'' + + +class Product(Base): + __tablename__ = f"{db_name_prefix}_product" + id = Column(Integer, primary_key=True, autoincrement=True) + s_id = Column(Integer, ForeignKey('{}.id'.format(f"{db_name_prefix}_sample")), nullable=False) + + int_name = Column(Integer, nullable=False) + int_tier = Column(Integer, nullable=False) + n_up_products = Column(Integer, nullable=False) + n_peer_products = Column(Integer, nullable=False) + n_positive_firms = Column(Integer, nullable=False) + n_all_firms = Column(Integer, nullable=False) + gini_acc_demand_per_age = Column(DECIMAL(10, 2), nullable=False) + gini_acc_wealth_per_age = Column(DECIMAL(10, 2), nullable=False) + gini_acc_demand_per_age_all = Column(DECIMAL(10, 2), nullable=False) + gini_acc_wealth_per_age_all = Column(DECIMAL(10, 2), nullable=False) + # lst_n_positive_firms = Column(Text(4294000000), nullable=False) + # lst_n_all_firms = Column(Text(4294000000), nullable=False) + # lst_gini_acc_demand_per_age = Column(Text(4294000000), nullable=False) + # lst_gini_acc_wealth_per_age = Column(Text(4294000000), nullable=False) + # lst_gini_acc_demand_per_age_all = Column(Text(4294000000), nullable=False) + # lst_gini_acc_wealth_per_age_all = Column(Text(4294000000), nullable=False) + + sample = relationship('Sample', back_populates='product', uselist=False) + firm = relationship('Firm', back_populates='product', lazy='dynamic') + + def __repr__(self): + return f'' + + +class Firm(Base): + __tablename__ = f"{db_name_prefix}_firm" + id = Column(Integer, primary_key=True, autoincrement=True) + p_id = Column(Integer, ForeignKey('{}.id'.format(f"{db_name_prefix}_product")), nullable=False) + + idx_firm = Column(Integer, nullable=False) + int_n_supplier = Column(Integer, nullable=False) + flt_fix_cost = Column(DECIMAL(20, 2), nullable=False) + flt_q_star = Column(DECIMAL(20, 2), nullable=False) + acc_demand_per_age = Column(DECIMAL(20, 2), nullable=False) + acc_wealth_per_age = Column(DECIMAL(20, 2), nullable=False) + std_demand_per_age = Column(DECIMAL(20, 2), nullable=False) + + product = relationship('Product', back_populates='firm', uselist=False) + + def __repr__(self): + return f'' + + +if __name__ == '__main__': + Base.metadata.drop_all() + Base.metadata.create_all() diff --git a/requirements.txt b/requirements.txt index 4a60a21..d3a9f65 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,6 +8,7 @@ cycler==0.11.0 decorator==5.1.1 dill==0.3.6 docutils==0.19 +greenlet==2.0.2 idna==3.4 imagesize==1.4.1 importlib-metadata==6.0.0 @@ -18,6 +19,7 @@ MarkupSafe==2.1.2 matplotlib==3.3.4 matplotlib-inline==0.1.6 multiprocess==0.70.14 +mysqlclient==2.1.1 networkx==2.5 numpy==1.20.3 numpydoc==1.1.0 @@ -26,10 +28,11 @@ pandas==1.4.1 pandas-stubs==1.2.0.39 Pillow==9.4.0 Pygments==2.14.0 -pygraphviz @ file:///C:/Users/ASUS/OneDrive/Project/ScrAbm/Dissertation/IIabm/pygraphviz-1.9-cp38-cp38-win_amd64.whl +pygraphviz @ file:///C:/Users/ASUS/Downloads/pygraphviz-1.9-cp38-cp38-win_amd64.whl pyparsing==3.0.9 python-dateutil==2.8.2 pytz==2022.7.1 +PyYAML==6.0 requests==2.28.2 SALib==1.4.7 scipy==1.10.1 @@ -42,7 +45,10 @@ sphinxcontrib-htmlhelp==2.0.1 sphinxcontrib-jsmath==1.0.1 sphinxcontrib-qthelp==1.0.3 sphinxcontrib-serializinghtml==1.1.5 +SQLAlchemy==2.0.5.post1 traitlets==5.9.0 +typing_extensions==4.5.0 urllib3==1.26.14 wincertstore==0.2 +yapf @ file:///tmp/build/80754af9/yapf_1615749224965/work zipp==3.15.0