From 475ba356138a6a6c54286ca880d48f7a76958c36 Mon Sep 17 00:00:00 2001 From: HaoYizhi Date: Sun, 21 May 2023 17:05:06 +0800 Subject: [PATCH] with_exp --- .vscode/launch.json | 1 + InitRemovalHighRisk.csv | 420 +++++++++++++++++++++++++++++++ SQL_export_high_risk_setting.sql | 17 ++ SQL_migrate_db.sql | 7 + SQL_running_check.sql | 5 +- __pycache__/firm.cpython-38.pyc | Bin 6174 -> 5937 bytes __pycache__/model.cpython-38.pyc | Bin 8199 -> 8199 bytes controller_db.py | 35 ++- oa_L27.txt | 30 +++ oa_with_exp.csv | 28 +++ oa.csv => oa_without_exp.csv | 0 test.ipynb | 32 +++ 12 files changed, 565 insertions(+), 10 deletions(-) create mode 100644 InitRemovalHighRisk.csv create mode 100644 SQL_export_high_risk_setting.sql create mode 100644 SQL_migrate_db.sql create mode 100644 oa_L27.txt create mode 100644 oa_with_exp.csv rename oa.csv => oa_without_exp.csv (100%) diff --git a/.vscode/launch.json b/.vscode/launch.json index f51306f..596058b 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -14,6 +14,7 @@ "args": [ "--exp", "without_exp", "--job", "24", + "--reset_db", "True", ] } ] diff --git a/InitRemovalHighRisk.csv b/InitRemovalHighRisk.csv new file mode 100644 index 0000000..37abf58 --- /dev/null +++ b/InitRemovalHighRisk.csv @@ -0,0 +1,420 @@ +e_id,count,max_max_ts,dct_lst_init_remove_firm_prod +83,50,3,... +135,50,3,... +138,50,2,... +179,50,3,... +184,50,3,... +190,50,3,... +227,50,2,... +307,50,2,... +358,50,1,... +372,50,1,... +383,50,3,... +405,50,3,... +411,50,2,... +412,50,2,... +445,50,2,... +29,25,1,... +256,24,1,... +257,23,1,... +259,20,1,... +28,19,1,... +30,19,1,... +258,18,1,... +262,18,1,... +260,17,1,... +26,16,1,... +32,16,1,... +27,13,1,... +53,13,1,... +173,13,1,... +400,13,1,... +5,12,1,... +31,12,1,... +89,12,1,... +140,12,1,... +142,12,2,... +234,12,1,... +21,11,1,... +64,11,1,... +244,11,1,... +290,11,1,... +444,11,1,... +65,10,1,... +66,10,1,... +68,10,1,... +92,10,1,... +94,10,1,... +265,10,1,... +335,10,2,... +370,10,1,... +380,10,1,... +388,10,1,... +401,10,1,... +43,9,1,... +45,9,1,... +48,9,1,... +67,9,1,... +90,9,1,... +95,9,1,... +96,9,1,... +143,9,1,... +147,9,1,... +163,9,1,... +165,9,1,... +201,9,1,... +300,9,1,... +306,9,1,... +327,9,1,... +330,9,1,... +381,9,1,... +387,9,1,... +389,9,1,... +8,8,1,... +40,8,1,... +44,8,1,... +46,8,1,... +62,8,1,... +87,8,1,... +91,8,1,... +93,8,1,... +203,8,1,... +210,8,1,... +211,8,1,... +243,8,1,... +313,8,1,... +326,8,1,... +371,8,1,... +382,8,1,... +385,8,1,... +386,8,1,... +391,8,1,... +422,8,1,... +435,8,1,... +6,7,2,... +14,7,1,... +20,7,2,... +47,7,1,... +63,7,1,... +141,7,2,... +154,7,1,... +164,7,1,... +200,7,1,... +204,7,1,... +207,7,1,... +237,7,1,... +246,7,1,... +252,7,1,... +254,7,2,... +261,7,1,... +268,7,1,... +270,7,1,... +286,7,1,... +432,7,1,... +434,7,1,... +436,7,1,... +446,7,1,... +451,7,1,... +4,6,1,... +15,6,1,... +22,6,1,... +25,6,1,... +39,6,1,... +41,6,1,... +42,6,1,... +52,6,1,... +55,6,1,... +56,6,1,... +75,6,1,... +81,6,1,... +82,6,1,... +84,6,1,... +86,6,1,... +113,6,1,... +114,6,1,... +127,6,1,... +139,6,1,... +159,6,1,... +162,6,1,... +233,6,1,... +235,6,1,... +267,6,1,... +292,6,1,... +295,6,1,... +339,6,1,... +349,6,1,... +363,6,1,... +449,6,1,... +460,6,1,... +461,6,1,... +473,6,1,... +474,6,1,... +2,5,1,... +3,5,1,... +7,5,2,... +13,5,1,... +77,5,1,... +101,5,1,... +108,5,1,... +112,5,1,... +123,5,1,... +166,5,1,... +167,5,1,... +202,5,1,... +205,5,1,... +206,5,1,... +215,5,1,... +222,5,1,... +232,5,1,... +238,5,1,... +239,5,1,... +248,5,1,... +253,5,1,... +266,5,1,... +277,5,1,... +331,5,1,... +336,5,1,... +337,5,1,... +348,5,1,... +367,5,1,... +369,5,1,... +376,5,1,... +390,5,1,... +420,5,1,... +423,5,1,... +431,5,1,... +462,5,1,... +465,5,1,... +471,5,1,... +472,5,1,... +16,4,1,... +17,4,1,... +59,4,1,... +60,4,1,... +61,4,1,... +79,4,1,... +85,4,1,... +88,4,1,... +110,4,1,... +126,4,1,... +145,4,1,... +156,4,1,... +169,4,1,... +170,4,1,... +172,4,1,... +219,4,1,... +220,4,1,... +221,4,1,... +224,4,1,... +228,4,1,... +229,4,1,... +245,4,1,... +249,4,1,... +251,4,1,... +287,4,1,... +291,4,1,... +293,4,1,... +305,4,1,... +328,4,1,... +329,4,1,... +338,4,1,... +352,4,1,... +366,4,1,... +374,4,1,... +384,4,1,... +414,4,1,... +415,4,1,... +416,4,1,... +419,4,1,... +421,4,1,... +424,4,1,... +429,4,1,... +430,4,1,... +450,4,1,... +464,4,1,... +466,4,1,... +469,4,1,... +24,3,1,... +36,3,1,... +50,3,1,... +57,3,1,... +76,3,1,... +115,3,1,... +122,3,1,... +144,3,1,... +146,3,2,... +155,3,1,... +157,3,1,... +158,3,1,... +160,3,1,... +161,3,1,... +168,3,1,... +209,3,1,... +216,3,1,... +217,3,1,... +218,3,1,... +241,3,1,... +247,3,1,... +274,3,1,... +275,3,1,... +276,3,1,... +278,3,1,... +282,3,1,... +285,3,1,... +289,3,1,... +296,3,1,... +298,3,1,... +303,3,1,... +310,3,1,... +314,3,1,... +333,3,1,... +347,3,1,... +353,3,1,... +355,3,1,... +364,3,1,... +365,3,1,... +373,3,1,... +377,3,1,... +378,3,1,... +409,3,1,... +413,3,1,... +417,3,1,... +418,3,1,... +428,3,1,... +459,3,1,... +467,3,1,... +470,3,1,... +9,2,1,... +34,2,1,... +37,2,1,... +58,2,1,... +80,2,1,... +104,2,1,... +109,2,1,... +111,2,1,... +116,2,1,... +117,2,1,... +124,2,1,... +125,2,1,... +130,2,1,... +132,2,1,... +133,2,1,... +149,2,1,... +150,2,1,... +151,2,1,... +153,2,1,... +171,2,1,... +177,2,1,... +178,2,1,... +181,2,1,... +187,2,1,... +195,2,1,... +214,2,1,... +223,2,1,... +230,2,1,... +236,2,1,... +280,2,1,... +288,2,1,... +297,2,1,... +304,2,1,... +309,2,1,... +312,2,1,... +322,2,1,... +324,2,1,... +325,2,1,... +334,2,1,... +345,2,1,... +346,2,1,... +354,2,1,... +359,2,1,... +361,2,1,... +375,2,1,... +392,2,1,... +396,2,1,... +398,2,1,... +403,2,1,... +404,2,1,... +406,2,1,... +427,2,1,... +438,2,1,... +442,2,1,... +454,2,1,... +457,2,1,... +468,2,1,... +18,1,1,... +19,1,1,... +35,1,1,... +38,1,1,... +49,1,1,... +51,1,1,... +54,1,1,... +70,1,1,... +71,1,1,... +72,1,1,... +73,1,1,... +74,1,1,... +78,1,1,... +98,1,1,... +102,1,1,... +103,1,1,... +105,1,1,... +106,1,1,... +107,1,1,... +128,1,1,... +152,1,1,... +174,1,1,... +176,1,1,... +180,1,1,... +191,1,1,... +192,1,1,... +194,1,1,... +196,1,1,... +197,1,1,... +212,1,1,... +225,1,1,... +226,1,1,... +231,1,1,... +250,1,1,... +255,1,1,... +269,1,1,... +271,1,1,... +279,1,1,... +281,1,1,... +283,1,1,... +284,1,1,... +294,1,1,... +299,1,1,... +308,1,1,... +315,1,1,... +316,1,1,... +317,1,1,... +319,1,1,... +321,1,1,... +340,1,1,... +341,1,1,... +342,1,1,... +343,1,1,... +344,1,1,... +350,1,1,... +356,1,1,... +357,1,1,... +362,1,1,... +379,1,1,... +393,1,1,... +394,1,1,... +395,1,1,... +397,1,1,... +399,1,1,... +402,1,1,... +410,1,1,... +425,1,1,... +439,1,1,... +440,1,1,... +441,1,1,... +443,1,1,... +447,1,1,... +448,1,1,... +455,1,1,... +456,1,1,... +458,1,1,... +463,1,1,... diff --git a/SQL_export_high_risk_setting.sql b/SQL_export_high_risk_setting.sql new file mode 100644 index 0000000..0fe1d06 --- /dev/null +++ b/SQL_export_high_risk_setting.sql @@ -0,0 +1,17 @@ +select distinct s_id from iiabmdb.without_exp_result where ts > 0; +select s_id, max(ts) as max_ts from iiabmdb.without_exp_result where ts > 0 group by s_id order by max_ts; +select e_id, count(id) as count, max(max_ts) as max_max_ts from iiabmdb.without_exp_sample as a +inner join (select s_id, max(ts) as max_ts from iiabmdb.without_exp_result where ts > 0 group by s_id) as b +on a.id = b.s_id +group by e_id +order by count desc; + +select e_id, count, max_max_ts, dct_lst_init_remove_firm_prod from iiabmdb.without_exp_experiment as a +inner join +(select e_id, count(id) as count, max(max_ts) as max_max_ts from iiabmdb.without_exp_sample as a +inner join (select s_id, max(ts) as max_ts from iiabmdb.without_exp_result where ts > 0 group by s_id) as b +on a.id = b.s_id +group by e_id) as b +on a.id = b.e_id +where count >= 9 +order by count desc; \ No newline at end of file diff --git a/SQL_migrate_db.sql b/SQL_migrate_db.sql new file mode 100644 index 0000000..57fe3c6 --- /dev/null +++ b/SQL_migrate_db.sql @@ -0,0 +1,7 @@ +CREATE DATABASE iiabmdb_dissertation; +RENAME TABLE iiabmdb.not_test_experiment TO iiabmdb_dissertation.not_test_experiment, +iiabmdb.not_test_result TO iiabmdb_dissertation.not_test_result, +iiabmdb.not_test_sample TO iiabmdb_dissertation.not_test_sample, +iiabmdb.test_experiment TO iiabmdb_dissertation.test_experiment, +iiabmdb.test_result TO iiabmdb_dissertation.test_result, +iiabmdb.test_sample TO iiabmdb_dissertation.test_sample; \ No newline at end of file diff --git a/SQL_running_check.sql b/SQL_running_check.sql index dae4b33..2145b42 100644 --- a/SQL_running_check.sql +++ b/SQL_running_check.sql @@ -1,3 +1,4 @@ -select id, e_id, idx_sample, seed, ts_done from iiabmdb.without_exp_sample where is_done_flag != -1; +select id, e_id, idx_sample, seed, ts_done from iiabmdb.without_exp_sample where is_done_flag != -1 order by ts_done; select count(id) from iiabmdb.without_exp_sample where is_done_flag != -1; -select count(id) from iiabmdb.without_exp_sample; \ No newline at end of file +select count(id) from iiabmdb.without_exp_sample; +select count(id) from iiabmdb.without_exp_sample where is_done_flag != -1; \ No newline at end of file diff --git a/__pycache__/firm.cpython-38.pyc b/__pycache__/firm.cpython-38.pyc index 544c94f9ee57983dd5b24c8e81d0a3d8996042f6..b5d38d84d45ad310131bc55561f01d84cc436610 100644 GIT binary patch delta 1017 zcmZuv&1(}u6rVTyz1hu2Yb+6Ky$I$a>cK*(AQs zULW|n-7xW{;TsVkB&Go=ki|cnfG3C{ClMNG6$s4%!k|LKDTV7DqYW>BQKq%*K2jh} z=*ny}hDn)?-yDfbRkKYoQa_GVXkZmcqY0a&P0(d17q)LgT^68$9XNvm9GLl*)@cl! z25!ZS4r>Wcw+XXi*d`yfCcHtq=<_;$msHH~mpseXon%bA0A$UJ>9UGACWi3_$qf-^ zkD~ySq-ruUL-I~hOdDgpcg}kd_Y(KDRG1MT65s48DI2&{EPJ;+%L5bWk$$A>NwMb@ z8v9Z_2uKIIIKdVy?v!X9B;SB?WPzgTp-xTF{iuebpHTfY)o-Hz3wh@@)Xs0Hw(S>tUz67WX2jjp<(`(l z128KlGV3rW_A*x?C4OXf9!%rr7gAAqrMy&owzLt|)>q46cm>fs4Ikqt9F=Aa&Cle^ z=WA@G{46Y2H_A~hLM*DGAyn)S?ZTmWGF*huVt05Ux0dD*q+=rgQBI_=ijrLvf76%5 z^o1`@J`?Jg8WzQ9c3w2H58ZjZ?&2F=7vHmanlyWGu?5*jGU}-`G9WBTvnYO#jEIAg F)qhu6@ht!V delta 1252 zcmZuw&1(}u6rVS{JG;r|yJ@uI$Et|bL(z-%pdylkAXcz`QCdsQrq)eAlG&|bnGi66 zt_MXJFCvEIB6twgUOjvE+`9*_{sa1Ew$-98?CiYvX5M?h_xtU~iT>&At!y?;@aePr z{^G~1>2z$g41B|LKuAmjQY9~7?$lF=04@5O_z**2ye4mqE`T~=#xSP7R=^s-*{;W@ z4a%3a-yS%dun4~4n>{D-R7n?xDq|{!^XfK!$4D4;T5OZzQETU@b_Z#7{3+Qa7-2es zZFQlo5?Fy7WOfq>@a<=`O@BA*B+dFMp__kW>81g#Y=ebAqMT5W(lI1}o-80b4CV)g zA!)D|z{|=~y$y;n9U|gYEcQU3+36hdTV#y*?j#QGJZUkXOq21<)W)b6*6R&#VL9@` z^=5PVvA4L!y{)6E4{)S{vE^FycW{Gwi+pXxlV8)-a)}p_h;6IMAFf8+!9BO-KKD!d zE^Y~ApaSxnd!}Si*#FZsGr)lUG;vpu$1>H7jtBV-C(>aj6qJA zilJ5$ff$_x2~v1Pr%UOJ*yC>{rIja6l%#ZG2l=2NsBnB28%5NgSu4>)p=ig2tYL+5Z+a{$Eh5C^xm8TUuE4m+SMq-deAR(bl2d34mMjRqo){ zeEux3+e#OgWk0u1ekx4E9m$HGTjbZ2TWLm~EcP$lz|Bk^C0DOS^R@LbT3e}eei*&u zlq)&$LuxsqmYcYQ>V1TZl9_Z%Z(&=0Dow!`2_rL;Q9+QPi2n^vFL3dTQ&M*FcdZuC zD#iFqAR$B%qM@XxWHWz2J|B5^e8ce;YPEVZ@-}kb4z5J^<0IU}qWrqNRKCj+4Dx09 qYI+8}Z}AN$zVJa!lpK0D@`rGdFVokp}=XeFcaB delta 20 acmZp7Xm{Wa<>lpK0D^sNNgKKU$O8Z{+yxK- diff --git a/controller_db.py b/controller_db.py index a5e0e36..d28b951 100644 --- a/controller_db.py +++ b/controller_db.py @@ -9,6 +9,7 @@ import pandas as pd import platform import networkx as nx import json +import pickle class ControllerDB: @@ -45,13 +46,31 @@ class ControllerDB: Firm.fillna(0, inplace=True) # fill dct_lst_init_remove_firm_prod - list_dct = [] - for _, row in Firm.iterrows(): - code = row['Code'] - row = row['1':] - for product_code in row.index[row == 1].to_list(): - dct = {code: [product_code]} - list_dct.append(dct) + # list_dct = [] + # for _, row in Firm.iterrows(): + # code = row['Code'] + # row = row['1':] + # for product_code in row.index[row == 1].to_list(): + # dct = {code: [product_code]} + # list_dct.append(dct) + str_sql = "select e_id, count, max_max_ts, " \ + "dct_lst_init_remove_firm_prod from " \ + "iiabmdb.without_exp_experiment as a " \ + "inner join " \ + "(select e_id, count(id) as count, max(max_ts) as max_max_ts " \ + "from iiabmdb.without_exp_sample as a " \ + "inner join (select s_id, max(ts) as max_ts from " \ + "iiabmdb.without_exp_result where ts > 0 group by s_id) as b " \ + "on a.id = b.s_id " \ + "group by e_id) as b " \ + "on a.id = b.e_id " \ + "order by count desc;" + result = pd.read_sql(sql=str_sql, con=engine) + result['dct_lst_init_remove_firm_prod'] = \ + result['dct_lst_init_remove_firm_prod'].apply( + lambda x: pickle.loads(x)) + list_dct = result.loc[result['count'] >= 9, + 'dct_lst_init_remove_firm_prod'].to_list() # list_dct = [{'140': ['1.4.5.1']}] # list_dct = [{'133': ['1.4.4.1']}] # list_dct = [{'2': ['1.1.3']}] @@ -74,7 +93,7 @@ class ControllerDB: # insert exp df_xv = pd.read_csv("xv.csv", index_col=None) # read the OA table - df_oa = pd.read_csv("oa.csv", index_col=None) + df_oa = pd.read_csv("oa_with_exp.csv", index_col=None) for idx_scenario, row in df_oa.iterrows(): dct_exp_para = {} for idx_col, para_level in enumerate(row): diff --git a/oa_L27.txt b/oa_L27.txt new file mode 100644 index 0000000..483e79d --- /dev/null +++ b/oa_L27.txt @@ -0,0 +1,30 @@ +X X X X X X X X X X X X X +1 2 3 4 5 6 7 8 9 10 11 12 13 +------------------------------------------- +1 1 1 1 1 1 1 1 1 1 1 1 1 +1 1 1 1 2 2 2 2 2 2 2 2 2 +1 1 1 1 3 3 3 3 3 3 3 3 3 +1 2 2 2 1 1 1 2 2 2 3 3 3 +1 2 2 2 2 2 2 3 3 3 1 1 1 +1 2 2 2 3 3 3 1 1 1 2 2 2 +1 3 3 3 1 1 1 3 3 3 2 2 2 +1 3 3 3 2 2 2 1 1 1 3 3 3 +1 3 3 3 3 3 3 2 2 2 1 1 1 +2 1 2 3 1 2 3 1 2 3 1 2 3 +2 1 2 3 2 3 1 2 3 1 2 3 1 +2 1 2 3 3 1 2 3 1 2 3 1 2 +2 2 3 1 1 2 3 2 3 1 3 1 2 +2 2 3 1 2 3 1 3 1 2 1 2 3 +2 2 3 1 3 1 2 1 2 3 2 3 1 +2 3 1 2 1 2 3 3 1 2 2 3 1 +2 3 1 2 2 3 1 1 2 3 3 1 2 +2 3 1 2 3 1 2 2 3 1 1 2 3 +3 1 3 2 1 3 2 1 3 2 1 3 2 +3 1 3 2 2 1 3 2 1 3 2 1 3 +3 1 3 2 3 2 1 3 2 1 3 2 1 +3 2 1 3 1 3 2 2 1 3 3 2 1 +3 2 1 3 2 1 3 3 2 1 1 3 2 +3 2 1 3 3 2 1 1 3 2 2 1 3 +3 3 2 1 1 3 2 3 2 1 2 1 3 +3 3 2 1 2 1 3 1 3 2 3 2 1 +3 3 2 1 3 2 1 2 1 3 1 3 2 \ No newline at end of file diff --git a/oa_with_exp.csv b/oa_with_exp.csv new file mode 100644 index 0000000..b000b5f --- /dev/null +++ b/oa_with_exp.csv @@ -0,0 +1,28 @@ +X1,X2,X3,X4,X5,X6,X7,X8,X9 +0,0,0,0,0,0,0,0,0 +0,0,0,0,1,1,1,1,1 +0,0,0,0,2,2,2,2,2 +0,1,1,1,0,0,0,1,1 +0,1,1,1,1,1,1,2,2 +0,1,1,1,2,2,2,0,0 +0,2,2,2,0,0,0,2,2 +0,2,2,2,1,1,1,0,0 +0,2,2,2,2,2,2,1,1 +1,0,1,2,0,1,2,0,1 +1,0,1,2,1,2,0,1,2 +1,0,1,2,2,0,1,2,0 +1,1,2,0,0,1,2,1,2 +1,1,2,0,1,2,0,2,0 +1,1,2,0,2,0,1,0,1 +1,2,0,1,0,1,2,2,0 +1,2,0,1,1,2,0,0,1 +1,2,0,1,2,0,1,1,2 +2,0,2,1,0,2,1,0,2 +2,0,2,1,1,0,2,1,0 +2,0,2,1,2,1,0,2,1 +2,1,0,2,0,2,1,1,0 +2,1,0,2,1,0,2,2,1 +2,1,0,2,2,1,0,0,2 +2,2,1,0,0,2,1,2,1 +2,2,1,0,1,0,2,0,2 +2,2,1,0,2,1,0,1,0 diff --git a/oa.csv b/oa_without_exp.csv similarity index 100% rename from oa.csv rename to oa_without_exp.csv diff --git a/test.ipynb b/test.ipynb index b3c97e9..37b947c 100644 --- a/test.ipynb +++ b/test.ipynb @@ -136,6 +136,38 @@ "\n", "print(mp.cpu_count())" ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "71\n" + ] + } + ], + "source": [ + "from orm import engine\n", + "import pandas as pd\n", + "import pickle\n", + "str_sql = \"select e_id, count, max_max_ts, dct_lst_init_remove_firm_prod from iiabmdb.without_exp_experiment as a \" \\\n", + "\"inner join \" \\\n", + "\"(select e_id, count(id) as count, max(max_ts) as max_max_ts from iiabmdb.without_exp_sample as a \" \\\n", + "\"inner join (select s_id, max(ts) as max_ts from iiabmdb.without_exp_result where ts > 0 group by s_id) as b \" \\\n", + "\"on a.id = b.s_id \" \\\n", + "\"group by e_id) as b \" \\\n", + "\"on a.id = b.e_id \" \\\n", + "\"order by count desc;\"\n", + "result = pd.read_sql(sql=str_sql, con=engine)\n", + "result['dct_lst_init_remove_firm_prod'] = result['dct_lst_init_remove_firm_prod'].apply(lambda x: pickle.loads(x))\n", + "# print(result)\n", + "list_dct = result.loc[result['count']>=9, 'dct_lst_init_remove_firm_prod'].to_list()\n", + "print(len(list_dct))" + ] } ], "metadata": {