data collection

This commit is contained in:
HaoYizhi 2023-03-12 12:02:01 +08:00
parent 586272c923
commit 0265e6faa7
14 changed files with 529 additions and 28 deletions

2
.vscode/launch.json vendored
View File

@ -8,7 +8,7 @@
"name": "Python: Current File",
"type": "python",
"request": "launch",
// "program": "C:\\Users\\ASUS\\OneDrive\\Project\\ScrAbm\\Dissertation\\IIabm\\model.py",
"program": "C:\\Users\\ASUS\\OneDrive\\Project\\ScrAbm\\Dissertation\\IIabm\\main.py",
"console": "integratedTerminal",
"justMyCode": true
}

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

52
computation.py Normal file
View File

@ -0,0 +1,52 @@
import os
import datetime
from model import Model
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from controller_db import ControllerDB
class Computation:
def __init__(self, c_db: 'ControllerDB'):
self.c_db = c_db
self.pid = os.getpid()
def run(self, str_code='0', s_id=None):
sample_random = self.c_db.fetch_a_sample(s_id)
if sample_random is None:
return True
# lock this row by update is_done_flag to 0
self.c_db.lock_the_sample(sample_random)
print(f"Pid {self.pid} ({str_code}) is running sample {sample_random.id} at {datetime.datetime.now()}")
dct_exp = {column: getattr(sample_random.experiment, column)
for column in sample_random.experiment.__table__.c.keys()}
del dct_exp['id']
dct_sample_para = {'sample': sample_random,
'seed': sample_random.seed,
**dct_exp}
model = Model(dct_sample_para)
results = model.run(display=False)
return False
if __name__ == '__main__':
str_exp = 'test'
from controller_db import ControllerDB
controller_db = ControllerDB(str_exp)
controller_db.reset_db()
# print(controller_db.dct_parameter)
exp = Computation(controller_db)
is_all_done = exp.run('999', 87)
# while 1:
# # time.sleep(random.uniform(0, 10))
# is_all_done = exp.run(str_exp)
# if is_all_done:
# break

16
conf_db.yaml Normal file
View File

@ -0,0 +1,16 @@
# read by orm
is_local_db: True
local:
user_name: iiabm_yz
password: iiabm_yz
db_name: iiabmdb
address: 'localhost'
port: 3306
remote:
user_name: iiabm_yz
password: iiabm_yz
db_name: iiabmdb
address: 'localhost'
port: 3307

1
conf_db_prefix.yaml Normal file
View File

@ -0,0 +1 @@
db_name_prefix: test

38
conf_experiment.yaml Normal file
View File

@ -0,0 +1,38 @@
# read by ControllerDB
# run settings
meta_seed: 0
fixed: # unchanged all the time
int_n_country: 2
max_int_n_supplier: 3 # make firms heterogeneous
flt_bm_price_ratio: 20.0
flt_beta_developing: 0.5
test: # only for test scenarios
int_n_product: 12
int_n_firm_per_product_per_country: 2
flt_demand_total: 1000.0
n_sample: 5
n_iter: 100
not_test: # normal scenarios
int_n_product: 50
int_n_firm_per_product_per_country: 10
flt_demand_total: 10000.0
n_sample: 50
n_iter: 10000
default:
is_eliminated: 0 # add when all positive profits and keep max n supplier; remove the worst when all negative wealth
flt_beta_developed: 0.5 # benchmarks flt_beta_developed
tariff_percentage: 0
experiment:
1:
range_lambda_tier: 0, 1, 0.1 # describe the network. 0: chain 1: one iter
2:
is_eliminated: 1
3:
range_flt_beta_developed: 0.5, 0.9, 0.1
range_tariff_percentage: 0, 1, 0.1

237
controller_db.py Normal file
View File

@ -0,0 +1,237 @@
# -*- coding: utf-8 -*-
from orm import db_session, engine, Base, ins
from orm import Experiment, Sample, Product, Firm
from sqlalchemy.exc import OperationalError
import yaml
import random
import numpy as np
import platform
class ControllerDB:
dct_parameter = None
is_test: bool = None
db_name_prefix: str = None
reset_flag: int
lst_saved_s_id_3: list
lst_saved_s_id_1_2: list
# n_sample_1_2: int
def __init__(self, prefix, reset_flag=0):
with open('conf_experiment.yaml') as yaml_file:
dct_conf_experiment = yaml.full_load(yaml_file)
self.is_test = prefix == 'test'
self.db_name_prefix = prefix
dct_para_in_test = dct_conf_experiment['test'] if self.is_test else dct_conf_experiment['not_test']
self.dct_parameter = {'meta_seed': dct_conf_experiment['meta_seed'],
'experiment': dct_conf_experiment['experiment'],
**dct_conf_experiment['fixed'], **dct_conf_experiment['default'], **dct_para_in_test}
self.reset_flag = reset_flag # 0, not reset; 1, reset self; 2, reset all
self.lst_saved_s_id_1_2, self.lst_saved_s_id_3 = [], []
def init_tables(self):
self.fill_experiment_table()
self.fill_sample_table()
@staticmethod
def get_lst_of_range(str_range: str):
s1, s2, s3 = tuple(str_range.split(','))
return list(np.linspace(float(s1), float(s2), num=int((float(s2) - float(s1)) / float(s3)) + 1))
def fill_experiment_table(self):
# prepare the list of lambda tier
lst_lambda = self.get_lst_of_range(self.dct_parameter['experiment'][1]['range_lambda_tier'])
# prepare the list of alpha_2nd_country
lst_beta_developed = self.get_lst_of_range(self.dct_parameter['experiment'][3]['range_flt_beta_developed'])
# prepare the list of tariff_percentage
lst_tariff = self.get_lst_of_range(self.dct_parameter['experiment'][3]['range_tariff_percentage'])
# prepare the default values
is_eliminated = int(self.dct_parameter['is_eliminated'])
beta_developed = float(self.dct_parameter['flt_beta_developed'])
tariff_percentage_1 = tariff_percentage_2 = float(self.dct_parameter['tariff_percentage'])
for idx_scenario in self.dct_parameter['experiment'].keys():
n_exp = 0
# if idx_scenario == 1: # add S1 experiments
# n_exp = self.add_experiment_1(idx_scenario, lst_lambda, is_eliminated,
# beta_developed, tariff_percentage_1, tariff_percentage_2)
# if idx_scenario == 2: # add S2 experiments
# n_exp = self.add_experiment_1(idx_scenario, lst_lambda,
# int(self.dct_parameter['experiment'][idx_scenario]['is_eliminated']),
# beta_developed, tariff_percentage_1, tariff_percentage_2)
if idx_scenario == 3:
# int_eliminated = int(self.dct_parameter['experiment'][idx_scenario-1]['is_eliminated'])
int_eliminated = is_eliminated # is_eliminated is 0 at default, so stop eliminating under S3
for beta_developed in lst_beta_developed:
# for beta_developed in [0.5]: # fix beta as 0.5
for tariff_percentage_1 in lst_tariff:
for tariff_percentage_2 in lst_tariff:
# fix lambda as 0.5
n_exp += self.add_experiment_1(idx_scenario, [0.5], int_eliminated,
beta_developed, tariff_percentage_1, tariff_percentage_2)
print(f'Inserted {n_exp} experiments for exp {idx_scenario}!')
def add_experiment_1(self, idx_exp, lst_lambda, is_eliminated, flt_beta_developed,
tariff_percentage_1: float, tariff_percentage_2: float):
lst_exp = []
for lambda_tier in lst_lambda:
e = Experiment(idx_exp=idx_exp,
int_n_country=int(self.dct_parameter['int_n_country']),
max_int_n_supplier=int(self.dct_parameter['max_int_n_supplier']),
int_n_product=int(self.dct_parameter['int_n_product']),
int_n_firm_per_product_per_country=int(
self.dct_parameter['int_n_firm_per_product_per_country']),
flt_demand_total=float(self.dct_parameter['flt_demand_total']),
flt_bm_price_ratio=float(self.dct_parameter['flt_bm_price_ratio']),
flt_beta_developing=float(self.dct_parameter['flt_beta_developing']),
n_sample=int(self.dct_parameter['n_sample']),
n_iter=int(self.dct_parameter['n_iter']),
is_eliminated=is_eliminated,
flt_beta_developed=flt_beta_developed,
tariff_percentage_1=tariff_percentage_1,
tariff_percentage_2=tariff_percentage_2,
lambda_tier=float(lambda_tier))
lst_exp.append(e)
db_session.bulk_save_objects(lst_exp)
db_session.commit()
return len(lst_exp)
def fill_sample_table(self):
rng = random.Random(self.dct_parameter['meta_seed'])
lst_seed = [rng.getrandbits(32) for _ in range(int(self.dct_parameter['n_sample']))]
lst_exp = db_session.query(Experiment).all()
lst_sample = []
for experiment in lst_exp:
for idx_sample in range(int(experiment.n_sample)):
s = Sample(e_id=experiment.id,
idx_sample=idx_sample+1,
seed=lst_seed[idx_sample],
is_done_flag=-1)
lst_sample.append(s)
db_session.bulk_save_objects(lst_sample)
db_session.commit()
print(f'Inserted {len(lst_sample)} samples!')
def reset_db(self, force_drop=False):
# first, check if tables exist
lst_table_obj = [Base.metadata.tables[str_table] for str_table in ins.get_table_names()
if str_table.startswith(self.db_name_prefix)]
is_exist = len(lst_table_obj) > 0
if force_drop:
while is_exist:
a_table = random.choice(lst_table_obj)
try:
Base.metadata.drop_all(bind=engine, tables=[a_table])
except KeyError:
pass
except OperationalError:
pass
else:
lst_table_obj.remove(a_table)
print(f"Table {a_table.name} is dropped for exp: {self.db_name_prefix}!!!")
finally:
is_exist = len(lst_table_obj) > 0
if is_exist:
print(f"All tables exist. No need to reset for exp: {self.db_name_prefix}.")
# change the is_done_flag from 0 to -1, to rerun the in-finished tasks
if self.reset_flag > 0:
if self.reset_flag == 2:
result = db_session.query(Sample).filter(Sample.is_done_flag == 0)
elif self.reset_flag == 1:
result = db_session.query(Sample).filter(Sample.is_done_flag == 0,
Sample.computer_name == platform.node())
else:
raise ValueError('Wrong reset flag')
if result.count() > 0:
for res in result:
qry_product = db_session.query(Product).filter_by(s_id=res.id)
if qry_product.count() > 0:
for p in qry_product:
db_session.query(Firm).filter(Firm.p_id == p.id).delete()
db_session.commit()
db_session.query(Product).filter(Product.id == p.id).delete()
db_session.commit()
res.is_done_flag = -1
db_session.commit()
print(f"Reset the task id {res.id} flag from 0 to -1")
else:
Base.metadata.create_all()
self.init_tables()
print(f"All tables are just created and initialized for exp: {self.db_name_prefix}.")
def prepare_list_sample(self):
res = db_session.execute(f'''SELECT count(*) FROM {self.db_name_prefix}_sample s,
{self.db_name_prefix}_experiment e WHERE s.e_id=e.id and e.idx_exp < 3''').scalar()
n_sample_1_2 = 0 if res is None else res
print(f'There are {n_sample_1_2} sample for exp 1 and 2.')
res = db_session.execute(f'SELECT id FROM {self.db_name_prefix}_sample WHERE is_done_flag = -1')
for row in res:
s_id = row[0]
if s_id <= n_sample_1_2:
self.lst_saved_s_id_1_2.append(s_id)
else:
self.lst_saved_s_id_3.append(s_id)
print(f'Left: {len(self.lst_saved_s_id_1_2)} for exp 1 and 2; {len(self.lst_saved_s_id_3)} for exp 3')
@staticmethod
def select_random_sample(lst_s_id):
while 1:
if len(lst_s_id) == 0:
return None
s_id = random.choice(lst_s_id)
lst_s_id.remove(s_id)
res = db_session.query(Sample).filter(Sample.id == int(s_id), Sample.is_done_flag == -1)
if res.count() == 1:
return res[0]
def fetch_a_sample(self, s_id=None):
if s_id is not None:
res = db_session.query(Sample).filter(Sample.id == int(s_id))
if res.count() == 0:
return None
else:
return res[0]
sample = self.select_random_sample(self.lst_saved_s_id_1_2)
if sample is not None:
return sample
sample = self.select_random_sample(self.lst_saved_s_id_3)
if sample is not None:
return sample
return None
@staticmethod
def lock_the_sample(sample: Sample):
sample.is_done_flag, sample.computer_name = 0, platform.node()
db_session.commit()
if __name__ == '__main__':
# pprint.pprint(dct_exp_config)
# pprint.pprint(dct_conf_problem)
db = ControllerDB('first')
ratio = db_session.execute('SELECT COUNT(*) / 332750 FROM first_sample s WHERE s.is_done_flag = 1').scalar()
print(ratio)
# db.fill_experiment_table()
# print(db.dct_parameter)
# db.init_tables()
# db.fill_sample_table()
# pprint.pprint(dct_conf_exp)
# db.update_bi()
# db.reset_db(force_drop=True)
# db.prepare_list_sample()
#
# for i in range(1000):
# if i % 10 == 0:
# print(i)
# print(len(db.lst_saved_s_id_1_2), len(db.lst_saved_s_id_3))
# r = db.fetch_a_sample()
# if i % 10 == 0:
# print(len(db.lst_saved_s_id_1_2), len(db.lst_saved_s_id_3))
# print(r, r.experiment.idx_exp)
# if i == 400:
# print()
# pass

48
main.py
View File

@ -19,34 +19,34 @@ def do_computation(c_db):
break
# if __name__ == '__main__':
# parser = argparse.ArgumentParser(description='setting')
# parser.add_argument('--exp', type=str, default='test')
# parser.add_argument('--job', type=int, default='3')
# parser.add_argument('--reset', type=int, default='0')
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='setting')
parser.add_argument('--exp', type=str, default='test')
parser.add_argument('--job', type=int, default='3')
parser.add_argument('--reset', type=int, default='0')
# args = parser.parse_args()
# assert args.job >= 1, 'Number of jobs should >= 1'
args = parser.parse_args()
assert args.job >= 1, 'Number of jobs should >= 1'
# prefix_file_name = 'conf_db_prefix.yaml'
# if os.path.exists(prefix_file_name):
# os.remove(prefix_file_name)
# with open(prefix_file_name, 'w', encoding='utf-8') as file:
# yaml.dump({'db_name_prefix': args.exp}, file)
prefix_file_name = 'conf_db_prefix.yaml'
if os.path.exists(prefix_file_name):
os.remove(prefix_file_name)
with open(prefix_file_name, 'w', encoding='utf-8') as file:
yaml.dump({'db_name_prefix': args.exp}, file)
# from controller_db import ControllerDB
# controller_db = ControllerDB(args.exp, reset_flag=args.reset)
# controller_db.reset_db()
from controller_db import ControllerDB
controller_db = ControllerDB(args.exp, reset_flag=args.reset)
controller_db.reset_db()
# controller_db.prepare_list_sample()
controller_db.prepare_list_sample()
# close_all_sessions()
close_all_sessions()
# process_list = []
# for i in range(int(args.job)):
# p = Process(target=do_computation, args=(controller_db,))
# p.start()
# process_list.append(p)
process_list = []
for i in range(int(args.job)):
p = Process(target=do_computation, args=(controller_db,))
p.start()
process_list.append(p)
# for i in process_list:
# i.join()
for i in process_list:
i.join()

View File

@ -324,5 +324,5 @@ class Model(ap.Model):
plt.savefig("network.png")
model = Model(dct_sample_para)
model.run()
# model = Model(dct_sample_para)
# model.run()

151
orm.py Normal file
View File

@ -0,0 +1,151 @@
# -*- coding: utf-8 -*-
from sqlalchemy import create_engine, inspect
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import Column, Integer, String, ForeignKey, BigInteger, DECIMAL, DateTime, Text
from sqlalchemy.sql import func
from sqlalchemy.orm import relationship, Session
from sqlalchemy.pool import NullPool
import yaml
with open('conf_db.yaml') as file:
dct_conf_db_all = yaml.full_load(file)
is_local_db = dct_conf_db_all['is_local_db']
if is_local_db:
dct_conf_db = dct_conf_db_all['local']
else:
dct_conf_db = dct_conf_db_all['remote']
with open('conf_db_prefix.yaml') as file:
dct_conf_db_prefix = yaml.full_load(file)
db_name_prefix = dct_conf_db_prefix['db_name_prefix']
str_login = 'mysql://{}:{}@{}:{}/{}'.format(dct_conf_db['user_name'], dct_conf_db['password'],
dct_conf_db['address'], dct_conf_db['port'], dct_conf_db['db_name'])
print('DB is {}:{}/{}'.format(dct_conf_db['address'], dct_conf_db['port'], dct_conf_db['db_name']))
engine = create_engine(str_login, poolclass=NullPool) # must be null pool to avoid connection lost error
ins = inspect(engine)
Base = declarative_base(constructor=engine)
db_session = Session(bind=engine)
class Experiment(Base):
__tablename__ = f"{db_name_prefix}_experiment"
id = Column(Integer, primary_key=True, autoincrement=True)
idx_exp = Column(Integer, nullable=False)
# fixed parameters
int_n_country = Column(Integer, nullable=False)
max_int_n_supplier = Column(Integer, nullable=False) # uni(1, max), random parameter 1 of firm
int_n_product = Column(Integer, nullable=False)
int_n_firm_per_product_per_country = Column(Integer, nullable=False)
flt_demand_total = Column(DECIMAL(10, 2), nullable=False) # tri(0, total_demand, mean), to compute random para a
flt_bm_price_ratio = Column(DECIMAL(10, 2), nullable=False) # benchmark value of b, same for both countries
flt_beta_developing = Column(DECIMAL(10, 2), nullable=False) # benchmark value of c(beta), for developing countries
n_sample = Column(Integer, nullable=False)
n_iter = Column(Integer, nullable=False)
# variables
is_eliminated = Column(Integer, nullable=False)
flt_beta_developed = Column(DECIMAL(10, 2), nullable=False) # larger, for developed countries
lambda_tier = Column(DECIMAL(10, 2), nullable=False)
tariff_percentage_1 = Column(DECIMAL(10, 2), nullable=False)
tariff_percentage_2 = Column(DECIMAL(10, 2), nullable=False)
sample = relationship('Sample', back_populates='experiment', lazy='dynamic')
def __repr__(self):
return f'<Experiment: {self.id}>'
class Sample(Base):
__tablename__ = f"{db_name_prefix}_sample"
id = Column(Integer, primary_key=True, autoincrement=True)
e_id = Column(Integer, ForeignKey('{}.id'.format(f"{db_name_prefix}_experiment")), nullable=False)
idx_sample = Column(Integer, nullable=False)
seed = Column(BigInteger, nullable=False)
is_done_flag = Column(Integer, nullable=False) # -1, waiting; 0, running; 1, done
computer_name = Column(String(64), nullable=True)
ts_done = Column(DateTime(timezone=True), onupdate=func.now())
stop_t = Column(Integer, nullable=True)
c1_wealth = Column(DECIMAL(20, 2), nullable=True) # country 1, developing countries
c2_wealth = Column(DECIMAL(20, 2), nullable=True) # country 2, developed countries
c1_wealth_dgt = Column(Integer, nullable=True)
c2_wealth_dgt = Column(Integer, nullable=True)
c1_tariff = Column(DECIMAL(20, 2), nullable=True) # country 1, developing countries
c2_tariff = Column(DECIMAL(20, 2), nullable=True) # country 2, developed countries
c1_tariff_dgt = Column(Integer, nullable=True)
c2_tariff_dgt = Column(Integer, nullable=True)
c1_n_firms = Column(Integer, nullable=True)
c2_n_firms = Column(Integer, nullable=True)
c1_n_positive_firms = Column(Integer, nullable=True)
c2_n_positive_firms = Column(Integer, nullable=True)
network = Column(Text(4294000000), nullable=True)
network_order = Column(Text(4294000000), nullable=True)
network_country = Column(Text(4294000000), nullable=True)
experiment = relationship('Experiment', back_populates='sample', uselist=False)
product = relationship('Product', back_populates='sample', lazy='dynamic')
def __repr__(self):
return f'<Sample id: {self.id}>'
class Product(Base):
__tablename__ = f"{db_name_prefix}_product"
id = Column(Integer, primary_key=True, autoincrement=True)
s_id = Column(Integer, ForeignKey('{}.id'.format(f"{db_name_prefix}_sample")), nullable=False)
int_name = Column(Integer, nullable=False)
int_tier = Column(Integer, nullable=False)
n_up_products = Column(Integer, nullable=False)
n_peer_products = Column(Integer, nullable=False)
n_positive_firms = Column(Integer, nullable=False)
n_all_firms = Column(Integer, nullable=False)
gini_acc_demand_per_age = Column(DECIMAL(10, 2), nullable=False)
gini_acc_wealth_per_age = Column(DECIMAL(10, 2), nullable=False)
gini_acc_demand_per_age_all = Column(DECIMAL(10, 2), nullable=False)
gini_acc_wealth_per_age_all = Column(DECIMAL(10, 2), nullable=False)
# lst_n_positive_firms = Column(Text(4294000000), nullable=False)
# lst_n_all_firms = Column(Text(4294000000), nullable=False)
# lst_gini_acc_demand_per_age = Column(Text(4294000000), nullable=False)
# lst_gini_acc_wealth_per_age = Column(Text(4294000000), nullable=False)
# lst_gini_acc_demand_per_age_all = Column(Text(4294000000), nullable=False)
# lst_gini_acc_wealth_per_age_all = Column(Text(4294000000), nullable=False)
sample = relationship('Sample', back_populates='product', uselist=False)
firm = relationship('Firm', back_populates='product', lazy='dynamic')
def __repr__(self):
return f'<Product id: {self.id}>'
class Firm(Base):
__tablename__ = f"{db_name_prefix}_firm"
id = Column(Integer, primary_key=True, autoincrement=True)
p_id = Column(Integer, ForeignKey('{}.id'.format(f"{db_name_prefix}_product")), nullable=False)
idx_firm = Column(Integer, nullable=False)
int_n_supplier = Column(Integer, nullable=False)
flt_fix_cost = Column(DECIMAL(20, 2), nullable=False)
flt_q_star = Column(DECIMAL(20, 2), nullable=False)
acc_demand_per_age = Column(DECIMAL(20, 2), nullable=False)
acc_wealth_per_age = Column(DECIMAL(20, 2), nullable=False)
std_demand_per_age = Column(DECIMAL(20, 2), nullable=False)
product = relationship('Product', back_populates='firm', uselist=False)
def __repr__(self):
return f'<Firm id: {self.id}>'
if __name__ == '__main__':
Base.metadata.drop_all()
Base.metadata.create_all()

View File

@ -8,6 +8,7 @@ cycler==0.11.0
decorator==5.1.1
dill==0.3.6
docutils==0.19
greenlet==2.0.2
idna==3.4
imagesize==1.4.1
importlib-metadata==6.0.0
@ -18,6 +19,7 @@ MarkupSafe==2.1.2
matplotlib==3.3.4
matplotlib-inline==0.1.6
multiprocess==0.70.14
mysqlclient==2.1.1
networkx==2.5
numpy==1.20.3
numpydoc==1.1.0
@ -26,10 +28,11 @@ pandas==1.4.1
pandas-stubs==1.2.0.39
Pillow==9.4.0
Pygments==2.14.0
pygraphviz @ file:///C:/Users/ASUS/OneDrive/Project/ScrAbm/Dissertation/IIabm/pygraphviz-1.9-cp38-cp38-win_amd64.whl
pygraphviz @ file:///C:/Users/ASUS/Downloads/pygraphviz-1.9-cp38-cp38-win_amd64.whl
pyparsing==3.0.9
python-dateutil==2.8.2
pytz==2022.7.1
PyYAML==6.0
requests==2.28.2
SALib==1.4.7
scipy==1.10.1
@ -42,7 +45,10 @@ sphinxcontrib-htmlhelp==2.0.1
sphinxcontrib-jsmath==1.0.1
sphinxcontrib-qthelp==1.0.3
sphinxcontrib-serializinghtml==1.1.5
SQLAlchemy==2.0.5.post1
traitlets==5.9.0
typing_extensions==4.5.0
urllib3==1.26.14
wincertstore==0.2
yapf @ file:///tmp/build/80754af9/yapf_1615749224965/work
zipp==3.15.0