761 KiB
761 KiB
In [1]:
import pandas as pd
count = pd.read_csv('analysis\\count.csv')In [2]:
count.head()Out [2]:
| s_id | id_firm | id_product | ts | |
|---|---|---|---|---|
| 0 | 2 | 126 | 1.4 | 1 |
| 1 | 2 | 0 | 1.4.4 | 0 |
| 2 | 4 | 0 | 1.4.4 | 0 |
| 3 | 4 | 126 | 1.4 | 1 |
| 4 | 5 | 126 | 1.4 | 1 |
In [3]:
count[count['id_firm'] == 126].shapeOut [3]:
(1955, 4)
In [4]:
# count row of each firm rename ts to count
count.groupby('id_firm').count()['ts'].rename('count').to_frame()Out [4]:
| count | |
|---|---|
| id_firm | |
| 0 | 352 |
| 1 | 31 |
| 2 | 33 |
| 3 | 124 |
| 4 | 34 |
| ... | ... |
| 166 | 31 |
| 167 | 21 |
| 168 | 175 |
| 169 | 22 |
| 170 | 1525 |
171 rows × 1 columns
In [5]:
firm = pd.read_csv('Firm.csv')
firm.head()Out [5]:
| Code | Name | Stock_Region | Stock_Name | Stock_Code | Chinese_Name | Report_Year | Assets | Revenue | Size | ... | 2.1.4.1.3 | 2.1.4.1.4 | 2.1.4.2 | 2.1.4.2.1 | 2.1.4.2.2 | 2.2 | 2.3 | 2.3.1 | 2.3.2 | 2.3.3 | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | 0 | 360科技 | SH | 三六零 | 601360.SH | 三六零安全科技股份有限公司 | 2021.0 | 4.204000e+10 | 1.089000e+10 | L | ... | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN |
| 1 | 1 | 51WORLD | NaN | 51WORLD | NaN | 北京五一视界数字孪生科技股份有限公司 | 2021.0 | 5.240000e+08 | 1.380000e+08 | M | ... | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN |
| 2 | 2 | 706所 | NaN | 706所 | NaN | 北京航天爱威电子技术有限公司 | NaN | NaN | NaN | M | ... | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN |
| 3 | 3 | 艾克斯特 | NaN | 艾克斯特 | NaN | 北京艾克斯特科技有限公司 | NaN | NaN | NaN | S | ... | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN |
| 4 | 4 | 爱创科技 | NaN | 爱创科技 | NaN | 北京爱创科技股份有限公司 | NaN | NaN | NaN | M | ... | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN |
5 rows × 120 columns
In [6]:
count.merge(firm[['Code', 'Assets', 'Revenue', 'Size']], how='left', left_on='id_firm', right_on='Code')Out [6]:
| s_id | id_firm | id_product | ts | Code | Assets | Revenue | Size | |
|---|---|---|---|---|---|---|---|---|
| 0 | 2 | 126 | 1.4 | 1 | 126 | NaN | 6.368070e+11 | L |
| 1 | 2 | 0 | 1.4.4 | 0 | 0 | 4.204000e+10 | 1.089000e+10 | L |
| 2 | 4 | 0 | 1.4.4 | 0 | 0 | 4.204000e+10 | 1.089000e+10 | L |
| 3 | 4 | 126 | 1.4 | 1 | 126 | NaN | 6.368070e+11 | L |
| 4 | 5 | 126 | 1.4 | 1 | 126 | NaN | 6.368070e+11 | L |
| ... | ... | ... | ... | ... | ... | ... | ... | ... |
| 31916 | 23694 | 86 | 1.1 | 1 | 86 | 8.490952e+11 | 5.969638e+11 | NaN |
| 31917 | 23695 | 169 | 1.1.1 | 0 | 169 | 2.302330e+11 | 3.470400e+10 | L |
| 31918 | 23695 | 105 | 1.1 | 1 | 105 | 1.158633e+12 | 5.436851e+11 | NaN |
| 31919 | 23699 | 169 | 1.1.1 | 0 | 169 | 2.302330e+11 | 3.470400e+10 | L |
| 31920 | 23699 | 105 | 1.1 | 1 | 105 | 1.158633e+12 | 5.436851e+11 | NaN |
31921 rows × 8 columns
In [7]:
from orm import db_session, Sample, engine
import pandas as pd
# Read your count data
count = pd.read_csv('analysis\\count.csv')
# Query to get s_id and g_firm (firm network)
query = db_session.query(
Sample.id.label('s_id'),
Sample.e_id,
Sample.g_firm
)
# Convert to dataframe using engine.connect()
with engine.connect() as conn:
sample_df = pd.read_sql(query.statement, conn)
# Now merge with your count data
count_with_gfirm = count.merge(
sample_df[['s_id', 'e_id', 'g_firm']],
how='left',
on='s_id'
)
count_with_gfirm.head()DB is localhost:3306/iiabmdb
[1;31m---------------------------------------------------------------------------[0m [1;31mKeyboardInterrupt[0m Traceback (most recent call last) Cell [1;32mIn[7], line 16[0m [0;32m 14[0m [38;5;66;03m# Convert to dataframe using engine.connect()[39;00m [0;32m 15[0m [38;5;28;01mwith[39;00m engine[38;5;241m.[39mconnect() [38;5;28;01mas[39;00m conn: [1;32m---> 16[0m sample_df [38;5;241m=[39m [43mpd[49m[38;5;241;43m.[39;49m[43mread_sql[49m[43m([49m[43mquery[49m[38;5;241;43m.[39;49m[43mstatement[49m[43m,[49m[43m [49m[43mconn[49m[43m)[49m [0;32m 18[0m [38;5;66;03m# Now merge with your count data[39;00m [0;32m 19[0m count_with_gfirm [38;5;241m=[39m count[38;5;241m.[39mmerge( [0;32m 20[0m sample_df[[[38;5;124m'[39m[38;5;124ms_id[39m[38;5;124m'[39m, [38;5;124m'[39m[38;5;124me_id[39m[38;5;124m'[39m, [38;5;124m'[39m[38;5;124mg_firm[39m[38;5;124m'[39m]], [0;32m 21[0m how[38;5;241m=[39m[38;5;124m'[39m[38;5;124mleft[39m[38;5;124m'[39m, [0;32m 22[0m on[38;5;241m=[39m[38;5;124m'[39m[38;5;124ms_id[39m[38;5;124m'[39m [0;32m 23[0m ) File [1;32mc:\Users\ASUS\Documents\Project\IIabm\venv\lib\site-packages\pandas\io\sql.py:592[0m, in [0;36mread_sql[1;34m(sql, con, index_col, coerce_float, params, parse_dates, columns, chunksize)[0m [0;32m 583[0m [38;5;28;01mreturn[39;00m pandas_sql[38;5;241m.[39mread_table( [0;32m 584[0m sql, [0;32m 585[0m index_col[38;5;241m=[39mindex_col, [1;32m (...)[0m [0;32m 589[0m chunksize[38;5;241m=[39mchunksize, [0;32m 590[0m ) [0;32m 591[0m [38;5;28;01melse[39;00m: [1;32m--> 592[0m [38;5;28;01mreturn[39;00m [43mpandas_sql[49m[38;5;241;43m.[39;49m[43mread_query[49m[43m([49m [0;32m 593[0m [43m [49m[43msql[49m[43m,[49m [0;32m 594[0m [43m [49m[43mindex_col[49m[38;5;241;43m=[39;49m[43mindex_col[49m[43m,[49m [0;32m 595[0m [43m [49m[43mparams[49m[38;5;241;43m=[39;49m[43mparams[49m[43m,[49m [0;32m 596[0m [43m [49m[43mcoerce_float[49m[38;5;241;43m=[39;49m[43mcoerce_float[49m[43m,[49m [0;32m 597[0m [43m [49m[43mparse_dates[49m[38;5;241;43m=[39;49m[43mparse_dates[49m[43m,[49m [0;32m 598[0m [43m [49m[43mchunksize[49m[38;5;241;43m=[39;49m[43mchunksize[49m[43m,[49m [0;32m 599[0m [43m [49m[43m)[49m File [1;32mc:\Users\ASUS\Documents\Project\IIabm\venv\lib\site-packages\pandas\io\sql.py:1557[0m, in [0;36mSQLDatabase.read_query[1;34m(self, sql, index_col, coerce_float, parse_dates, params, chunksize, dtype)[0m [0;32m 1509[0m [38;5;250m[39m[38;5;124;03m"""[39;00m [0;32m 1510[0m [38;5;124;03mRead SQL query into a DataFrame.[39;00m [0;32m 1511[0m [1;32m (...)[0m [0;32m 1553[0m [0;32m 1554[0m [38;5;124;03m"""[39;00m [0;32m 1555[0m args [38;5;241m=[39m _convert_params(sql, params) [1;32m-> 1557[0m result [38;5;241m=[39m [38;5;28;43mself[39;49m[38;5;241;43m.[39;49m[43mexecute[49m[43m([49m[38;5;241;43m*[39;49m[43margs[49m[43m)[49m [0;32m 1558[0m columns [38;5;241m=[39m result[38;5;241m.[39mkeys() [0;32m 1560[0m [38;5;28;01mif[39;00m chunksize [38;5;129;01mis[39;00m [38;5;129;01mnot[39;00m [38;5;28;01mNone[39;00m: File [1;32mc:\Users\ASUS\Documents\Project\IIabm\venv\lib\site-packages\pandas\io\sql.py:1402[0m, in [0;36mSQLDatabase.execute[1;34m(self, *args, **kwargs)[0m [0;32m 1400[0m [38;5;28;01mdef[39;00m[38;5;250m [39m[38;5;21mexecute[39m([38;5;28mself[39m, [38;5;241m*[39margs, [38;5;241m*[39m[38;5;241m*[39mkwargs): [0;32m 1401[0m [38;5;250m [39m[38;5;124;03m"""Simple passthrough to SQLAlchemy connectable"""[39;00m [1;32m-> 1402[0m [38;5;28;01mreturn[39;00m [38;5;28;43mself[39;49m[38;5;241;43m.[39;49m[43mconnectable[49m[38;5;241;43m.[39;49m[43mexecution_options[49m[43m([49m[43m)[49m[38;5;241;43m.[39;49m[43mexecute[49m[43m([49m[38;5;241;43m*[39;49m[43margs[49m[43m,[49m[43m [49m[38;5;241;43m*[39;49m[38;5;241;43m*[39;49m[43mkwargs[49m[43m)[49m File [1;32mc:\Users\ASUS\Documents\Project\IIabm\venv\lib\site-packages\sqlalchemy\engine\base.py:1419[0m, in [0;36mConnection.execute[1;34m(self, statement, parameters, execution_options)[0m [0;32m 1417[0m [38;5;28;01mraise[39;00m exc[38;5;241m.[39mObjectNotExecutableError(statement) [38;5;28;01mfrom[39;00m[38;5;250m [39m[38;5;21;01merr[39;00m [0;32m 1418[0m [38;5;28;01melse[39;00m: [1;32m-> 1419[0m [38;5;28;01mreturn[39;00m [43mmeth[49m[43m([49m [0;32m 1420[0m [43m [49m[38;5;28;43mself[39;49m[43m,[49m [0;32m 1421[0m [43m [49m[43mdistilled_parameters[49m[43m,[49m [0;32m 1422[0m [43m [49m[43mexecution_options[49m[43m [49m[38;5;129;43;01mor[39;49;00m[43m [49m[43mNO_OPTIONS[49m[43m,[49m [0;32m 1423[0m [43m [49m[43m)[49m File [1;32mc:\Users\ASUS\Documents\Project\IIabm\venv\lib\site-packages\sqlalchemy\sql\elements.py:527[0m, in [0;36mClauseElement._execute_on_connection[1;34m(self, connection, distilled_params, execution_options)[0m [0;32m 525[0m [38;5;28;01mif[39;00m TYPE_CHECKING: [0;32m 526[0m [38;5;28;01massert[39;00m [38;5;28misinstance[39m([38;5;28mself[39m, Executable) [1;32m--> 527[0m [38;5;28;01mreturn[39;00m [43mconnection[49m[38;5;241;43m.[39;49m[43m_execute_clauseelement[49m[43m([49m [0;32m 528[0m [43m [49m[38;5;28;43mself[39;49m[43m,[49m[43m [49m[43mdistilled_params[49m[43m,[49m[43m [49m[43mexecution_options[49m [0;32m 529[0m [43m [49m[43m)[49m [0;32m 530[0m [38;5;28;01melse[39;00m: [0;32m 531[0m [38;5;28;01mraise[39;00m exc[38;5;241m.[39mObjectNotExecutableError([38;5;28mself[39m) File [1;32mc:\Users\ASUS\Documents\Project\IIabm\venv\lib\site-packages\sqlalchemy\engine\base.py:1641[0m, in [0;36mConnection._execute_clauseelement[1;34m(self, elem, distilled_parameters, execution_options)[0m [0;32m 1629[0m compiled_cache: Optional[CompiledCacheType] [38;5;241m=[39m execution_options[38;5;241m.[39mget( [0;32m 1630[0m [38;5;124m"[39m[38;5;124mcompiled_cache[39m[38;5;124m"[39m, [38;5;28mself[39m[38;5;241m.[39mengine[38;5;241m.[39m_compiled_cache [0;32m 1631[0m ) [0;32m 1633[0m compiled_sql, extracted_params, cache_hit [38;5;241m=[39m elem[38;5;241m.[39m_compile_w_cache( [0;32m 1634[0m dialect[38;5;241m=[39mdialect, [0;32m 1635[0m compiled_cache[38;5;241m=[39mcompiled_cache, [1;32m (...)[0m [0;32m 1639[0m linting[38;5;241m=[39m[38;5;28mself[39m[38;5;241m.[39mdialect[38;5;241m.[39mcompiler_linting [38;5;241m|[39m compiler[38;5;241m.[39mWARN_LINTING, [0;32m 1640[0m ) [1;32m-> 1641[0m ret [38;5;241m=[39m [38;5;28;43mself[39;49m[38;5;241;43m.[39;49m[43m_execute_context[49m[43m([49m [0;32m 1642[0m [43m [49m[43mdialect[49m[43m,[49m [0;32m 1643[0m [43m [49m[43mdialect[49m[38;5;241;43m.[39;49m[43mexecution_ctx_cls[49m[38;5;241;43m.[39;49m[43m_init_compiled[49m[43m,[49m [0;32m 1644[0m [43m [49m[43mcompiled_sql[49m[43m,[49m [0;32m 1645[0m [43m [49m[43mdistilled_parameters[49m[43m,[49m [0;32m 1646[0m [43m [49m[43mexecution_options[49m[43m,[49m [0;32m 1647[0m [43m [49m[43mcompiled_sql[49m[43m,[49m [0;32m 1648[0m [43m [49m[43mdistilled_parameters[49m[43m,[49m [0;32m 1649[0m [43m [49m[43melem[49m[43m,[49m [0;32m 1650[0m [43m [49m[43mextracted_params[49m[43m,[49m [0;32m 1651[0m [43m [49m[43mcache_hit[49m[38;5;241;43m=[39;49m[43mcache_hit[49m[43m,[49m [0;32m 1652[0m [43m[49m[43m)[49m [0;32m 1653[0m [38;5;28;01mif[39;00m has_events: [0;32m 1654[0m [38;5;28mself[39m[38;5;241m.[39mdispatch[38;5;241m.[39mafter_execute( [0;32m 1655[0m [38;5;28mself[39m, [0;32m 1656[0m elem, [1;32m (...)[0m [0;32m 1660[0m ret, [0;32m 1661[0m ) File [1;32mc:\Users\ASUS\Documents\Project\IIabm\venv\lib\site-packages\sqlalchemy\engine\base.py:1846[0m, in [0;36mConnection._execute_context[1;34m(self, dialect, constructor, statement, parameters, execution_options, *args, **kw)[0m [0;32m 1844[0m [38;5;28;01mreturn[39;00m [38;5;28mself[39m[38;5;241m.[39m_exec_insertmany_context(dialect, context) [0;32m 1845[0m [38;5;28;01melse[39;00m: [1;32m-> 1846[0m [38;5;28;01mreturn[39;00m [38;5;28;43mself[39;49m[38;5;241;43m.[39;49m[43m_exec_single_context[49m[43m([49m [0;32m 1847[0m [43m [49m[43mdialect[49m[43m,[49m[43m [49m[43mcontext[49m[43m,[49m[43m [49m[43mstatement[49m[43m,[49m[43m [49m[43mparameters[49m [0;32m 1848[0m [43m [49m[43m)[49m File [1;32mc:\Users\ASUS\Documents\Project\IIabm\venv\lib\site-packages\sqlalchemy\engine\base.py:1986[0m, in [0;36mConnection._exec_single_context[1;34m(self, dialect, context, statement, parameters)[0m [0;32m 1983[0m result [38;5;241m=[39m context[38;5;241m.[39m_setup_result_proxy() [0;32m 1985[0m [38;5;28;01mexcept[39;00m [38;5;167;01mBaseException[39;00m [38;5;28;01mas[39;00m e: [1;32m-> 1986[0m [38;5;28;43mself[39;49m[38;5;241;43m.[39;49m[43m_handle_dbapi_exception[49m[43m([49m [0;32m 1987[0m [43m [49m[43me[49m[43m,[49m[43m [49m[43mstr_statement[49m[43m,[49m[43m [49m[43meffective_parameters[49m[43m,[49m[43m [49m[43mcursor[49m[43m,[49m[43m [49m[43mcontext[49m [0;32m 1988[0m [43m [49m[43m)[49m [0;32m 1990[0m [38;5;28;01mreturn[39;00m result File [1;32mc:\Users\ASUS\Documents\Project\IIabm\venv\lib\site-packages\sqlalchemy\engine\base.py:2366[0m, in [0;36mConnection._handle_dbapi_exception[1;34m(self, e, statement, parameters, cursor, context, is_sub_exec)[0m [0;32m 2364[0m [38;5;28;01melse[39;00m: [0;32m 2365[0m [38;5;28;01massert[39;00m exc_info[[38;5;241m1[39m] [38;5;129;01mis[39;00m [38;5;129;01mnot[39;00m [38;5;28;01mNone[39;00m [1;32m-> 2366[0m [38;5;28;01mraise[39;00m exc_info[[38;5;241m1[39m][38;5;241m.[39mwith_traceback(exc_info[[38;5;241m2[39m]) [0;32m 2367[0m [38;5;28;01mfinally[39;00m: [0;32m 2368[0m [38;5;28;01mdel[39;00m [38;5;28mself[39m[38;5;241m.[39m_reentrant_error File [1;32mc:\Users\ASUS\Documents\Project\IIabm\venv\lib\site-packages\sqlalchemy\engine\base.py:1967[0m, in [0;36mConnection._exec_single_context[1;34m(self, dialect, context, statement, parameters)[0m [0;32m 1965[0m [38;5;28;01mbreak[39;00m [0;32m 1966[0m [38;5;28;01mif[39;00m [38;5;129;01mnot[39;00m evt_handled: [1;32m-> 1967[0m [38;5;28;43mself[39;49m[38;5;241;43m.[39;49m[43mdialect[49m[38;5;241;43m.[39;49m[43mdo_execute[49m[43m([49m [0;32m 1968[0m [43m [49m[43mcursor[49m[43m,[49m[43m [49m[43mstr_statement[49m[43m,[49m[43m [49m[43meffective_parameters[49m[43m,[49m[43m [49m[43mcontext[49m [0;32m 1969[0m [43m [49m[43m)[49m [0;32m 1971[0m [38;5;28;01mif[39;00m [38;5;28mself[39m[38;5;241m.[39m_has_events [38;5;129;01mor[39;00m [38;5;28mself[39m[38;5;241m.[39mengine[38;5;241m.[39m_has_events: [0;32m 1972[0m [38;5;28mself[39m[38;5;241m.[39mdispatch[38;5;241m.[39mafter_cursor_execute( [0;32m 1973[0m [38;5;28mself[39m, [0;32m 1974[0m cursor, [1;32m (...)[0m [0;32m 1978[0m context[38;5;241m.[39mexecutemany, [0;32m 1979[0m ) File [1;32mc:\Users\ASUS\Documents\Project\IIabm\venv\lib\site-packages\sqlalchemy\engine\default.py:952[0m, in [0;36mDefaultDialect.do_execute[1;34m(self, cursor, statement, parameters, context)[0m [0;32m 951[0m [38;5;28;01mdef[39;00m[38;5;250m [39m[38;5;21mdo_execute[39m([38;5;28mself[39m, cursor, statement, parameters, context[38;5;241m=[39m[38;5;28;01mNone[39;00m): [1;32m--> 952[0m [43mcursor[49m[38;5;241;43m.[39;49m[43mexecute[49m[43m([49m[43mstatement[49m[43m,[49m[43m [49m[43mparameters[49m[43m)[49m File [1;32mc:\Users\ASUS\Documents\Project\IIabm\venv\lib\site-packages\pymysql\cursors.py:153[0m, in [0;36mCursor.execute[1;34m(self, query, args)[0m [0;32m 149[0m [38;5;28;01mpass[39;00m [0;32m 151[0m query [38;5;241m=[39m [38;5;28mself[39m[38;5;241m.[39mmogrify(query, args) [1;32m--> 153[0m result [38;5;241m=[39m [38;5;28;43mself[39;49m[38;5;241;43m.[39;49m[43m_query[49m[43m([49m[43mquery[49m[43m)[49m [0;32m 154[0m [38;5;28mself[39m[38;5;241m.[39m_executed [38;5;241m=[39m query [0;32m 155[0m [38;5;28;01mreturn[39;00m result File [1;32mc:\Users\ASUS\Documents\Project\IIabm\venv\lib\site-packages\pymysql\cursors.py:322[0m, in [0;36mCursor._query[1;34m(self, q)[0m [0;32m 320[0m conn [38;5;241m=[39m [38;5;28mself[39m[38;5;241m.[39m_get_db() [0;32m 321[0m [38;5;28mself[39m[38;5;241m.[39m_clear_result() [1;32m--> 322[0m [43mconn[49m[38;5;241;43m.[39;49m[43mquery[49m[43m([49m[43mq[49m[43m)[49m [0;32m 323[0m [38;5;28mself[39m[38;5;241m.[39m_do_get_result() [0;32m 324[0m [38;5;28;01mreturn[39;00m [38;5;28mself[39m[38;5;241m.[39mrowcount File [1;32mc:\Users\ASUS\Documents\Project\IIabm\venv\lib\site-packages\pymysql\connections.py:575[0m, in [0;36mConnection.query[1;34m(self, sql, unbuffered)[0m [0;32m 573[0m sql [38;5;241m=[39m sql[38;5;241m.[39mencode([38;5;28mself[39m[38;5;241m.[39mencoding, [38;5;124m"[39m[38;5;124msurrogateescape[39m[38;5;124m"[39m) [0;32m 574[0m [38;5;28mself[39m[38;5;241m.[39m_execute_command(COMMAND[38;5;241m.[39mCOM_QUERY, sql) [1;32m--> 575[0m [38;5;28mself[39m[38;5;241m.[39m_affected_rows [38;5;241m=[39m [38;5;28;43mself[39;49m[38;5;241;43m.[39;49m[43m_read_query_result[49m[43m([49m[43munbuffered[49m[38;5;241;43m=[39;49m[43munbuffered[49m[43m)[49m [0;32m 576[0m [38;5;28;01mreturn[39;00m [38;5;28mself[39m[38;5;241m.[39m_affected_rows File [1;32mc:\Users\ASUS\Documents\Project\IIabm\venv\lib\site-packages\pymysql\connections.py:826[0m, in [0;36mConnection._read_query_result[1;34m(self, unbuffered)[0m [0;32m 824[0m result[38;5;241m.[39minit_unbuffered_query() [0;32m 825[0m [38;5;28;01melse[39;00m: [1;32m--> 826[0m [43mresult[49m[38;5;241;43m.[39;49m[43mread[49m[43m([49m[43m)[49m [0;32m 827[0m [38;5;28mself[39m[38;5;241m.[39m_result [38;5;241m=[39m result [0;32m 828[0m [38;5;28;01mif[39;00m result[38;5;241m.[39mserver_status [38;5;129;01mis[39;00m [38;5;129;01mnot[39;00m [38;5;28;01mNone[39;00m: File [1;32mc:\Users\ASUS\Documents\Project\IIabm\venv\lib\site-packages\pymysql\connections.py:1210[0m, in [0;36mMySQLResult.read[1;34m(self)[0m [0;32m 1208[0m [38;5;28mself[39m[38;5;241m.[39m_read_load_local_packet(first_packet) [0;32m 1209[0m [38;5;28;01melse[39;00m: [1;32m-> 1210[0m [38;5;28;43mself[39;49m[38;5;241;43m.[39;49m[43m_read_result_packet[49m[43m([49m[43mfirst_packet[49m[43m)[49m [0;32m 1211[0m [38;5;28;01mfinally[39;00m: [0;32m 1212[0m [38;5;28mself[39m[38;5;241m.[39mconnection [38;5;241m=[39m [38;5;28;01mNone[39;00m File [1;32mc:\Users\ASUS\Documents\Project\IIabm\venv\lib\site-packages\pymysql\connections.py:1287[0m, in [0;36mMySQLResult._read_result_packet[1;34m(self, first_packet)[0m [0;32m 1285[0m [38;5;28mself[39m[38;5;241m.[39mfield_count [38;5;241m=[39m first_packet[38;5;241m.[39mread_length_encoded_integer() [0;32m 1286[0m [38;5;28mself[39m[38;5;241m.[39m_get_descriptions() [1;32m-> 1287[0m [38;5;28;43mself[39;49m[38;5;241;43m.[39;49m[43m_read_rowdata_packet[49m[43m([49m[43m)[49m File [1;32mc:\Users\ASUS\Documents\Project\IIabm\venv\lib\site-packages\pymysql\connections.py:1334[0m, in [0;36mMySQLResult._read_rowdata_packet[1;34m(self)[0m [0;32m 1332[0m rows [38;5;241m=[39m [] [0;32m 1333[0m [38;5;28;01mwhile[39;00m [38;5;28;01mTrue[39;00m: [1;32m-> 1334[0m packet [38;5;241m=[39m [38;5;28;43mself[39;49m[38;5;241;43m.[39;49m[43mconnection[49m[38;5;241;43m.[39;49m[43m_read_packet[49m[43m([49m[43m)[49m [0;32m 1335[0m [38;5;28;01mif[39;00m [38;5;28mself[39m[38;5;241m.[39m_check_packet_is_eof(packet): [0;32m 1336[0m [38;5;28mself[39m[38;5;241m.[39mconnection [38;5;241m=[39m [38;5;28;01mNone[39;00m [38;5;66;03m# release reference to kill cyclic reference.[39;00m File [1;32mc:\Users\ASUS\Documents\Project\IIabm\venv\lib\site-packages\pymysql\connections.py:751[0m, in [0;36mConnection._read_packet[1;34m(self, packet_type)[0m [0;32m 749[0m buff [38;5;241m=[39m [38;5;28mbytearray[39m() [0;32m 750[0m [38;5;28;01mwhile[39;00m [38;5;28;01mTrue[39;00m: [1;32m--> 751[0m packet_header [38;5;241m=[39m [38;5;28;43mself[39;49m[38;5;241;43m.[39;49m[43m_read_bytes[49m[43m([49m[38;5;241;43m4[39;49m[43m)[49m [0;32m 752[0m [38;5;66;03m# if DEBUG: dump_packet(packet_header)[39;00m [0;32m 754[0m btrl, btrh, packet_number [38;5;241m=[39m struct[38;5;241m.[39munpack([38;5;124m"[39m[38;5;124m<HBB[39m[38;5;124m"[39m, packet_header) File [1;32mc:\Users\ASUS\Documents\Project\IIabm\venv\lib\site-packages\pymysql\connections.py:789[0m, in [0;36mConnection._read_bytes[1;34m(self, num_bytes)[0m [0;32m 787[0m [38;5;28;01mwhile[39;00m [38;5;28;01mTrue[39;00m: [0;32m 788[0m [38;5;28;01mtry[39;00m: [1;32m--> 789[0m data [38;5;241m=[39m [38;5;28;43mself[39;49m[38;5;241;43m.[39;49m[43m_rfile[49m[38;5;241;43m.[39;49m[43mread[49m[43m([49m[43mnum_bytes[49m[43m)[49m [0;32m 790[0m [38;5;28;01mbreak[39;00m [0;32m 791[0m [38;5;28;01mexcept[39;00m [38;5;167;01mOSError[39;00m [38;5;28;01mas[39;00m e: File [1;32m~\AppData\Local\Programs\Python\Python38\lib\socket.py:669[0m, in [0;36mSocketIO.readinto[1;34m(self, b)[0m [0;32m 667[0m [38;5;28;01mwhile[39;00m [38;5;28;01mTrue[39;00m: [0;32m 668[0m [38;5;28;01mtry[39;00m: [1;32m--> 669[0m [38;5;28;01mreturn[39;00m [38;5;28;43mself[39;49m[38;5;241;43m.[39;49m[43m_sock[49m[38;5;241;43m.[39;49m[43mrecv_into[49m[43m([49m[43mb[49m[43m)[49m [0;32m 670[0m [38;5;28;01mexcept[39;00m timeout: [0;32m 671[0m [38;5;28mself[39m[38;5;241m.[39m_timeout_occurred [38;5;241m=[39m [38;5;28;01mTrue[39;00m [1;31mKeyboardInterrupt[0m:
In [8]:
import json
import networkx as nx
# Function to convert string back to networkx graph
def string_to_graph(g_string):
"""Convert JSON string back to networkx graph"""
if pd.isna(g_string):
return None
g_data = json.loads(g_string)
return nx.adjacency_graph(g_data)
# Option 3: Comprehensive firm metrics for MultiDiGraph
def get_comprehensive_firm_metrics(g_string, node_id):
"""Get both direct MultiDiGraph metrics and weighted graph metrics"""
if pd.isna(g_string):
return {col: None for col in [
'num_supplier_firms', 'num_customer_firms',
'num_products_supplied', 'num_products_received',
'weighted_betweenness', 'weighted_closeness', 'pagerank'
]}
G_multi = string_to_graph(g_string)
if node_id not in G_multi.nodes():
return {col: None for col in [
'num_supplier_firms', 'num_customer_firms',
'num_products_supplied', 'num_products_received',
'weighted_betweenness', 'weighted_closeness', 'pagerank'
]}
# Direct MultiDiGraph metrics
num_suppliers = G_multi.in_degree(node_id)
num_customers = G_multi.out_degree(node_id)
num_products_in = len(list(G_multi.in_edges(node_id)))
num_products_out = len(list(G_multi.out_edges(node_id)))
# Convert to weighted DiGraph for advanced metrics
G_weighted = nx.DiGraph()
for u, v in G_multi.edges():
if G_weighted.has_edge(u, v):
G_weighted[u][v]['weight'] += 1
else:
G_weighted.add_edge(u, v, weight=1)
G_weighted.add_nodes_from(G_multi.nodes())
# Calculate centrality on weighted graph
betweenness = nx.betweenness_centrality(G_weighted, weight='weight')
closeness = nx.closeness_centrality(G_weighted, distance='weight')
pagerank = nx.pagerank(G_weighted, weight='weight')
return {
'num_supplier_firms': num_suppliers,
'num_customer_firms': num_customers,
'num_products_supplied': num_products_out,
'num_products_received': num_products_in,
'weighted_betweenness': betweenness.get(node_id),
'weighted_closeness': closeness.get(node_id),
'pagerank': pagerank.get(node_id)
}
# Convert id_firm to string to match graph node IDs
count_with_gfirm['id_firm_str'] = count_with_gfirm['id_firm'].astype(str)
# Calculate metrics for each row
print("Calculating firm-level centrality metrics...")
firm_metrics = count_with_gfirm.apply(
lambda row: pd.Series(get_comprehensive_firm_metrics(row['g_firm'], row['id_firm_str'])),
axis=1
)
firm_metrics.columns = ['firm_' + col for col in firm_metrics.columns]
# Add the metrics as new columns
count_with_gfirm = pd.concat([count_with_gfirm, firm_metrics], axis=1)
print("\nColumns in final dataframe:")
print(count_with_gfirm.columns.tolist())
count_with_gfirm.head()[1;31m---------------------------------------------------------------------------[0m
[1;31mNameError[0m Traceback (most recent call last)
Cell [1;32mIn[8], line 62[0m
[0;32m 51[0m [38;5;28;01mreturn[39;00m {
[0;32m 52[0m [38;5;124m'[39m[38;5;124mnum_supplier_firms[39m[38;5;124m'[39m: num_suppliers,
[0;32m 53[0m [38;5;124m'[39m[38;5;124mnum_customer_firms[39m[38;5;124m'[39m: num_customers,
[1;32m (...)[0m
[0;32m 58[0m [38;5;124m'[39m[38;5;124mpagerank[39m[38;5;124m'[39m: pagerank[38;5;241m.[39mget(node_id)
[0;32m 59[0m }
[0;32m 61[0m [38;5;66;03m# Convert id_firm to string to match graph node IDs[39;00m
[1;32m---> 62[0m count_with_gfirm[[38;5;124m'[39m[38;5;124mid_firm_str[39m[38;5;124m'[39m] [38;5;241m=[39m [43mcount_with_gfirm[49m[[38;5;124m'[39m[38;5;124mid_firm[39m[38;5;124m'[39m][38;5;241m.[39mastype([38;5;28mstr[39m)
[0;32m 64[0m [38;5;66;03m# Calculate metrics for each row[39;00m
[0;32m 65[0m [38;5;28mprint[39m([38;5;124m"[39m[38;5;124mCalculating firm-level centrality metrics...[39m[38;5;124m"[39m)
[1;31mNameError[0m: name 'count_with_gfirm' is not definedIn [9]:
firm = pd.read_csv('Firm.csv')
count_with_gfirm = count_with_gfirm.merge(firm[['Code', 'Assets', 'Revenue', 'Size']], how='left', left_on='id_firm', right_on='Code').drop(columns=['Code'])
count_with_gfirm.to_csv('count_with_gfirm.csv', index=False)
count_with_gfirm.head()[1;31m---------------------------------------------------------------------------[0m [1;31mNameError[0m Traceback (most recent call last) Cell [1;32mIn[9], line 2[0m [0;32m 1[0m firm [38;5;241m=[39m pd[38;5;241m.[39mread_csv([38;5;124m'[39m[38;5;124mFirm.csv[39m[38;5;124m'[39m) [1;32m----> 2[0m count_with_gfirm [38;5;241m=[39m [43mcount_with_gfirm[49m[38;5;241m.[39mmerge(firm[[[38;5;124m'[39m[38;5;124mCode[39m[38;5;124m'[39m, [38;5;124m'[39m[38;5;124mAssets[39m[38;5;124m'[39m, [38;5;124m'[39m[38;5;124mRevenue[39m[38;5;124m'[39m, [38;5;124m'[39m[38;5;124mSize[39m[38;5;124m'[39m]], how[38;5;241m=[39m[38;5;124m'[39m[38;5;124mleft[39m[38;5;124m'[39m, left_on[38;5;241m=[39m[38;5;124m'[39m[38;5;124mid_firm[39m[38;5;124m'[39m, right_on[38;5;241m=[39m[38;5;124m'[39m[38;5;124mCode[39m[38;5;124m'[39m)[38;5;241m.[39mdrop(columns[38;5;241m=[39m[[38;5;124m'[39m[38;5;124mCode[39m[38;5;124m'[39m]) [0;32m 3[0m count_with_gfirm[38;5;241m.[39mto_csv([38;5;124m'[39m[38;5;124mcount_with_gfirm.csv[39m[38;5;124m'[39m, index[38;5;241m=[39m[38;5;28;01mFalse[39;00m) [0;32m 4[0m count_with_gfirm[38;5;241m.[39mhead() [1;31mNameError[0m: name 'count_with_gfirm' is not defined
In [11]:
# Group by firm and aggregate
import pandas as pd
count_with_gfirm = pd.read_csv('count_with_gfirm.csv')
firm_summary = count_with_gfirm.groupby('id_firm').agg({
# Count number of disruptions
'ts': 'count', # Will rename to 'count'
# Average of network metrics
'firm_num_supplier_firms': 'mean',
'firm_num_customer_firms': 'mean',
'firm_num_products_supplied': 'mean',
'firm_num_products_received': 'mean',
'firm_weighted_betweenness': 'mean',
'firm_weighted_closeness': 'mean',
'firm_pagerank': 'mean',
# First element of firm attributes (should be same within group)
'Assets': 'first',
'Revenue': 'first',
'Size': 'first'
}).rename(columns={'ts': 'count'})
# Reset index to make id_firm a column
firm_summary = firm_summary.reset_index()
print(f"Total number of firms: {len(firm_summary)}")
print(f"\nColumns: {firm_summary.columns.tolist()}")
print(f"\nSummary statistics:")
print(firm_summary.describe())
firm_summary.head(10)Out [11]:
Total number of firms: 171
Columns: ['id_firm', 'count', 'firm_num_supplier_firms', 'firm_num_customer_firms', 'firm_num_products_supplied', 'firm_num_products_received', 'firm_weighted_betweenness', 'firm_weighted_closeness', 'firm_pagerank', 'Assets', 'Revenue', 'Size']
Summary statistics:
id_firm count firm_num_supplier_firms \
count 171.000000 171.000000 171.000000
mean 85.000000 186.672515 3.190330
std 49.507575 303.402661 7.900570
min 0.000000 17.000000 0.000000
25% 42.500000 33.000000 0.000000
50% 85.000000 51.000000 0.000000
75% 127.500000 189.000000 0.000000
max 170.000000 1955.000000 47.204703
firm_num_customer_firms firm_num_products_supplied \
count 171.000000 171.000000
mean 3.398434 3.398434
std 5.946080 5.946080
min 0.000000 0.000000
25% 1.159220 1.159220
50% 1.794118 1.794118
75% 2.803584 2.803584
max 51.010654 51.010654
firm_num_products_received firm_weighted_betweenness \
count 171.000000 171.000000
mean 3.190330 0.000365
std 7.900570 0.001466
min 0.000000 0.000000
25% 0.000000 0.000000
50% 0.000000 0.000000
75% 0.000000 0.000000
max 47.204703 0.013216
firm_weighted_closeness firm_pagerank Assets Revenue
count 171.000000 171.000000 1.020000e+02 1.110000e+02
mean 0.020191 0.005886 2.773894e+11 1.578147e+11
std 0.054180 0.014406 5.238742e+11 3.774420e+11
min 0.000000 0.001817 1.560000e+08 4.545320e+07
25% 0.000000 0.001925 3.043750e+09 1.318000e+09
50% 0.000000 0.002247 1.808500e+10 6.894667e+09
75% 0.000000 0.002283 3.432049e+11 1.641537e+11
max 0.287842 0.141979 2.893377e+12 3.232375e+12
| id_firm | count | firm_num_supplier_firms | firm_num_customer_firms | firm_num_products_supplied | firm_num_products_received | firm_weighted_betweenness | firm_weighted_closeness | firm_pagerank | Assets | Revenue | Size | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | 0 | 352 | 7.863636 | 0.823864 | 0.823864 | 7.863636 | 0.00102 | 0.046257 | 0.008342 | 4.204000e+10 | 1.089000e+10 | L |
| 1 | 1 | 31 | 0.000000 | 1.935484 | 1.935484 | 0.000000 | 0.00000 | 0.000000 | 0.001823 | 5.240000e+08 | 1.380000e+08 | M |
| 2 | 2 | 33 | 0.000000 | 1.909091 | 1.909091 | 0.000000 | 0.00000 | 0.000000 | 0.001825 | NaN | NaN | M |
| 3 | 3 | 124 | 0.000000 | 9.169355 | 9.169355 | 0.000000 | 0.00000 | 0.000000 | 0.001821 | NaN | NaN | S |
| 4 | 4 | 34 | 0.000000 | 2.411765 | 2.411765 | 0.000000 | 0.00000 | 0.000000 | 0.001824 | NaN | NaN | M |
| 5 | 5 | 128 | 0.000000 | 4.000000 | 4.000000 | 0.000000 | 0.00000 | 0.000000 | 0.001823 | NaN | NaN | S |
| 6 | 6 | 148 | 0.000000 | 8.527027 | 8.527027 | 0.000000 | 0.00000 | 0.000000 | 0.001823 | 9.020000e+08 | 1.590000e+08 | M |
| 7 | 7 | 24 | 0.000000 | 1.625000 | 1.625000 | 0.000000 | 0.00000 | 0.000000 | 0.001824 | NaN | NaN | L |
| 8 | 8 | 30 | 0.000000 | 1.000000 | 1.000000 | 0.000000 | 0.00000 | 0.000000 | 0.001821 | NaN | NaN | M |
| 9 | 9 | 65 | 0.000000 | 4.861538 | 4.861538 | 0.000000 | 0.00000 | 0.000000 | 0.001823 | NaN | NaN | L |
In [12]:
import matplotlib.pyplot as plt
import numpy as np
from scipy.stats import pearsonr
# Metrics to plot (all columns except id_firm, count, and Size)
metrics = [
'firm_num_supplier_firms',
'firm_num_customer_firms',
'firm_num_products_supplied',
'firm_num_products_received',
'firm_weighted_betweenness',
'firm_weighted_closeness',
'firm_pagerank',
'Assets',
'Revenue'
]
# Create a grid of subplots (3 rows x 3 columns)
fig, axes = plt.subplots(3, 3, figsize=(15, 12))
fig.suptitle('Disruption Count vs Firm Metrics', fontsize=16, y=0.995)
# Flatten axes array for easier iteration
axes = axes.flatten()
# Plot each metric
for idx, metric in enumerate(metrics):
ax = axes[idx]
# Remove NaN values for plotting
plot_data = firm_summary[['count', metric]].dropna()
# Scatter plot
ax.scatter(plot_data['count'], plot_data[metric], alpha=0.6, s=50)
# Add trend line
if len(plot_data) > 1:
z = np.polyfit(plot_data['count'], plot_data[metric], 1)
p = np.poly1d(z)
x_trend = np.linspace(plot_data['count'].min(), plot_data['count'].max(), 100)
ax.plot(x_trend, p(x_trend), "r--", alpha=0.8, linewidth=2)
# Labels and formatting
ax.set_xlabel('Count (Number of Disruptions)', fontsize=10)
ax.set_ylabel(metric.replace('firm_', '').replace('_', ' ').title(), fontsize=10)
ax.grid(True, alpha=0.3)
# Calculate and display correlation
if len(plot_data) > 1:
corr = plot_data['count'].corr(plot_data[metric])
ax.text(0.05, 0.95, f'r = {corr:.3f}',
transform=ax.transAxes,
verticalalignment='top',
bbox=dict(boxstyle='round', facecolor='wheat', alpha=0.5))
plt.tight_layout()
plt.show()
# Print summary statistics
print("\nCorrelation with disruption count:")
print(f"{'Metric':<35} {'r':>8} {'p-value':>10} {'n':>6}")
print("="*62)
for metric in metrics:
if metric in firm_summary.columns:
plot_data = firm_summary[['count', metric]].dropna()
if len(plot_data) > 1:
r, p_value = pearsonr(plot_data['count'], plot_data[metric])
n = len(plot_data)
print(f"{metric:<35} {r:8.4f} {p_value:10.4e} {n:6d}")Correlation with disruption count: Metric r p-value n ============================================================== firm_num_supplier_firms 0.8648 1.9185e-52 171 firm_num_customer_firms 0.2781 2.3045e-04 171 firm_num_products_supplied 0.2781 2.3045e-04 171 firm_num_products_received 0.8648 1.9185e-52 171 firm_weighted_betweenness 0.7633 6.8804e-34 171 firm_weighted_closeness 0.7400 6.4124e-31 171 firm_pagerank 0.7519 2.1191e-32 171 Assets 0.3723 1.1656e-04 102 Revenue 0.2302 1.5081e-02 111
In [13]:
import matplotlib.pyplot as plt
import matplotlib
import numpy as np
# 设置中文字体为宋体
matplotlib.rcParams['font.sans-serif'] = ['SimSun'] # 宋体
matplotlib.rcParams['axes.unicode_minus'] = False # 解决负号显示问题
# 选择的指标
selected_metrics = [
('firm_weighted_betweenness', '网络中介性(Betweenness)'),
('firm_pagerank', '系统重要性(PageRank)'),
('firm_num_supplier_firms', '连接广度(上游供应商数量)'),
('Revenue', '规模属性(营业收入)')
]
# 创建一行四列的子图,共享y轴
fig, axes = plt.subplots(1, 4, figsize=(12, 4), dpi=300, sharey=True)
# 绘制每个指标
for idx, (metric, chinese_label) in enumerate(selected_metrics):
ax = axes[idx]
# 移除NaN值
plot_data = firm_summary[['count', metric]].dropna()
# 散点图(指标为x轴,count为y轴)
ax.scatter(plot_data[metric], plot_data['count'], alpha=0.6, s=50)
# 添加回归线(黑色虚线)
if len(plot_data) > 1:
z = np.polyfit(plot_data[metric], plot_data['count'], 1)
p = np.poly1d(z)
x_trend = np.linspace(plot_data[metric].min(), plot_data[metric].max(), 100)
ax.plot(x_trend, p(x_trend), "k--", alpha=0.8, linewidth=2) # 黑色虚线
# 设置标签
ax.set_xlabel(chinese_label, fontsize=12)
# 只在第一个子图显示y轴标签
if idx == 0:
ax.set_ylabel('级联失效频率', fontsize=12)
ax.grid(True, alpha=0.3)
plt.tight_layout()
plt.show()In [ ]:
In [ ]: