{ "cells": [ { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "['Hexagon AB Series B']\n", "['Hollysys Automation Technologies Ltd.']\n", "['China Electronics Huada Technology Co. Ltd.']\n", "['Kingsoft Cloud Holdings Ltd. ADR']\n", "['JD.com Inc. ADR']\n", "['Autodesk Inc.']\n", "['ABB Ltd. ADR']\n", "['Altair Engineering Inc. Cl A']\n", "['Ansys Inc.']\n", "['Fanuc Corp.']\n", "['Honeywell International Inc.']\n", "['International Business Machines Corp.']\n", "['Oracle Corp.']\n", "['PTC Inc.']\n", "['Salesforce Inc.']\n", "['SAP SE ADR']\n", "['Emerson Electric Co.']\n", "['Dassault Systemes SE']\n", "['Dell Technologies Inc. Cl C']\n", "['Texas Instruments Inc.']\n", "['Hewlett Packard Enterprise Co.']\n", "['Rockwell Automation Inc.']\n", "['Omron Corp.']\n", "['Cadence Design Systems Inc.']\n", "['Mitsubishi Electric Corp.']\n", "['Schneider Electric SE']\n", "['Cisco Systems Inc.']\n", "['General Electric Co.']\n", "['Microsoft Corp.']\n", "['Siemens AG']\n", "['Synopsys Inc.']\n", "['Analog Devices Inc.']\n", "['Amazon.com Inc.']\n", "['STMicroelectronics N.V.']\n", "['Infineon Technologies AG']\n", "['Intel Corp.']\n", "['Alibaba Group Holding Ltd. ADR']\n", "['Baidu Inc.']\n", "['Kingdee International Software Group Co. Ltd.']\n", "['Tencent Holdings Ltd.']\n", "['AsiaInfo Technologies Ltd.']\n" ] } ], "source": [ "from urllib.request import urlopen\n", "from bs4 import BeautifulSoup\n", "import time\n", "import json\n", "import pandas as pd\n", "\n", "select_year = 2021\n", "Firm = pd.read_csv(\"Firm.csv\", index_col=0)\n", "select_firm = Firm.loc[Firm[\"Source\"]==\"marketwatch\",:\"Type_Region\"]\n", "select_firm[\"Exchange\"] = None\n", "select_firm[\"Crawled_Name\"] = None\n", "select_firm[\"Stock_Currency\"] = None\n", "# select_firm = select_firm.iloc[[0],:]\n", "\n", "for code, row in select_firm.iterrows():\n", " try:\n", " url = f\"https://www.marketwatch.com/investing/stock/{row['Stock_Code']}/financials/balance-sheet\"\n", " if row['Stock_Region'] != \"US\":\n", " url += f\"?countrycode={row['Stock_Region']}\"\n", " html = urlopen(url)\n", " time.sleep(0.1)\n", " bsObj = BeautifulSoup(html.read(), 'html.parser')\n", " except Exception as e:\n", " print(e)\n", " print(\"Can't open url.\")\n", " else:\n", " basic_info = bsObj.find(name=\"script\", attrs={'type':\"application/ld+json\"}).contents[0]\n", " basic_info = json.loads(basic_info.replace('\\n','').replace('\\t',''))\n", " select_firm.loc[select_firm[\"Stock_Code\"]==row['Stock_Code'], \"Exchange\"] = basic_info[\"exchange\"]\n", " select_firm.loc[select_firm[\"Stock_Code\"]==row['Stock_Code'], \"Crawled_Name\"] = basic_info[\"name\"]\n", " select_firm.loc[select_firm[\"Stock_Code\"]==row['Stock_Code'], \"Stock_Currency\"] = basic_info[\"priceCurrency\"]\n", "\n", " bs_table = bsObj.find(name=\"table\", attrs={\"aria-label\":\"Financials - Assets data table\"})\n", " if bs_table is not None:\n", " bs = bs_table.find(name=\"tr\", attrs={\"class\":\"table__row\"}).find_all(lambda tag: tag.name == 'th' and \n", " tag.get('class') == ['overflow__heading'])\n", " years_list = [i.div.contents[0] for i in bs][0:-1]\n", " years_list = list(map(int, years_list))\n", " bs = bs_table.find_all(name=\"tr\", attrs={\"class\":\"table__row is-highlighted\"})\n", " for i in bs:\n", " if i.td.div.contents[0] == 'Total Assets':\n", " string = i.find(\"div\", attrs={\"class\":\"chart--financials js-financial-chart\"}).attrs['data-chart-data']\n", " total_assets_list = string.split(',')\n", " total_assets_list = list(map(lambda x: float(x) if x!='' else 0, total_assets_list))\n", " total_assets = pd.DataFrame({\"year\":years_list,'total_asset':total_assets_list})\n", " select_firm.loc[select_firm[\"Stock_Code\"]==row['Stock_Code'], \"Assets\"] = total_assets.loc[total_assets['year']==select_year,'total_asset'].to_list()[0]\n", " try:\n", " url = f\"https://www.marketwatch.com/investing/stock/{row['Stock_Code']}/financials\"\n", " if row['Stock_Region'] != \"US\":\n", " url += f\"?countrycode={row['Stock_Region']}\"\n", " html = urlopen(url)\n", " time.sleep(0.1)\n", " bsObj = BeautifulSoup(html.read(), 'html.parser')\n", " except Exception as e:\n", " print(e)\n", " print(\"Can't open url.\")\n", " else:\n", " basic_info = bsObj.find(name=\"script\", attrs={'type':\"application/ld+json\"}).contents[0]\n", " basic_info = json.loads(basic_info.replace('\\n','').replace('\\t',''))\n", " select_firm.loc[select_firm[\"Stock_Code\"]==row['Stock_Code'], \"Exchange\"] = basic_info[\"exchange\"]\n", " select_firm.loc[select_firm[\"Stock_Code\"]==row['Stock_Code'], \"Crawled_Name\"] = basic_info[\"name\"]\n", " select_firm.loc[select_firm[\"Stock_Code\"]==row['Stock_Code'], \"Stock_Currency\"] = basic_info[\"priceCurrency\"]\n", "\n", " bs_table = bsObj.find(name=\"table\", attrs={\"aria-label\":\"Financials - data table\"})\n", " if bs_table is not None:\n", " bs = bs_table.find(name=\"tr\", attrs={\"class\":\"table__row\"}).find_all(lambda tag: tag.name == 'th' and \n", " tag.get('class') == ['overflow__heading'])\n", " years_list = [i.div.contents[0] for i in bs][0:-1]\n", " years_list = list(map(int, years_list))\n", " bs = bs_table.find_all(name=\"tr\", attrs={\"class\":\"table__row is-highlighted\"})\n", " for i in bs:\n", " if i.td.div.contents[0] == 'Sales/Revenue':\n", " string = i.find(\"div\", attrs={\"class\":\"chart--financials js-financial-chart\"}).attrs['data-chart-data']\n", " total_revenue_list = string.split(',')\n", " total_revenue_list = list(map(lambda x: float(x) if x!='' else 0, total_revenue_list))\n", " total_revenue = pd.DataFrame({\"year\":years_list,'total_revenue':total_revenue_list})\n", " select_firm.loc[select_firm[\"Stock_Code\"]==row['Stock_Code'], \"Revenue\"] = total_revenue.loc[total_revenue['year']==select_year,'total_revenue'].to_list()[0]\n", " print(select_firm.loc[select_firm[\"Stock_Code\"]==row['Stock_Code'], \"Crawled_Name\"].to_list())\n", "\n", "select_firm['Exchange_Rate'] = select_firm['Stock_Currency'].map({'EUR': 7.2505, 'HKD': 0.88, 'JPY': 0.051, 'SEK': 0.65, 'USD': 6.88})\n", "select_firm['Assets_zh'] = select_firm['Assets'] * select_firm['Exchange_Rate'] * 0.00000001\n", "select_firm['Revenue_zh'] = select_firm['Revenue'] * select_firm['Exchange_Rate'] * 0.00000001\n", "select_firm.to_csv('MarketWatch.csv', encoding='utf-8-sig')" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "base", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.8" }, "orig_nbformat": 4, "vscode": { "interpreter": { "hash": "bcdafc093860683ffb58d6956591562b7f8ed5d58147d17d71a5d4d6605a08df" } } }, "nbformat": 4, "nbformat_minor": 2 }