1. get_stocks.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup as bs\n",
    "import yfinance as yf\n",
    "from tqdm import tqdm\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get Ticker Names\n",
    "url = 'https://topforeignstocks.com/stock-lists/the-complete-list-of-oil-gas-production-stocks-trading-on-the-nyse/'\n",
    "r = requests.get(url)\n",
    "soup = bs(r.text)\n",
    "txts = []\n",
    "for link in soup.find_all('a'):\n",
    "    txts.append(link.text)\n",
    "\n",
    "tickers = txts[12:113]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 101/101 [00:13<00:00,  7.29it/s]\n"
     ]
    }
   ],
   "source": [
    "# Download Data\n",
    "data = []\n",
    "for i in tqdm(range(len(tickers))):\n",
    "    ticker = yf.Ticker(tickers[i])\n",
    "    data.append(ticker.history(period='1y'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "79\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "251\n",
      "252\n",
      "252\n",
      "252\n",
      "229\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "251\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "225\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "252\n",
      "227\n"
     ]
    }
   ],
   "source": [
    "for d in data:\n",
    "    print(len(d))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Clean Data\n",
    "col_names = []\n",
    "close = []\n",
    "for i in range(len(data)):\n",
    "    if len(data[i]) != 252:\n",
    "        continue\n",
    "    else:\n",
    "        col_names.append(tickers[i])\n",
    "        close.append(data[i]['Close'])\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.concat(close, axis=1, keys=col_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AMPY</th>\n",
       "      <th>AR</th>\n",
       "      <th>BRN</th>\n",
       "      <th>BATL</th>\n",
       "      <th>BSM</th>\n",
       "      <th>BCEI</th>\n",
       "      <th>BORR</th>\n",
       "      <th>MNRL</th>\n",
       "      <th>COG</th>\n",
       "      <th>CPE</th>\n",
       "      <th>...</th>\n",
       "      <th>TTI</th>\n",
       "      <th>TPL</th>\n",
       "      <th>TOT</th>\n",
       "      <th>RIG</th>\n",
       "      <th>EGY</th>\n",
       "      <th>VET</th>\n",
       "      <th>VIST</th>\n",
       "      <th>VOC</th>\n",
       "      <th>WTI</th>\n",
       "      <th>WLL</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Date</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2020-02-24</th>\n",
       "      <td>4.527368</td>\n",
       "      <td>1.66</td>\n",
       "      <td>0.860</td>\n",
       "      <td>11.000</td>\n",
       "      <td>9.282617</td>\n",
       "      <td>17.610001</td>\n",
       "      <td>3.020</td>\n",
       "      <td>14.480885</td>\n",
       "      <td>14.725897</td>\n",
       "      <td>24.799999</td>\n",
       "      <td>...</td>\n",
       "      <td>1.34</td>\n",
       "      <td>722.169495</td>\n",
       "      <td>41.927280</td>\n",
       "      <td>3.79</td>\n",
       "      <td>2.06</td>\n",
       "      <td>12.750500</td>\n",
       "      <td>7.05</td>\n",
       "      <td>3.796676</td>\n",
       "      <td>3.23</td>\n",
       "      <td>2.540000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020-02-25</th>\n",
       "      <td>4.160526</td>\n",
       "      <td>1.65</td>\n",
       "      <td>0.820</td>\n",
       "      <td>10.410</td>\n",
       "      <td>8.314322</td>\n",
       "      <td>16.600000</td>\n",
       "      <td>2.870</td>\n",
       "      <td>13.801669</td>\n",
       "      <td>14.285586</td>\n",
       "      <td>23.500000</td>\n",
       "      <td>...</td>\n",
       "      <td>1.33</td>\n",
       "      <td>696.954163</td>\n",
       "      <td>40.656754</td>\n",
       "      <td>3.56</td>\n",
       "      <td>2.06</td>\n",
       "      <td>12.062833</td>\n",
       "      <td>6.73</td>\n",
       "      <td>3.759635</td>\n",
       "      <td>3.13</td>\n",
       "      <td>2.410000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020-02-26</th>\n",
       "      <td>3.874210</td>\n",
       "      <td>1.53</td>\n",
       "      <td>0.740</td>\n",
       "      <td>9.475</td>\n",
       "      <td>8.035004</td>\n",
       "      <td>15.990000</td>\n",
       "      <td>2.763</td>\n",
       "      <td>13.484702</td>\n",
       "      <td>14.344294</td>\n",
       "      <td>22.299999</td>\n",
       "      <td>...</td>\n",
       "      <td>1.43</td>\n",
       "      <td>683.293823</td>\n",
       "      <td>40.426590</td>\n",
       "      <td>3.43</td>\n",
       "      <td>2.02</td>\n",
       "      <td>11.021780</td>\n",
       "      <td>6.58</td>\n",
       "      <td>3.685554</td>\n",
       "      <td>2.90</td>\n",
       "      <td>2.260000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020-02-27</th>\n",
       "      <td>3.623684</td>\n",
       "      <td>1.37</td>\n",
       "      <td>0.660</td>\n",
       "      <td>8.400</td>\n",
       "      <td>8.044314</td>\n",
       "      <td>14.420000</td>\n",
       "      <td>2.679</td>\n",
       "      <td>13.068116</td>\n",
       "      <td>13.219060</td>\n",
       "      <td>20.400000</td>\n",
       "      <td>...</td>\n",
       "      <td>1.39</td>\n",
       "      <td>651.964600</td>\n",
       "      <td>39.358608</td>\n",
       "      <td>3.01</td>\n",
       "      <td>1.95</td>\n",
       "      <td>9.967793</td>\n",
       "      <td>6.16</td>\n",
       "      <td>3.546651</td>\n",
       "      <td>2.83</td>\n",
       "      <td>1.710000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020-02-28</th>\n",
       "      <td>3.740000</td>\n",
       "      <td>1.60</td>\n",
       "      <td>0.590</td>\n",
       "      <td>7.480</td>\n",
       "      <td>8.239836</td>\n",
       "      <td>16.290001</td>\n",
       "      <td>2.405</td>\n",
       "      <td>14.444659</td>\n",
       "      <td>13.630015</td>\n",
       "      <td>22.700001</td>\n",
       "      <td>...</td>\n",
       "      <td>1.26</td>\n",
       "      <td>668.883728</td>\n",
       "      <td>39.717670</td>\n",
       "      <td>3.35</td>\n",
       "      <td>2.00</td>\n",
       "      <td>9.744778</td>\n",
       "      <td>6.23</td>\n",
       "      <td>3.333667</td>\n",
       "      <td>2.60</td>\n",
       "      <td>1.850000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-02-16</th>\n",
       "      <td>3.240000</td>\n",
       "      <td>9.49</td>\n",
       "      <td>3.700</td>\n",
       "      <td>11.200</td>\n",
       "      <td>9.760000</td>\n",
       "      <td>26.940001</td>\n",
       "      <td>1.560</td>\n",
       "      <td>15.890000</td>\n",
       "      <td>18.309999</td>\n",
       "      <td>24.610001</td>\n",
       "      <td>...</td>\n",
       "      <td>2.69</td>\n",
       "      <td>1164.229980</td>\n",
       "      <td>44.150002</td>\n",
       "      <td>3.68</td>\n",
       "      <td>3.07</td>\n",
       "      <td>5.810000</td>\n",
       "      <td>3.03</td>\n",
       "      <td>3.070000</td>\n",
       "      <td>3.02</td>\n",
       "      <td>29.959999</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-02-17</th>\n",
       "      <td>3.000000</td>\n",
       "      <td>9.78</td>\n",
       "      <td>3.540</td>\n",
       "      <td>12.180</td>\n",
       "      <td>9.750000</td>\n",
       "      <td>27.010000</td>\n",
       "      <td>1.390</td>\n",
       "      <td>15.930000</td>\n",
       "      <td>18.580000</td>\n",
       "      <td>24.780001</td>\n",
       "      <td>...</td>\n",
       "      <td>2.63</td>\n",
       "      <td>1142.829956</td>\n",
       "      <td>45.049999</td>\n",
       "      <td>3.67</td>\n",
       "      <td>3.00</td>\n",
       "      <td>5.790000</td>\n",
       "      <td>3.07</td>\n",
       "      <td>3.030000</td>\n",
       "      <td>3.00</td>\n",
       "      <td>31.620001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-02-18</th>\n",
       "      <td>2.750000</td>\n",
       "      <td>8.96</td>\n",
       "      <td>4.200</td>\n",
       "      <td>11.990</td>\n",
       "      <td>9.340000</td>\n",
       "      <td>26.680000</td>\n",
       "      <td>1.280</td>\n",
       "      <td>15.430000</td>\n",
       "      <td>18.059999</td>\n",
       "      <td>22.270000</td>\n",
       "      <td>...</td>\n",
       "      <td>2.41</td>\n",
       "      <td>1109.829956</td>\n",
       "      <td>44.049999</td>\n",
       "      <td>3.42</td>\n",
       "      <td>2.95</td>\n",
       "      <td>5.530000</td>\n",
       "      <td>3.01</td>\n",
       "      <td>3.030000</td>\n",
       "      <td>2.89</td>\n",
       "      <td>31.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-02-19</th>\n",
       "      <td>2.970000</td>\n",
       "      <td>8.96</td>\n",
       "      <td>4.210</td>\n",
       "      <td>11.160</td>\n",
       "      <td>9.450000</td>\n",
       "      <td>27.059999</td>\n",
       "      <td>1.340</td>\n",
       "      <td>15.500000</td>\n",
       "      <td>18.559999</td>\n",
       "      <td>22.830000</td>\n",
       "      <td>...</td>\n",
       "      <td>2.55</td>\n",
       "      <td>1109.000000</td>\n",
       "      <td>44.970001</td>\n",
       "      <td>3.49</td>\n",
       "      <td>2.98</td>\n",
       "      <td>5.520000</td>\n",
       "      <td>3.05</td>\n",
       "      <td>3.030000</td>\n",
       "      <td>2.92</td>\n",
       "      <td>32.590000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-02-22</th>\n",
       "      <td>3.110000</td>\n",
       "      <td>8.76</td>\n",
       "      <td>4.298</td>\n",
       "      <td>11.100</td>\n",
       "      <td>9.690000</td>\n",
       "      <td>29.190001</td>\n",
       "      <td>1.330</td>\n",
       "      <td>15.640000</td>\n",
       "      <td>18.340000</td>\n",
       "      <td>24.690001</td>\n",
       "      <td>...</td>\n",
       "      <td>2.49</td>\n",
       "      <td>1100.010010</td>\n",
       "      <td>45.549999</td>\n",
       "      <td>3.52</td>\n",
       "      <td>3.10</td>\n",
       "      <td>6.240000</td>\n",
       "      <td>2.70</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>2.99</td>\n",
       "      <td>34.470001</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>252 rows × 95 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                AMPY    AR    BRN    BATL       BSM       BCEI   BORR  \\\n",
       "Date                                                                    \n",
       "2020-02-24  4.527368  1.66  0.860  11.000  9.282617  17.610001  3.020   \n",
       "2020-02-25  4.160526  1.65  0.820  10.410  8.314322  16.600000  2.870   \n",
       "2020-02-26  3.874210  1.53  0.740   9.475  8.035004  15.990000  2.763   \n",
       "2020-02-27  3.623684  1.37  0.660   8.400  8.044314  14.420000  2.679   \n",
       "2020-02-28  3.740000  1.60  0.590   7.480  8.239836  16.290001  2.405   \n",
       "...              ...   ...    ...     ...       ...        ...    ...   \n",
       "2021-02-16  3.240000  9.49  3.700  11.200  9.760000  26.940001  1.560   \n",
       "2021-02-17  3.000000  9.78  3.540  12.180  9.750000  27.010000  1.390   \n",
       "2021-02-18  2.750000  8.96  4.200  11.990  9.340000  26.680000  1.280   \n",
       "2021-02-19  2.970000  8.96  4.210  11.160  9.450000  27.059999  1.340   \n",
       "2021-02-22  3.110000  8.76  4.298  11.100  9.690000  29.190001  1.330   \n",
       "\n",
       "                 MNRL        COG        CPE  ...   TTI          TPL  \\\n",
       "Date                                         ...                      \n",
       "2020-02-24  14.480885  14.725897  24.799999  ...  1.34   722.169495   \n",
       "2020-02-25  13.801669  14.285586  23.500000  ...  1.33   696.954163   \n",
       "2020-02-26  13.484702  14.344294  22.299999  ...  1.43   683.293823   \n",
       "2020-02-27  13.068116  13.219060  20.400000  ...  1.39   651.964600   \n",
       "2020-02-28  14.444659  13.630015  22.700001  ...  1.26   668.883728   \n",
       "...               ...        ...        ...  ...   ...          ...   \n",
       "2021-02-16  15.890000  18.309999  24.610001  ...  2.69  1164.229980   \n",
       "2021-02-17  15.930000  18.580000  24.780001  ...  2.63  1142.829956   \n",
       "2021-02-18  15.430000  18.059999  22.270000  ...  2.41  1109.829956   \n",
       "2021-02-19  15.500000  18.559999  22.830000  ...  2.55  1109.000000   \n",
       "2021-02-22  15.640000  18.340000  24.690001  ...  2.49  1100.010010   \n",
       "\n",
       "                  TOT   RIG   EGY        VET  VIST       VOC   WTI        WLL  \n",
       "Date                                                                           \n",
       "2020-02-24  41.927280  3.79  2.06  12.750500  7.05  3.796676  3.23   2.540000  \n",
       "2020-02-25  40.656754  3.56  2.06  12.062833  6.73  3.759635  3.13   2.410000  \n",
       "2020-02-26  40.426590  3.43  2.02  11.021780  6.58  3.685554  2.90   2.260000  \n",
       "2020-02-27  39.358608  3.01  1.95   9.967793  6.16  3.546651  2.83   1.710000  \n",
       "2020-02-28  39.717670  3.35  2.00   9.744778  6.23  3.333667  2.60   1.850000  \n",
       "...               ...   ...   ...        ...   ...       ...   ...        ...  \n",
       "2021-02-16  44.150002  3.68  3.07   5.810000  3.03  3.070000  3.02  29.959999  \n",
       "2021-02-17  45.049999  3.67  3.00   5.790000  3.07  3.030000  3.00  31.620001  \n",
       "2021-02-18  44.049999  3.42  2.95   5.530000  3.01  3.030000  2.89  31.000000  \n",
       "2021-02-19  44.970001  3.49  2.98   5.520000  3.05  3.030000  2.92  32.590000  \n",
       "2021-02-22  45.549999  3.52  3.10   6.240000  2.70  3.000000  2.99  34.470001  \n",
       "\n",
       "[252 rows x 95 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('oilgas_1y.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}