Data_Preparation_All_objects.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Kasutaja\\Anaconda3\\envs\\myEnv\\lib\\site-packages\\IPython\\core\\interactiveshell.py:3063: DtypeWarning: Columns (4,8) have mixed types.Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    }
   ],
   "source": [
    "import csv\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "DS = pd.read_csv(\"MergedDSFULL.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text_ID</th>\n",
       "      <th>start</th>\n",
       "      <th>end</th>\n",
       "      <th>object</th>\n",
       "      <th>value</th>\n",
       "      <th>unit</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>46809</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>RR</td>\n",
       "      <td>130</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>RR 130/80 mmHg.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>43588</td>\n",
       "      <td>2421</td>\n",
       "      <td>2433</td>\n",
       "      <td>ALP</td>\n",
       "      <td>54</td>\n",
       "      <td>U/L</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>UURINGUD JA PROTSEDUURID \\nLÄBIVAATUS: Objekt...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>43050</td>\n",
       "      <td>720</td>\n",
       "      <td>735</td>\n",
       "      <td>HCT</td>\n",
       "      <td>40</td>\n",
       "      <td>%</td>\n",
       "      <td>NaN</td>\n",
       "      <td>41.2</td>\n",
       "      <td>UURINGUD JA PROTSEDUURID \\n|Uuring|24.04|25.0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>45389</td>\n",
       "      <td>165</td>\n",
       "      <td>171</td>\n",
       "      <td>RR</td>\n",
       "      <td>110</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Kergelt liigkaaluline. Nahk, limaskestad taval...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>42960</td>\n",
       "      <td>2314</td>\n",
       "      <td>2332</td>\n",
       "      <td>HCT</td>\n",
       "      <td>45</td>\n",
       "      <td>%</td>\n",
       "      <td>40.0</td>\n",
       "      <td>50.0</td>\n",
       "      <td>UURINGUD JA PROTSEDUURID \\nObjektiivselt: Nor...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   text_ID  start   end object value unit   min   max  \\\n",
       "0    46809      0     6     RR   130  NaN   NaN   NaN   \n",
       "1    43588   2421  2433    ALP    54  U/L   NaN   NaN   \n",
       "2    43050    720   735    HCT    40    %   NaN  41.2   \n",
       "3    45389    165   171     RR   110  NaN   NaN   NaN   \n",
       "4    42960   2314  2332    HCT    45    %  40.0  50.0   \n",
       "\n",
       "                                                text  \n",
       "0                                    RR 130/80 mmHg.  \n",
       "1   UURINGUD JA PROTSEDUURID \\nLÄBIVAATUS: Objekt...  \n",
       "2   UURINGUD JA PROTSEDUURID \\n|Uuring|24.04|25.0...  \n",
       "3  Kergelt liigkaaluline. Nahk, limaskestad taval...  \n",
       "4   UURINGUD JA PROTSEDUURID \\nObjektiivselt: Nor...  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DS.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# DS150 = DS[DS['text'].apply(len)<150]\n",
    "DS150 = DS[DS['text'].apply(lambda x: len(str(x)) < 150)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "DSIds = set(DS150.text_ID)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "DSIDSList = list(DSIds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "DSIDSList.sort()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "304534"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DSIDSList[12999]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "DS150Subset = DS150[DS150['text_ID'] <= 304534]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "33849"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(DS150Subset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "13000"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(set(DS150Subset[\"text_ID\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "176"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(set(DS150Subset[\"object\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "ObjectStats = DS[\"object\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RR                                           138495\n",
       "MCV                                           54799\n",
       "MCH                                           52742\n",
       "MCHC                                          52696\n",
       "fr                                            51654\n",
       "                                              ...  \n",
       "juurde tulnud - talvega                           1\n",
       "kaalulangus 10 kg viimase 2 kuu jooksul           1\n",
       "juurde võtnud umbes                               1\n",
       "kaalus alla. RR kodus ja tööl 124-138/85-         1\n",
       "langenud - seega                                  1\n",
       "Name: object, Length: 2125, dtype: int64"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ObjectStats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count      2125.000000\n",
       "mean        768.403294\n",
       "std        5068.159364\n",
       "min           1.000000\n",
       "25%           1.000000\n",
       "50%           2.000000\n",
       "75%          15.000000\n",
       "max      138495.000000\n",
       "Name: object, dtype: float64"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ObjectStats.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "DS150Subset.to_csv(\"DS150SubsetAllObjects.csv\",index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text_ID</th>\n",
       "      <th>start</th>\n",
       "      <th>end</th>\n",
       "      <th>object</th>\n",
       "      <th>value</th>\n",
       "      <th>unit</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>46809</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>RR</td>\n",
       "      <td>130</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>RR 130/80 mmHg.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>47482</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>RR</td>\n",
       "      <td>142</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>RR142/89mmHg, HR 76 x min, p167cm, k89kg.  EKG...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>53393</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>RR</td>\n",
       "      <td>143</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>RR143/87mmHg, HR60 x min, p180cm, k93kg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>51616</td>\n",
       "      <td>98</td>\n",
       "      <td>103</td>\n",
       "      <td>fr</td>\n",
       "      <td>72</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>\\nNahaleid: puhas\\nHingamiselundite leid: vesi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>101498</td>\n",
       "      <td>15</td>\n",
       "      <td>23</td>\n",
       "      <td>fr</td>\n",
       "      <td>83</td>\n",
       "      <td>x'</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Cor regul rütm fr 83 x', toonid ii, RR 150/100...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    text_ID  start  end object value unit  min  max  \\\n",
       "0     46809      0    6     RR   130  NaN  NaN  NaN   \n",
       "5     47482      0    5     RR   142  NaN  NaN  NaN   \n",
       "6     53393      0    5     RR   143  NaN  NaN  NaN   \n",
       "8     51616     98  103     fr    72  NaN  NaN  NaN   \n",
       "26   101498     15   23     fr    83   x'  NaN  NaN   \n",
       "\n",
       "                                                 text  \n",
       "0                                     RR 130/80 mmHg.  \n",
       "5   RR142/89mmHg, HR 76 x min, p167cm, k89kg.  EKG...  \n",
       "6             RR143/87mmHg, HR60 x min, p180cm, k93kg  \n",
       "8   \\nNahaleid: puhas\\nHingamiselundite leid: vesi...  \n",
       "26  Cor regul rütm fr 83 x', toonid ii, RR 150/100...  "
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DS150Subset.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "DS = DS150Subset.copy()\n",
    "new_df = pd.DataFrame(DS[\"text\"].str.split().tolist(), index=DS.text_ID).stack()\n",
    "new_df = new_df.reset_index([0, 'text_ID'])\n",
    "new_df.columns = ['text_ID', 'word']\n",
    "new_df[\"tag\"]= np.NaN\n",
    "myList= list(new_df.text_ID.unique())\n",
    "for i in range(len(myList)):\n",
    "    Original = DS.loc[DS[\"text_ID\"] == myList[i]]\n",
    "    words = new_df.loc[new_df[\"text_ID\"] == myList[i]]\n",
    "    for OriginalIndex, OriginalRow in Original.iterrows():\n",
    "        for index, row in words.iterrows():\n",
    "                if(str(Original[\"object\"][OriginalIndex]) in str(words[\"word\"][index])):\n",
    "                    new_df[\"tag\"][index] = \"object\"\n",
    "                elif(not float(Original[\"value\"][OriginalIndex]).is_integer()):\n",
    "                    if(str(Original[\"value\"][OriginalIndex]) in str(words[\"word\"][index])):\n",
    "                        new_df[\"tag\"][index] = \"object\"\n",
    "                elif(str(int(Original[\"value\"][OriginalIndex])) in str(words[\"word\"][index])):\n",
    "                    new_df[\"tag\"][index] = \"object\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_df = new_df[0:319755]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "12100"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(set(new_df.text_ID))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_df.to_csv(\"DS150SubsetAllObjectsPrepared.csv\",index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "157280"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(new_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "13643"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(new_df.text_ID.unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "object    9955\n",
       "Name: tag, dtype: int64"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_df[\"tag\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "309800"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_df[\"tag\"].isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text_ID</th>\n",
       "      <th>word</th>\n",
       "      <th>tag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>46809</td>\n",
       "      <td>RR</td>\n",
       "      <td>object</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>46809</td>\n",
       "      <td>130/80</td>\n",
       "      <td>object</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>46809</td>\n",
       "      <td>mmHg.</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>47482</td>\n",
       "      <td>RR142/89mmHg,</td>\n",
       "      <td>object</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>47482</td>\n",
       "      <td>HR</td>\n",
       "      <td>object</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   text_ID           word     tag\n",
       "0    46809             RR  object\n",
       "1    46809         130/80  object\n",
       "2    46809          mmHg.     NaN\n",
       "3    47482  RR142/89mmHg,  object\n",
       "4    47482             HR  object"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "words = list(set(new_df[\"word\"].values))\n",
    "n_words = len(words);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "17330"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}