Data_Preprocessing_Classification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Clean Data: The Foundation of Machine Learning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Import Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/mhossain/anaconda3/lib/python3.8/site-packages/sklearn/utils/deprecation.py:143: FutureWarning: The sklearn.neighbors.base module is  deprecated in version 0.22 and will be removed in version 0.24. The corresponding classes / functions should instead be imported from sklearn.neighbors. Anything that cannot be imported from sklearn.neighbors is now part of the private API.\n",
      "  warnings.warn(message, FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.feature_selection import VarianceThreshold\n",
    "from sklearn.preprocessing import OrdinalEncoder\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.impute import KNNImputer\n",
    "from sklearn.experimental import enable_iterative_imputer\n",
    "from sklearn.impute import IterativeImputer\n",
    "from missingpy import MissForest"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Load Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "missing_values = [\"n/a\", \"na\", \"--\", \"?\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv('Data/heart_disease.csv', na_values = missing_values)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Results of EDA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "numeric_features_final = ['age', 'trestbps', 'chol', 'thalach', 'oldpeak']\n",
    "nominal_features_final = ['sex', 'cp', 'exang', 'slope', 'thal', 'fbs', 'restecg']\n",
    "discrete_features_final = ['ca']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "### Nominal Features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "* Apply OneHot Encoder "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "### Discrete Features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "* Apply Standerd Scaler"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "### Numeric Features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "* Try Power Transformer with Normalizer\n",
    "* Try Power Transformer with Robust scaler"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "### Experiment with"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "* PCA + RFECV\n",
    "* Outlier + PCA + RFECV"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Delete Columns That Contain a Single Value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[]\n",
      "(303, 14)\n",
      "(303, 14)\n"
     ]
    }
   ],
   "source": [
    "# get number of unique values for each column\n",
    "counts = df.nunique()\n",
    "# record columns to delete\n",
    "to_del = [i for i,v in enumerate(counts) if v == 1]\n",
    "print (to_del)\n",
    "print(df.shape)\n",
    "# drop useless columns\n",
    "df.drop(to_del, axis=1, inplace=True)\n",
    "print(df.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Delete Duplicate Rows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.duplicated().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Missing Data Imputation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>cp</th>\n",
       "      <th>trestbps</th>\n",
       "      <th>chol</th>\n",
       "      <th>fbs</th>\n",
       "      <th>restecg</th>\n",
       "      <th>thalach</th>\n",
       "      <th>exang</th>\n",
       "      <th>oldpeak</th>\n",
       "      <th>slope</th>\n",
       "      <th>ca</th>\n",
       "      <th>thal</th>\n",
       "      <th>num</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>63.0</td>\n",
       "      <td>male</td>\n",
       "      <td>typical angina</td>\n",
       "      <td>145.0</td>\n",
       "      <td>233.0</td>\n",
       "      <td>True</td>\n",
       "      <td>2.0</td>\n",
       "      <td>150.0</td>\n",
       "      <td>no</td>\n",
       "      <td>2.3</td>\n",
       "      <td>downsloping</td>\n",
       "      <td>0.0</td>\n",
       "      <td>fixed defect</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>67.0</td>\n",
       "      <td>male</td>\n",
       "      <td>asymptomatic</td>\n",
       "      <td>160.0</td>\n",
       "      <td>286.0</td>\n",
       "      <td>False</td>\n",
       "      <td>2.0</td>\n",
       "      <td>108.0</td>\n",
       "      <td>yes</td>\n",
       "      <td>1.5</td>\n",
       "      <td>flat</td>\n",
       "      <td>3.0</td>\n",
       "      <td>normal</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>67.0</td>\n",
       "      <td>male</td>\n",
       "      <td>asymptomatic</td>\n",
       "      <td>120.0</td>\n",
       "      <td>229.0</td>\n",
       "      <td>False</td>\n",
       "      <td>2.0</td>\n",
       "      <td>129.0</td>\n",
       "      <td>yes</td>\n",
       "      <td>2.6</td>\n",
       "      <td>flat</td>\n",
       "      <td>2.0</td>\n",
       "      <td>reversable defect</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>37.0</td>\n",
       "      <td>male</td>\n",
       "      <td>non-anginal pain</td>\n",
       "      <td>130.0</td>\n",
       "      <td>250.0</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0</td>\n",
       "      <td>187.0</td>\n",
       "      <td>no</td>\n",
       "      <td>3.5</td>\n",
       "      <td>downsloping</td>\n",
       "      <td>0.0</td>\n",
       "      <td>normal</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>41.0</td>\n",
       "      <td>female</td>\n",
       "      <td>atypical angina</td>\n",
       "      <td>130.0</td>\n",
       "      <td>204.0</td>\n",
       "      <td>False</td>\n",
       "      <td>2.0</td>\n",
       "      <td>172.0</td>\n",
       "      <td>no</td>\n",
       "      <td>1.4</td>\n",
       "      <td>upsloping</td>\n",
       "      <td>0.0</td>\n",
       "      <td>normal</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    age     sex                cp  trestbps   chol    fbs  restecg  thalach  \\\n",
       "0  63.0    male    typical angina     145.0  233.0   True      2.0    150.0   \n",
       "1  67.0    male      asymptomatic     160.0  286.0  False      2.0    108.0   \n",
       "2  67.0    male      asymptomatic     120.0  229.0  False      2.0    129.0   \n",
       "3  37.0    male  non-anginal pain     130.0  250.0  False      0.0    187.0   \n",
       "4  41.0  female   atypical angina     130.0  204.0  False      2.0    172.0   \n",
       "\n",
       "  exang  oldpeak        slope   ca               thal  num  \n",
       "0    no      2.3  downsloping  0.0       fixed defect    0  \n",
       "1   yes      1.5         flat  3.0             normal    1  \n",
       "2   yes      2.6         flat  2.0  reversable defect    1  \n",
       "3    no      3.5  downsloping  0.0             normal    0  \n",
       "4    no      1.4    upsloping  0.0             normal    0  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.isna().sum().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'ca': 1.3201320132013201, 'thal': 0.6600660066006601}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "null_columns = {}\n",
    "\n",
    "all_columns = df.isnull().sum().sort_values(ascending=False)\n",
    "for item in all_columns.index:\n",
    "    if all_columns[item] > 0:\n",
    "        null_columns[item] = 100* all_columns[item]/len(df)\n",
    "        \n",
    "null_columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0    176\n",
       "1.0     65\n",
       "2.0     38\n",
       "3.0     20\n",
       "Name: ca, dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"ca\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "normal               166\n",
       "reversable defect    117\n",
       "fixed defect          18\n",
       "Name: thal, dtype: int64"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"thal\"].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "thal: 3 = normal; 6 = fixed defect; 7 = reversable defect"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "df['thal'] = df['thal'].map({'normal':3, 'fixed defect':6, 'reversable defect':7})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3.0    166\n",
       "7.0    117\n",
       "6.0     18\n",
       "Name: thal, dtype: int64"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"thal\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "X = df.drop(\"num\", axis=1)\n",
    "y = df[\"num\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "cat_features = nominal_features_final.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['sex', 'cp', 'exang', 'slope', 'thal', 'fbs', 'restecg']"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nominal_features_final"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "cat_features.remove(\"thal\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['sex', 'cp', 'exang', 'slope', 'fbs', 'restecg']"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat_features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "### Statistical Imputation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import RepeatedStratifiedKFold\n",
    "from sklearn.pipeline import Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "strategies = ['mean', 'median', 'most_frequent', 'constant']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">mean 0.821 (0.062)\n",
      ">median 0.809 (0.064)\n",
      ">most_frequent 0.814 (0.074)\n",
      ">constant 0.816 (0.072)\n"
     ]
    }
   ],
   "source": [
    "for s in strategies:\n",
    "    transformer = ColumnTransformer(transformers=[('cat', OneHotEncoder(drop='first', sparse=False), \n",
    "                                                   cat_features)], remainder='passthrough')\n",
    "    # create the modeling pipeline\n",
    "    pipeline = Pipeline(steps=[(\"trans\", transformer), \n",
    "                               ('i', SimpleImputer(strategy=s)), \n",
    "                               ('m', RandomForestClassifier())])\n",
    "    # evaluate the model\n",
    "    cv = RepeatedStratifiedKFold(n_splits=10, n_repeats=3, random_state=1)\n",
    "    scores = cross_val_score(pipeline, X, y, scoring='accuracy', cv=cv, n_jobs=-1)\n",
    "    print('>%s %.3f (%.3f)' % (s, np.mean(scores), np.std(scores)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "### KNN Imputer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "from sklearn.impute import KNNImputer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "strategies = [i for i in range(1, 21)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">1 0.809 (0.067)\n",
      ">2 0.816 (0.067)\n",
      ">3 0.814 (0.069)\n",
      ">4 0.804 (0.067)\n",
      ">5 0.801 (0.074)\n",
      ">6 0.795 (0.078)\n",
      ">7 0.817 (0.065)\n",
      ">8 0.809 (0.068)\n",
      ">9 0.803 (0.075)\n",
      ">10 0.799 (0.073)\n",
      ">11 0.812 (0.071)\n",
      ">12 0.809 (0.071)\n",
      ">13 0.804 (0.067)\n",
      ">14 0.814 (0.070)\n",
      ">15 0.807 (0.072)\n",
      ">16 0.812 (0.071)\n",
      ">17 0.821 (0.076)\n",
      ">18 0.813 (0.069)\n",
      ">19 0.812 (0.066)\n",
      ">20 0.803 (0.065)\n"
     ]
    }
   ],
   "source": [
    "for s in strategies:\n",
    "    transformer = ColumnTransformer(transformers=[('cat', OneHotEncoder(drop='first', sparse=False), \n",
    "                                                   cat_features)], remainder='passthrough')\n",
    "    # create the modeling pipeline\n",
    "    pipeline = Pipeline(steps=[(\"trans\", transformer),\n",
    "                               ('i', KNNImputer(n_neighbors=int(s))), \n",
    "                               ('m', RandomForestClassifier())])\n",
    "    # evaluate the model\n",
    "    cv = RepeatedStratifiedKFold(n_splits=10, n_repeats=3, random_state=1)\n",
    "    scores = cross_val_score(pipeline, X, y, scoring='accuracy', cv=cv, n_jobs=-1)\n",
    "    print('>%s %.3f (%.3f)' % (s, np.mean(scores), np.std(scores)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "### Iterative Imputation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "from sklearn.experimental import enable_iterative_imputer\n",
    "from sklearn.impute import IterativeImputer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "strategies = ['ascending', 'descending', 'roman', 'arabic', 'random']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">ascending 0.815 (0.070)\n",
      ">descending 0.811 (0.064)\n",
      ">roman 0.820 (0.063)\n",
      ">arabic 0.816 (0.062)\n",
      ">random 0.811 (0.062)\n"
     ]
    }
   ],
   "source": [
    "for s in strategies:\n",
    "    transformer = ColumnTransformer(transformers=[('cat', OneHotEncoder(drop='first', sparse=False), \n",
    "                                                   cat_features)], remainder='passthrough')\n",
    "    # create the modeling pipeline\n",
    "    pipeline = Pipeline(steps=[(\"trans\", transformer),\n",
    "                               ('i', IterativeImputer(imputation_order=s)), \n",
    "                               ('m', RandomForestClassifier())])\n",
    "    # evaluate the model\n",
    "    cv = RepeatedStratifiedKFold(n_splits=10, n_repeats=3, random_state=1)\n",
    "    scores = cross_val_score(pipeline, X, y, scoring='accuracy', cv=cv, n_jobs=-1)\n",
    "    print('>%s %.3f (%.3f)' % (s, np.mean(scores), np.std(scores)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "### MissForest Imputation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "from missingpy import MissForest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nan (nan)\n"
     ]
    }
   ],
   "source": [
    "transformer = ColumnTransformer(transformers=[('cat', OneHotEncoder(drop='first', sparse=False), \n",
    "                                                   cat_features)], remainder='passthrough')\n",
    "# create the modeling pipeline\n",
    "pipeline = Pipeline(steps=[(\"trans\", transformer),\n",
    "                               ('i', MissForest()), \n",
    "                               ('m', RandomForestClassifier())])\n",
    "# evaluate the model\n",
    "cv = RepeatedStratifiedKFold(n_splits=10, n_repeats=3, random_state=1)\n",
    "scores = cross_val_score(pipeline, X, y, scoring='accuracy', cv=cv, n_jobs=-1)\n",
    "print('%.3f (%.3f)' % (np.mean(scores), np.std(scores)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "### Missing Data Imputation Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "imputer = IterativeImputer(imputation_order='random')\n",
    "df_null_imputation = X[[\"ca\", \"thal\"]]\n",
    "# fit on the dataset\n",
    "imputer.fit(df_null_imputation)\n",
    "# transform the dataset\n",
    "Xtrans = imputer.transform(df_null_imputation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "df_new = pd.DataFrame(Xtrans, columns=[\"ca\", \"thal\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ca</th>\n",
       "      <th>thal</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2.0</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    ca  thal\n",
       "0  0.0   6.0\n",
       "1  3.0   3.0\n",
       "2  2.0   7.0\n",
       "3  0.0   3.0\n",
       "4  0.0   3.0"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_new.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3.000000    166\n",
       "7.000000    117\n",
       "6.000000     18\n",
       "4.388078      2\n",
       "Name: thal, dtype: int64"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_new[\"thal\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "df_new[\"thal_new\"] = df_new[\"thal\"].apply(lambda x: 3.0 if x == 4.388078 else x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "idx = np.where((df_new[\"thal_new\"] > 3.0) & (df_new[\"thal_new\"]<5.0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([ 87, 266]),)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "df_new.loc[87][\"thal_new\"] = 3.0\n",
    "df_new.loc[266][\"thal_new\"] = 3.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3.0    168\n",
       "7.0    117\n",
       "6.0     18\n",
       "Name: thal_new, dtype: int64"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_new[\"thal_new\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.000000    176\n",
       "1.000000     65\n",
       "2.000000     38\n",
       "3.000000     20\n",
       "0.941760      2\n",
       "0.467182      2\n",
       "Name: ca, dtype: int64"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_new[\"ca\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "df_new[\"ca_new\"] = df_new[\"ca\"].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "idx = np.where((df_new[\"ca_new\"] > 0.0) & (df_new[\"ca_new\"]<1.0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ca</th>\n",
       "      <th>thal</th>\n",
       "      <th>thal_new</th>\n",
       "      <th>ca_new</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>166</th>\n",
       "      <td>0.467182</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.467182</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>192</th>\n",
       "      <td>0.941760</td>\n",
       "      <td>7.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.941760</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>287</th>\n",
       "      <td>0.941760</td>\n",
       "      <td>7.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.941760</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>302</th>\n",
       "      <td>0.467182</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.467182</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           ca  thal  thal_new    ca_new\n",
       "166  0.467182   3.0       3.0  0.467182\n",
       "192  0.941760   7.0       7.0  0.941760\n",
       "287  0.941760   7.0       7.0  0.941760\n",
       "302  0.467182   3.0       3.0  0.467182"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_new.loc[idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "df_new.loc[166][\"ca_new\"] = 0.0\n",
    "df_new.loc[302][\"ca_new\"] = 0.0\n",
    "df_new.loc[192][\"ca_new\"] = 1.0\n",
    "df_new.loc[287][\"ca_new\"] = 1.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0    178\n",
       "1.0     67\n",
       "2.0     38\n",
       "3.0     20\n",
       "Name: ca_new, dtype: int64"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_new[\"ca_new\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "df[\"ca\"] = df_new[\"ca_new\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "df[\"thal\"] = df_new[\"thal_new\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.isnull().sum().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "##  Minimum = 0, in Numeric Column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['restecg', 'oldpeak', 'ca', 'num']"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "statistics = df.describe()\n",
    "min_value_zero_columns = [item for item in statistics if statistics[item]['min'] == 0]\n",
    "min_value_zero_columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "ca: number of major vessels (0-3) colored by flourosopy.\n",
    "\"ca\" can contain zero (0) value as per definition"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "num: diagnosis of heart disease (angiographic disease status).\n",
    "simply attemp to distinguish presence (values 1) from absence (value 0)\n",
    "So, by defintion \"num\" can contain zero (0) value"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "restecg: resting electrocardiographic results -- Value 0: normal. So, \"restecg\" can contain zero (0) valjues"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "oldpeak = ST depression induced by exercise relative to rest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0    99\n",
       "1.2    17\n",
       "1.0    14\n",
       "0.6    14\n",
       "0.8    13\n",
       "1.4    13\n",
       "0.2    12\n",
       "1.6    11\n",
       "1.8    10\n",
       "2.0     9\n",
       "0.4     9\n",
       "0.1     7\n",
       "2.8     6\n",
       "2.6     6\n",
       "0.5     5\n",
       "3.0     5\n",
       "1.5     5\n",
       "1.9     5\n",
       "2.2     4\n",
       "3.6     4\n",
       "3.4     3\n",
       "0.3     3\n",
       "2.4     3\n",
       "0.9     3\n",
       "4.0     3\n",
       "4.2     2\n",
       "2.3     2\n",
       "3.2     2\n",
       "2.5     2\n",
       "1.1     2\n",
       "3.5     1\n",
       "1.3     1\n",
       "5.6     1\n",
       "0.7     1\n",
       "2.9     1\n",
       "3.1     1\n",
       "3.8     1\n",
       "2.1     1\n",
       "6.2     1\n",
       "4.4     1\n",
       "Name: oldpeak, dtype: int64"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"oldpeak\"].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "\"Need to consult with domain expert regarding zero (0) values\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Data Preprocessing Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "numeric_features_final = ['age', 'trestbps', 'chol', 'thalach', 'oldpeak']\n",
    "nominal_features_final = ['sex', 'cp', 'exang', 'slope', 'thal', 'fbs', 'restecg']\n",
    "discrete_features_final = ['ca']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(303, 14)"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>cp</th>\n",
       "      <th>trestbps</th>\n",
       "      <th>chol</th>\n",
       "      <th>fbs</th>\n",
       "      <th>restecg</th>\n",
       "      <th>thalach</th>\n",
       "      <th>exang</th>\n",
       "      <th>oldpeak</th>\n",
       "      <th>slope</th>\n",
       "      <th>ca</th>\n",
       "      <th>thal</th>\n",
       "      <th>num</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>63.0</td>\n",
       "      <td>male</td>\n",
       "      <td>typical angina</td>\n",
       "      <td>145.0</td>\n",
       "      <td>233.0</td>\n",
       "      <td>True</td>\n",
       "      <td>2.0</td>\n",
       "      <td>150.0</td>\n",
       "      <td>no</td>\n",
       "      <td>2.3</td>\n",
       "      <td>downsloping</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>67.0</td>\n",
       "      <td>male</td>\n",
       "      <td>asymptomatic</td>\n",
       "      <td>160.0</td>\n",
       "      <td>286.0</td>\n",
       "      <td>False</td>\n",
       "      <td>2.0</td>\n",
       "      <td>108.0</td>\n",
       "      <td>yes</td>\n",
       "      <td>1.5</td>\n",
       "      <td>flat</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>67.0</td>\n",
       "      <td>male</td>\n",
       "      <td>asymptomatic</td>\n",
       "      <td>120.0</td>\n",
       "      <td>229.0</td>\n",
       "      <td>False</td>\n",
       "      <td>2.0</td>\n",
       "      <td>129.0</td>\n",
       "      <td>yes</td>\n",
       "      <td>2.6</td>\n",
       "      <td>flat</td>\n",
       "      <td>2.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>37.0</td>\n",
       "      <td>male</td>\n",
       "      <td>non-anginal pain</td>\n",
       "      <td>130.0</td>\n",
       "      <td>250.0</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0</td>\n",
       "      <td>187.0</td>\n",
       "      <td>no</td>\n",
       "      <td>3.5</td>\n",
       "      <td>downsloping</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>41.0</td>\n",
       "      <td>female</td>\n",
       "      <td>atypical angina</td>\n",
       "      <td>130.0</td>\n",
       "      <td>204.0</td>\n",
       "      <td>False</td>\n",
       "      <td>2.0</td>\n",
       "      <td>172.0</td>\n",
       "      <td>no</td>\n",
       "      <td>1.4</td>\n",
       "      <td>upsloping</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    age     sex                cp  trestbps   chol    fbs  restecg  thalach  \\\n",
       "0  63.0    male    typical angina     145.0  233.0   True      2.0    150.0   \n",
       "1  67.0    male      asymptomatic     160.0  286.0  False      2.0    108.0   \n",
       "2  67.0    male      asymptomatic     120.0  229.0  False      2.0    129.0   \n",
       "3  37.0    male  non-anginal pain     130.0  250.0  False      0.0    187.0   \n",
       "4  41.0  female   atypical angina     130.0  204.0  False      2.0    172.0   \n",
       "\n",
       "  exang  oldpeak        slope   ca  thal  num  \n",
       "0    no      2.3  downsloping  0.0   6.0    0  \n",
       "1   yes      1.5         flat  3.0   3.0    1  \n",
       "2   yes      2.6         flat  2.0   7.0    1  \n",
       "3    no      3.5  downsloping  0.0   3.0    0  \n",
       "4    no      1.4    upsloping  0.0   3.0    0  "
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "### Save Clean Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "#saving the dataframe \n",
    "df.to_csv('Data/heart_disease_clean.csv', index=None) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "### Data Prparation Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.pipeline import FeatureUnion\n",
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn.preprocessing import PowerTransformer, QuantileTransformer\n",
    "from sklearn.preprocessing import MinMaxScaler, StandardScaler, RobustScaler\n",
    "from sklearn.decomposition import PCA, TruncatedSVD\n",
    "from sklearn.neighbors import LocalOutlierFactor\n",
    "from sklearn.feature_selection import RFE, RFECV, SelectFromModel\n",
    "from xgboost import XGBClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "t = [('cat', OneHotEncoder(drop='first', sparse=False), nominal_features_final), \\\n",
    "     ('discrete', StandardScaler(), discrete_features_final),\n",
    "     ('power', PowerTransformer(), numeric_features_final),\n",
    "     ('min_max', MinMaxScaler(), numeric_features_final)]\n",
    "\n",
    "col_transform = ColumnTransformer(transformers=t)\n",
    "\n",
    "pipeline = Pipeline(steps=[('prep', col_transform), ('m', RandomForestClassifier())])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "X = df.drop(\"num\", axis=1)\n",
    "y = df[\"num\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.796 (0.072)\n"
     ]
    }
   ],
   "source": [
    "# evaluate the model\n",
    "cv = RepeatedStratifiedKFold(n_splits=10, n_repeats=3, random_state=1)\n",
    "scores = cross_val_score(pipeline, X, y, scoring='accuracy', cv=cv, n_jobs=-1)\n",
    "print('%.3f (%.3f)' % (np.mean(scores), np.std(scores)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}