notebooks/mod_05_topic_09_ensembles.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Практика застосування ансамблів. EDA датасету Employee"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Знайомство з даними"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import pandas as pd\n",
    "from sklearn.ensemble import (\n",
    "    StackingClassifier,\n",
    "    VotingClassifier,\n",
    "    GradientBoostingClassifier,\n",
    "    AdaBoostClassifier,\n",
    "    BaggingClassifier,\n",
    "    RandomForestClassifier)\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from imblearn.over_sampling import SMOTE\n",
    "import category_encoders as ce\n",
    "from sklearn.metrics import f1_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Education</th>\n",
       "      <th>JoiningYear</th>\n",
       "      <th>City</th>\n",
       "      <th>PaymentTier</th>\n",
       "      <th>Age</th>\n",
       "      <th>Gender</th>\n",
       "      <th>EverBenched</th>\n",
       "      <th>ExperienceInCurrentDomain</th>\n",
       "      <th>LeaveOrNot</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Bachelors</td>\n",
       "      <td>2017</td>\n",
       "      <td>Bangalore</td>\n",
       "      <td>3</td>\n",
       "      <td>34</td>\n",
       "      <td>Male</td>\n",
       "      <td>No</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Bachelors</td>\n",
       "      <td>2013</td>\n",
       "      <td>Pune</td>\n",
       "      <td>1</td>\n",
       "      <td>28</td>\n",
       "      <td>Female</td>\n",
       "      <td>No</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Bachelors</td>\n",
       "      <td>2014</td>\n",
       "      <td>New Delhi</td>\n",
       "      <td>3</td>\n",
       "      <td>38</td>\n",
       "      <td>Female</td>\n",
       "      <td>No</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Masters</td>\n",
       "      <td>2016</td>\n",
       "      <td>Bangalore</td>\n",
       "      <td>3</td>\n",
       "      <td>27</td>\n",
       "      <td>Male</td>\n",
       "      <td>No</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Masters</td>\n",
       "      <td>2017</td>\n",
       "      <td>Pune</td>\n",
       "      <td>3</td>\n",
       "      <td>24</td>\n",
       "      <td>Male</td>\n",
       "      <td>Yes</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Education  JoiningYear  ... ExperienceInCurrentDomain  LeaveOrNot\n",
       "0  Bachelors         2017  ...                         0           0\n",
       "1  Bachelors         2013  ...                         3           1\n",
       "2  Bachelors         2014  ...                         2           0\n",
       "3    Masters         2016  ...                         5           1\n",
       "4    Masters         2017  ...                         2           1\n",
       "\n",
       "[5 rows x 9 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv('../datasets/mod_05_topic_09_employee_data.csv')\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Перевірка типів даних і відсутніх значень"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 4653 entries, 0 to 4652\n",
      "Data columns (total 9 columns):\n",
      " #   Column                     Non-Null Count  Dtype \n",
      "---  ------                     --------------  ----- \n",
      " 0   Education                  4653 non-null   object\n",
      " 1   JoiningYear                4653 non-null   int64 \n",
      " 2   City                       4653 non-null   object\n",
      " 3   PaymentTier                4653 non-null   int64 \n",
      " 4   Age                        4653 non-null   int64 \n",
      " 5   Gender                     4653 non-null   object\n",
      " 6   EverBenched                4653 non-null   object\n",
      " 7   ExperienceInCurrentDomain  4653 non-null   int64 \n",
      " 8   LeaveOrNot                 4653 non-null   int64 \n",
      "dtypes: int64(5), object(4)\n",
      "memory usage: 327.3+ KB\n"
     ]
    }
   ],
   "source": [
    "data.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Первинна адаптація ознак"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['JoiningYear'] = data['JoiningYear'].max() - data['JoiningYear']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['PaymentTier'] = data['PaymentTier'].astype(str)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Практика застосування ансамблів. Підготовка й обробка даних"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Розбиття на тренувальну і тестову вибірки"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = (\n",
    "    train_test_split(\n",
    "        data.drop('LeaveOrNot', axis=1),\n",
    "        data['LeaveOrNot'],\n",
    "        test_size=0.33,\n",
    "        random_state=42))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Кодування категоріальних змінних"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoder = ce.TargetEncoder()\n",
    "\n",
    "X_train = encoder.fit_transform(X_train, y_train)\n",
    "X_test = encoder.transform(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Нормалізація змінних"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "scaler = StandardScaler().set_output(transform='pandas')\n",
    "\n",
    "X_train = scaler.fit_transform(X_train)\n",
    "X_test = scaler.transform(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Балансування класів"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LeaveOrNot\n",
       "0    0.657363\n",
       "1    0.342637\n",
       "Name: proportion, dtype: float64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train.value_counts(normalize=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "sm = SMOTE(random_state=42)\n",
    "X_res, y_res = sm.fit_resample(X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Практика застосування ансамблів. Навчання й оцінка ансамблів моделей"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "f1_scores = {}\n",
    "\n",
    "\n",
    "def measure_f1_time_decorator(func):\n",
    "    def wrapper(*args, **kwargs):\n",
    "        start_time = time.time()\n",
    "        predictions = func(*args, **kwargs)\n",
    "        end_time = time.time()\n",
    "        f1 = f1_score(args[-1], predictions)\n",
    "        model_name = args[0].__class__.__name__\n",
    "        execution_time = end_time - start_time\n",
    "        f1_scores[model_name] = [f1, execution_time]\n",
    "        print(f'{model_name} F1 Metric: {f1:.4f}')\n",
    "        print(f'{model_name} Inference: {execution_time:.4f} s')\n",
    "        return predictions\n",
    "    return wrapper\n",
    "\n",
    "\n",
    "@measure_f1_time_decorator\n",
    "def predict_with_measure(model, Xt, yt):\n",
    "    return model.predict(Xt)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Побудова базової моделі (baseline)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression F1 Metric: 0.6376\n",
      "LogisticRegression Inference: 0.0030 s\n"
     ]
    }
   ],
   "source": [
    "mod_log_reg = (LogisticRegression(\n",
    "    # n_jobs=-1\n",
    ").fit(X_res, y_res))\n",
    "\n",
    "prd_log_reg = predict_with_measure(mod_log_reg, X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Побудова RandomForestClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForestClassifier F1 Metric: 0.7594\n",
      "RandomForestClassifier Inference: 0.0200 s\n"
     ]
    }
   ],
   "source": [
    "mod_rnd_frs = (RandomForestClassifier(\n",
    "    random_state=42,\n",
    "    # n_jobs=-1\n",
    ")\n",
    "    .fit(X_res, y_res))\n",
    "\n",
    "prd_rnd_frs = predict_with_measure(mod_rnd_frs, X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Побудова Bagging Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BaggingClassifier F1 Metric: 0.7518\n",
      "BaggingClassifier Inference: 0.1310 s\n"
     ]
    }
   ],
   "source": [
    "mod_bag_knn = (BaggingClassifier(\n",
    "    KNeighborsClassifier(),\n",
    "    max_samples=0.75,\n",
    "    max_features=0.75,\n",
    "    # n_jobs=-1,\n",
    "    random_state=42)\n",
    "    .fit(X_res, y_res))\n",
    "\n",
    "prd_bag_knn = predict_with_measure(mod_bag_knn, X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Побудова AdaBoostClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AdaBoostClassifier F1 Metric: 0.7271\n",
      "AdaBoostClassifier Inference: 0.0090 s\n"
     ]
    }
   ],
   "source": [
    "mod_ada_bst = (AdaBoostClassifier(\n",
    "    algorithm='SAMME',\n",
    "    random_state=42)\n",
    "    .fit(X_res, y_res))\n",
    "\n",
    "prd_ada_bst = predict_with_measure(mod_ada_bst, X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Побудова GradientBoostingClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GradientBoostingClassifier F1 Metric: 0.7876\n",
      "GradientBoostingClassifier Inference: 0.0030 s\n"
     ]
    }
   ],
   "source": [
    "mod_grd_bst = (GradientBoostingClassifier(\n",
    "    learning_rate=0.3,\n",
    "    subsample=0.75,\n",
    "    max_features='sqrt',\n",
    "    random_state=42)\n",
    "    .fit(X_res, y_res))\n",
    "\n",
    "prd_grd_bst = predict_with_measure(mod_grd_bst, X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Побудова VotingClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "VotingClassifier F1 Metric: 0.7007\n",
      "VotingClassifier Inference: 0.0250 s\n"
     ]
    }
   ],
   "source": [
    "clf1 = LogisticRegression()\n",
    "clf2 = KNeighborsClassifier()\n",
    "clf3 = GaussianNB()\n",
    "\n",
    "estimators = [('lnr', clf1),\n",
    "              ('knn', clf2),\n",
    "              ('gnb', clf3)]\n",
    "\n",
    "mod_vot_clf = VotingClassifier(\n",
    "    estimators=estimators,\n",
    "    voting='soft').fit(X_res, y_res)\n",
    "\n",
    "prd_vot_clf = predict_with_measure(mod_vot_clf, X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Побудова StackingClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "StackingClassifier F1 Metric: 0.7295\n",
      "StackingClassifier Inference: 0.0270 s\n"
     ]
    }
   ],
   "source": [
    "final_estimator = GradientBoostingClassifier(\n",
    "    subsample=0.75,\n",
    "    max_features='sqrt',\n",
    "    random_state=42)\n",
    "\n",
    "mod_stk_clf = StackingClassifier(\n",
    "    estimators=estimators,\n",
    "    final_estimator=final_estimator).fit(X_res, y_res)\n",
    "\n",
    "prd_stk_clf = predict_with_measure(mod_stk_clf, X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Порівняння ефективності ансамблів"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>f1</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>GradientBoostingClassifier</th>\n",
       "      <td>0.787565</td>\n",
       "      <td>0.002976</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RandomForestClassifier</th>\n",
       "      <td>0.759443</td>\n",
       "      <td>0.019978</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BaggingClassifier</th>\n",
       "      <td>0.751773</td>\n",
       "      <td>0.131000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>StackingClassifier</th>\n",
       "      <td>0.729524</td>\n",
       "      <td>0.027001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AdaBoostClassifier</th>\n",
       "      <td>0.727094</td>\n",
       "      <td>0.008998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>VotingClassifier</th>\n",
       "      <td>0.700704</td>\n",
       "      <td>0.024989</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LogisticRegression</th>\n",
       "      <td>0.637584</td>\n",
       "      <td>0.003029</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  f1      time\n",
       "GradientBoostingClassifier  0.787565  0.002976\n",
       "RandomForestClassifier      0.759443  0.019978\n",
       "BaggingClassifier           0.751773  0.131000\n",
       "StackingClassifier          0.729524  0.027001\n",
       "AdaBoostClassifier          0.727094  0.008998\n",
       "VotingClassifier            0.700704  0.024989\n",
       "LogisticRegression          0.637584  0.003029"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores = pd.DataFrame.from_dict(\n",
    "    f1_scores,\n",
    "    orient='index',\n",
    "    columns=['f1', 'time'])\n",
    "\n",
    "scores.sort_values('f1', ascending=False)"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3 (Spyder)",
   "language": "python3",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}