set04_22_pair_combi_fake.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ff3d4b88",
   "metadata": {},
   "outputs": [],
   "source": [
    "# calculate elapsed time\n",
    "import time\n",
    "start_time = time.time()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aff04440-8080-4635-ac32-430ac6f7eff4",
   "metadata": {},
   "source": [
    "#### IMPORT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "af71c6bf-604a-4f0c-a18c-6f91bbcf9bc3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os, platform\n",
    "import py7zr\n",
    "\n",
    "from IPython.display import clear_output\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "from matplotlib import pyplot\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow as tf\n",
    "\n",
    "from sklearn.preprocessing import LabelBinarizer\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "from keras.utils.np_utils import to_categorical \n",
    "from tensorflow.keras.callbacks import EarlyStopping\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, \\\n",
    "    BatchNormalization, Flatten, LSTM\n",
    "# from scikeras.wrappers import KerasClassifier\n",
    "from keras.wrappers.scikit_learn import KerasClassifier\n",
    "from keras.models import load_model\n",
    "\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.metrics import accuracy_score, confusion_matrix\n",
    "from sklearn.metrics import roc_curve, roc_auc_score, auc\n",
    "\n",
    "from itertools import cycle\n",
    "from scipy import interp\n",
    "from scipy.optimize import brentq\n",
    "from scipy.interpolate import interp1d"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cb45a7c3",
   "metadata": {},
   "source": [
    "#### PARAMETERS\n",
    "- Set the condition\n",
    "> * N_FEATURES: Number of Features\n",
    "> * CHECK_BLANKS: Check for blank data. If any blank data is found, the whole row of data will be deleted.\n",
    "> * CHECK_CLASS_IMBALANCE: Check for dataset class imbalance. The more balance the dataset, the less biases the model will be."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b7460824",
   "metadata": {},
   "outputs": [],
   "source": [
    "##############################\n",
    "# # deep learning features # #\n",
    "##############################\n",
    "SEED = 1005 # random seed for reproducibility\n",
    "\n",
    "# should make this dynamic\n",
    "N_FEATURES = 440\n",
    "TIMESTEPS = 1\n",
    "EPOCH=200\n",
    "BATCH_SIZE=100\n",
    "\n",
    "SPLIT_RATIO=0.2\n",
    "\n",
    "###############\n",
    "# # preview # #\n",
    "###############\n",
    "f = False\n",
    "t = True\n",
    "\n",
    "# checking dataset\n",
    "CHECK_BLANKS = True\n",
    "CHECK_CLASS_IMBALANCE = True\n",
    "\n",
    "###############\n",
    "# # dataset # #\n",
    "###############\n",
    "CLASSES_COL_NAME = \"Subject\"\n",
    "CLASSES_COL_NUM = 0\n",
    "FEATURES_COL_NUM = 2\n",
    "\n",
    "DATASET_DIR_NAME = \"dataset\\set04\\\\22_pair_combi\"\n",
    "SAMPLE_DATASET_NAME = \"pair_train_own\"\n",
    "ACTUAL_DATASET_NAME = \"pair_combi_fake\"\n",
    "\n",
    "MODEL_DIR_NAME = \"dataset\\set04\\\\02_pair_own\"\n",
    "MODEL_NAME = \"model_\" + \"set04_02_pair_own\"\n",
    "\n",
    "RESULT_DIR_NAME = \"dataset\\set04\\\\22_pair_combi\"\n",
    "RESULT_NAME = \"result_\" + \"set04_22_pair_combi_fake\"\n",
    "\n",
    "DATASET_DIR_PATH = os.path.join(os.getcwd(), DATASET_DIR_NAME)\n",
    "SAMPLE_DATASET_PATH = os.path.join(DATASET_DIR_PATH, SAMPLE_DATASET_NAME + \".csv\")\n",
    "ACTUAL_DATASET_PATH = os.path.join(DATASET_DIR_PATH, ACTUAL_DATASET_NAME + \".csv\")\n",
    "\n",
    "MODEL_DIR_PATH = os.path.join(os.getcwd(), MODEL_DIR_NAME)\n",
    "MODEL_PATH = os.path.join(MODEL_DIR_PATH, MODEL_NAME + \".h5\")\n",
    "\n",
    "RESULT_DIR_PATH = os.path.join(os.getcwd(), RESULT_DIR_NAME)\n",
    "RESULT_PATH = os.path.join(RESULT_DIR_PATH, RESULT_NAME + \".csv\")\n",
    "\n",
    "#################\n",
    "# # sns theme # #\n",
    "#################\n",
    "# sns.set_theme(style=\"darkgrid\") # (dark background with white gridlines)\n",
    "sns.set_theme(style=\"whitegrid\") # (white background with grey gridlines)\n",
    "# sns.set_theme(style=\"dark\") # (dark background with no gridlines)\n",
    "# sns.set_theme(style=\"white\") # (white background with no gridlines)\n",
    "# sns.set_theme(style=\"ticks\") # (white background with axis ticks and no gridlines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "971b4aa5",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset directory exists at: C:\\Users\\andyc\\Documents\\GitHub\\ict4001-cap\\dataset\\set04\\22_pair_combi\n",
      "Model directory exists at: C:\\Users\\andyc\\Documents\\GitHub\\ict4001-cap\\dataset\\set04\\02_pair_own\n",
      "Training dataset exists at: C:\\Users\\andyc\\Documents\\GitHub\\ict4001-cap\\dataset\\set04\\22_pair_combi\\pair_train_own.csv\n"
     ]
    }
   ],
   "source": [
    "if os.path.isdir(DATASET_DIR_PATH) is True:\n",
    "    print(f\"Dataset directory exists at: {DATASET_DIR_PATH}\")\n",
    "else:\n",
    "    try:\n",
    "        os.mkdir(DATASET_DIR_PATH, 666)\n",
    "        print(f\"Dataset directory have been created at: {DATASET_DIR_PATH}\")\n",
    "    except:\n",
    "        print(\"Error: Dataset Directory not created\")\n",
    "        \n",
    "if os.path.isdir(MODEL_DIR_PATH) is True:\n",
    "    print(f\"Model directory exists at: {MODEL_DIR_PATH}\")\n",
    "else:\n",
    "    try:\n",
    "        os.mkdir(MODEL_DIR_PATH, 666)\n",
    "        print(f\"Model directory have been created at: {MODEL_DIR_PATH}\")\n",
    "    except:\n",
    "        print(\"Error: Model Directory not created\")\n",
    "    \n",
    "if os.path.exists(SAMPLE_DATASET_PATH) is True:\n",
    "    print(f\"Training dataset exists at: {SAMPLE_DATASET_PATH}\")\n",
    "else:\n",
    "    try:\n",
    "        SAMPLE_7Z_PATH = os.path.join(DATASET_DIR_PATH, SAMPLE_DATASET_NAME + \".7z\")\n",
    "        with py7zr.SevenZipFile(SAMPLE_7Z_PATH, mode='r') as z:\n",
    "            z.extractall(path=DATASET_DIR_PATH)\n",
    "        print(f\"Decompress 7z file to: {SAMPLE_7Z_PATH}\")\n",
    "    except:\n",
    "        print(\"Error: Failed to decompress 7z file.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0c52cdd5",
   "metadata": {},
   "source": [
    "#### CREATE MODEL\n",
    "- Create base model\n",
    "- Wrap it with KerasClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "f4f633b5",
   "metadata": {},
   "outputs": [],
   "source": [
    "#create base model\n",
    "def create_base_model():\n",
    "    model = Sequential()\n",
    "    model.add(LSTM(units=256, return_sequences=True,\n",
    "             input_shape=(TIMESTEPS,N_FEATURES)))\n",
    "    model.add(Dense(256, activation='sigmoid'))\n",
    "    model.add(Dropout(0.2))\n",
    "    model.add(BatchNormalization())\n",
    "    model.add(LSTM(units=128, return_sequences=True))\n",
    "    model.add(Dense(128, activation='sigmoid'))\n",
    "    model.add(Dropout(0.2))\n",
    "    model.add(BatchNormalization())\n",
    "    model.add(LSTM(units=64, return_sequences=True))\n",
    "    model.add(Dense(64, activation='sigmoid'))\n",
    "    model.add(Dropout(0.2))\n",
    "    model.add(BatchNormalization())\n",
    "    model.add(LSTM(units=32, return_sequences=True))\n",
    "    model.add(Dense(32, activation='sigmoid'))\n",
    "    model.add(Dropout(0.2))\n",
    "\n",
    "    # softmax for multi-class classification\n",
    "    model.add(Flatten())\n",
    "    print(\"There are a total of \" + str(n_classes) + \" classes.\")\n",
    "    model.add(Dense(n_classes, activation='softmax'))\n",
    "    model.compile(loss='categorical_crossentropy', optimizer='adam',\n",
    "                metrics=['accuracy'])\n",
    "    return model\n",
    "\n",
    "# wrap model in KerasClassifier\n",
    "def create_model():\n",
    "    model = KerasClassifier(build_fn=create_base_model, epochs=EPOCH, \n",
    "                            batch_size=BATCH_SIZE)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b2280ddf",
   "metadata": {},
   "outputs": [],
   "source": [
    "# divide data into features X and target (classes) Y\n",
    "def prepare_dataset(df):\n",
    "    X = df.values[:,FEATURES_COL_NUM:].astype(float)\n",
    "    Y = df.values[:,CLASSES_COL_NUM].astype(str)\n",
    "\n",
    "    # convert target Y to labelbinarizer Y for model\n",
    "    # fit_transform is not used to reuse lb\n",
    "    Y = Y.reshape(-1, 1)\n",
    "    lb = LabelBinarizer().fit(Y)\n",
    "    Y = lb.transform(Y)\n",
    "\n",
    "    #################################\n",
    "    # # get all the encoded class # #\n",
    "    #################################\n",
    "    print(\"LabelBinarizer is able to decipher: \")\n",
    "    print(lb.classes_)\n",
    "    print(\"\\n\")\n",
    "\n",
    "    ###########################\n",
    "    # # print X and Y shape # #\n",
    "    ###########################\n",
    "    print(f\"X | Features | Dataset Shape: {X.shape}\")\n",
    "    print(f\"Y | Classes  | Dataset Shape: {Y.shape}\")\n",
    "\n",
    "    return X, Y, lb, lb.classes_.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "0c2cd6d1",
   "metadata": {},
   "outputs": [],
   "source": [
    "def split_shape_dataset(X, Y, SPLIT_RATIO, TIMESTEPS, SEED):\n",
    "\n",
    "    ##############################################################\n",
    "    # # split dataset into train and test set of 0.8/0.2 ratio # #\n",
    "    ##############################################################\n",
    "    X_train, X_test, y_train, y_test = train_test_split(\n",
    "        X, Y, test_size=SPLIT_RATIO, random_state=SEED)\n",
    "\n",
    "    ############################\n",
    "    # # reshaping of dataset # #\n",
    "    ############################\n",
    "\n",
    "    # reshaping the dataset to include LSTM Timesteps\n",
    "    print(X_train.dtype)\n",
    "    X_train = np.asarray(X_train, dtype=np.float32)\n",
    "    print(X_train.dtype)\n",
    "    X_train = np.reshape(X_train, (X_train.shape[0], TIMESTEPS, X_train.shape[1]))\n",
    "    print(X_test.dtype)\n",
    "    X_test = np.asarray(X_test, dtype=np.float32)\n",
    "    print(X_test.dtype)\n",
    "    X_test = np.reshape(X_test, (X_test.shape[0], TIMESTEPS, X_test.shape[1]))\n",
    "\n",
    "    print(f\"X train shape: {X_train.shape}\")\n",
    "    print(f\"Y train shape: {y_train.shape}\")\n",
    "    print(f\"X test shape: {X_test.shape}\")\n",
    "    print(f\"Y test shape: {y_test.shape}\")\n",
    "    \n",
    "    return X_train, X_test, y_train, y_test\n",
    "\n",
    "def reshape_dataset(X, TIMESTEPS):\n",
    "    # reshaping the dataset to include LSTM Timesteps\n",
    "    X = np.asarray(X, dtype=np.float32)\n",
    "    X = np.reshape(X, (X.shape[0], TIMESTEPS, X.shape[1]))\n",
    "    return X"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6244ab5e",
   "metadata": {},
   "source": [
    "#### LOAD DATASET"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "8126e914",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Subject</th>\n",
       "      <th>Password</th>\n",
       "      <th>T2-D|0</th>\n",
       "      <th>T2-I|0+1</th>\n",
       "      <th>T2-PF|0+1</th>\n",
       "      <th>T2-RF|0+1</th>\n",
       "      <th>T2-NG|0+1</th>\n",
       "      <th>T2-D|1</th>\n",
       "      <th>T2-I|1+2</th>\n",
       "      <th>T2-PF|1+2</th>\n",
       "      <th>...</th>\n",
       "      <th>T7-RF|VAR.1</th>\n",
       "      <th>T7-NG|VAR.1</th>\n",
       "      <th>T7-I|SD.1</th>\n",
       "      <th>T7-PF|SD.1</th>\n",
       "      <th>T7-RF|SD.1</th>\n",
       "      <th>T7-NG|SD.1</th>\n",
       "      <th>T8-I|0+7.1</th>\n",
       "      <th>T8-PF|0+7.1</th>\n",
       "      <th>T8-RF|0+7.1</th>\n",
       "      <th>T8-NG|0+7.1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>andy</td>\n",
       "      <td>Co5mical</td>\n",
       "      <td>0.124989</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.124989</td>\n",
       "      <td>0.12364</td>\n",
       "      <td>0.248628</td>\n",
       "      <td>0.12364</td>\n",
       "      <td>0.011015</td>\n",
       "      <td>0.134655</td>\n",
       "      <td>...</td>\n",
       "      <td>0.006047</td>\n",
       "      <td>0.003990</td>\n",
       "      <td>0.047034</td>\n",
       "      <td>0.032436</td>\n",
       "      <td>0.077762</td>\n",
       "      <td>0.063164</td>\n",
       "      <td>0.616391</td>\n",
       "      <td>0.718327</td>\n",
       "      <td>0.697449</td>\n",
       "      <td>0.799384</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>andy</td>\n",
       "      <td>Co5mical</td>\n",
       "      <td>0.124989</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.124989</td>\n",
       "      <td>0.12364</td>\n",
       "      <td>0.248628</td>\n",
       "      <td>0.12364</td>\n",
       "      <td>0.011015</td>\n",
       "      <td>0.134655</td>\n",
       "      <td>...</td>\n",
       "      <td>0.003187</td>\n",
       "      <td>0.002233</td>\n",
       "      <td>0.024718</td>\n",
       "      <td>0.015517</td>\n",
       "      <td>0.056457</td>\n",
       "      <td>0.047256</td>\n",
       "      <td>0.550968</td>\n",
       "      <td>0.641309</td>\n",
       "      <td>0.628636</td>\n",
       "      <td>0.718976</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>andy</td>\n",
       "      <td>Co5mical</td>\n",
       "      <td>0.124989</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.124989</td>\n",
       "      <td>0.12364</td>\n",
       "      <td>0.248628</td>\n",
       "      <td>0.12364</td>\n",
       "      <td>0.011015</td>\n",
       "      <td>0.134655</td>\n",
       "      <td>...</td>\n",
       "      <td>0.004878</td>\n",
       "      <td>0.005244</td>\n",
       "      <td>0.046083</td>\n",
       "      <td>0.048659</td>\n",
       "      <td>0.069841</td>\n",
       "      <td>0.072416</td>\n",
       "      <td>0.562252</td>\n",
       "      <td>0.664963</td>\n",
       "      <td>0.652923</td>\n",
       "      <td>0.755634</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>andy</td>\n",
       "      <td>Co5mical</td>\n",
       "      <td>0.124989</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.124989</td>\n",
       "      <td>0.12364</td>\n",
       "      <td>0.248628</td>\n",
       "      <td>0.12364</td>\n",
       "      <td>0.011015</td>\n",
       "      <td>0.134655</td>\n",
       "      <td>...</td>\n",
       "      <td>0.016760</td>\n",
       "      <td>0.017344</td>\n",
       "      <td>0.101889</td>\n",
       "      <td>0.104127</td>\n",
       "      <td>0.129459</td>\n",
       "      <td>0.131697</td>\n",
       "      <td>0.638064</td>\n",
       "      <td>0.765986</td>\n",
       "      <td>0.722751</td>\n",
       "      <td>0.850673</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>andy</td>\n",
       "      <td>Co5mical</td>\n",
       "      <td>0.124989</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.124989</td>\n",
       "      <td>0.12364</td>\n",
       "      <td>0.248628</td>\n",
       "      <td>0.12364</td>\n",
       "      <td>0.011015</td>\n",
       "      <td>0.134655</td>\n",
       "      <td>...</td>\n",
       "      <td>0.002298</td>\n",
       "      <td>0.003914</td>\n",
       "      <td>0.022615</td>\n",
       "      <td>0.037240</td>\n",
       "      <td>0.047934</td>\n",
       "      <td>0.062559</td>\n",
       "      <td>0.508190</td>\n",
       "      <td>0.630242</td>\n",
       "      <td>0.585637</td>\n",
       "      <td>0.707689</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 442 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  Subject  Password    T2-D|0  T2-I|0+1  T2-PF|0+1  T2-RF|0+1  T2-NG|0+1  \\\n",
       "0    andy  Co5mical  0.124989       0.0   0.124989    0.12364   0.248628   \n",
       "1    andy  Co5mical  0.124989       0.0   0.124989    0.12364   0.248628   \n",
       "2    andy  Co5mical  0.124989       0.0   0.124989    0.12364   0.248628   \n",
       "3    andy  Co5mical  0.124989       0.0   0.124989    0.12364   0.248628   \n",
       "4    andy  Co5mical  0.124989       0.0   0.124989    0.12364   0.248628   \n",
       "\n",
       "    T2-D|1  T2-I|1+2  T2-PF|1+2  ...  T7-RF|VAR.1  T7-NG|VAR.1  T7-I|SD.1  \\\n",
       "0  0.12364  0.011015   0.134655  ...     0.006047     0.003990   0.047034   \n",
       "1  0.12364  0.011015   0.134655  ...     0.003187     0.002233   0.024718   \n",
       "2  0.12364  0.011015   0.134655  ...     0.004878     0.005244   0.046083   \n",
       "3  0.12364  0.011015   0.134655  ...     0.016760     0.017344   0.101889   \n",
       "4  0.12364  0.011015   0.134655  ...     0.002298     0.003914   0.022615   \n",
       "\n",
       "   T7-PF|SD.1  T7-RF|SD.1  T7-NG|SD.1  T8-I|0+7.1  T8-PF|0+7.1  T8-RF|0+7.1  \\\n",
       "0    0.032436    0.077762    0.063164    0.616391     0.718327     0.697449   \n",
       "1    0.015517    0.056457    0.047256    0.550968     0.641309     0.628636   \n",
       "2    0.048659    0.069841    0.072416    0.562252     0.664963     0.652923   \n",
       "3    0.104127    0.129459    0.131697    0.638064     0.765986     0.722751   \n",
       "4    0.037240    0.047934    0.062559    0.508190     0.630242     0.585637   \n",
       "\n",
       "   T8-NG|0+7.1  \n",
       "0     0.799384  \n",
       "1     0.718976  \n",
       "2     0.755634  \n",
       "3     0.850673  \n",
       "4     0.707689  \n",
       "\n",
       "[5 rows x 442 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# load dataset\n",
    "df = pd.read_csv(SAMPLE_DATASET_PATH)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ced9b784",
   "metadata": {},
   "source": [
    "#### PREPARE DATASET\n",
    "- Split data based on X / Features and Y / Classes\n",
    "- Binarize Y into binary labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "8119c175",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LabelBinarizer is able to decipher: \n",
      "['andy' 'azfar' 'chris' 'cy' 'gerald' 'ken' 'qk' 'sz' 'vale' 'ye' 'ys']\n",
      "\n",
      "\n",
      "X | Features | Dataset Shape: (108900, 440)\n",
      "Y | Classes  | Dataset Shape: (108900, 11)\n",
      "Number of Classes: 11\n"
     ]
    }
   ],
   "source": [
    "X, Y, lb, class_list = prepare_dataset(df)\n",
    "n_classes = len(class_list)\n",
    "print(f\"Number of Classes: {n_classes}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "12799eac",
   "metadata": {},
   "source": [
    "# LIVE TESTING\n",
    "> Live test with new dataset to check if model function as it should"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c0074460",
   "metadata": {},
   "source": [
    "#### LOAD MODEL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "1da540d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "##################\n",
    "# # load model # #\n",
    "##################\n",
    "\n",
    "# model = create_model()\n",
    "model = load_model(MODEL_PATH)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4a5f0646",
   "metadata": {},
   "source": [
    "#### LOAD DATA\n",
    "- Import new dataset to verify the model is able to predict accurately"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "29bff51c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Subject</th>\n",
       "      <th>Actual</th>\n",
       "      <th>Password</th>\n",
       "      <th>T2-D|0</th>\n",
       "      <th>T2-I|0+1</th>\n",
       "      <th>T2-PF|0+1</th>\n",
       "      <th>T2-RF|0+1</th>\n",
       "      <th>T2-NG|0+1</th>\n",
       "      <th>T2-D|1</th>\n",
       "      <th>T2-I|1+2</th>\n",
       "      <th>...</th>\n",
       "      <th>T7-RF|VAR.1</th>\n",
       "      <th>T7-NG|VAR.1</th>\n",
       "      <th>T7-I|SD.1</th>\n",
       "      <th>T7-PF|SD.1</th>\n",
       "      <th>T7-RF|SD.1</th>\n",
       "      <th>T7-NG|SD.1</th>\n",
       "      <th>T8-I|0+7.1</th>\n",
       "      <th>T8-PF|0+7.1</th>\n",
       "      <th>T8-RF|0+7.1</th>\n",
       "      <th>T8-NG|0+7.1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>azfar</td>\n",
       "      <td>andy</td>\n",
       "      <td>play4pay</td>\n",
       "      <td>0.103503</td>\n",
       "      <td>-0.047849</td>\n",
       "      <td>0.055654</td>\n",
       "      <td>0.136592</td>\n",
       "      <td>0.240095</td>\n",
       "      <td>0.184441</td>\n",
       "      <td>-0.080906</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000125</td>\n",
       "      <td>0.000131</td>\n",
       "      <td>0.016900</td>\n",
       "      <td>0.005756</td>\n",
       "      <td>0.011193</td>\n",
       "      <td>0.011463</td>\n",
       "      <td>0.744166</td>\n",
       "      <td>0.872098</td>\n",
       "      <td>0.911203</td>\n",
       "      <td>1.039135</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>azfar</td>\n",
       "      <td>andy</td>\n",
       "      <td>play4pay</td>\n",
       "      <td>0.103503</td>\n",
       "      <td>-0.047849</td>\n",
       "      <td>0.055654</td>\n",
       "      <td>0.136592</td>\n",
       "      <td>0.240095</td>\n",
       "      <td>0.184441</td>\n",
       "      <td>-0.080906</td>\n",
       "      <td>...</td>\n",
       "      <td>0.001102</td>\n",
       "      <td>0.000280</td>\n",
       "      <td>0.021553</td>\n",
       "      <td>0.005081</td>\n",
       "      <td>0.033199</td>\n",
       "      <td>0.016728</td>\n",
       "      <td>0.728093</td>\n",
       "      <td>0.848707</td>\n",
       "      <td>0.880007</td>\n",
       "      <td>1.000621</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>azfar</td>\n",
       "      <td>andy</td>\n",
       "      <td>play4pay</td>\n",
       "      <td>0.103503</td>\n",
       "      <td>-0.047849</td>\n",
       "      <td>0.055654</td>\n",
       "      <td>0.136592</td>\n",
       "      <td>0.240095</td>\n",
       "      <td>0.184441</td>\n",
       "      <td>-0.080906</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000771</td>\n",
       "      <td>0.000122</td>\n",
       "      <td>0.033484</td>\n",
       "      <td>0.016773</td>\n",
       "      <td>0.027768</td>\n",
       "      <td>0.011057</td>\n",
       "      <td>0.647457</td>\n",
       "      <td>0.807363</td>\n",
       "      <td>0.807494</td>\n",
       "      <td>0.967400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>azfar</td>\n",
       "      <td>andy</td>\n",
       "      <td>play4pay</td>\n",
       "      <td>0.103503</td>\n",
       "      <td>-0.047849</td>\n",
       "      <td>0.055654</td>\n",
       "      <td>0.136592</td>\n",
       "      <td>0.240095</td>\n",
       "      <td>0.184441</td>\n",
       "      <td>-0.080906</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000289</td>\n",
       "      <td>0.000022</td>\n",
       "      <td>0.017208</td>\n",
       "      <td>0.038917</td>\n",
       "      <td>0.016993</td>\n",
       "      <td>0.004716</td>\n",
       "      <td>0.679519</td>\n",
       "      <td>0.839164</td>\n",
       "      <td>0.864348</td>\n",
       "      <td>1.023993</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>azfar</td>\n",
       "      <td>andy</td>\n",
       "      <td>play4pay</td>\n",
       "      <td>0.127932</td>\n",
       "      <td>-0.056047</td>\n",
       "      <td>0.071885</td>\n",
       "      <td>0.103925</td>\n",
       "      <td>0.231857</td>\n",
       "      <td>0.159972</td>\n",
       "      <td>-0.048710</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000801</td>\n",
       "      <td>0.007317</td>\n",
       "      <td>0.022186</td>\n",
       "      <td>0.079417</td>\n",
       "      <td>0.028307</td>\n",
       "      <td>0.085538</td>\n",
       "      <td>0.832086</td>\n",
       "      <td>0.935589</td>\n",
       "      <td>0.984156</td>\n",
       "      <td>1.087659</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 443 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  Subject Actual  Password    T2-D|0  T2-I|0+1  T2-PF|0+1  T2-RF|0+1  \\\n",
       "0   azfar   andy  play4pay  0.103503 -0.047849   0.055654   0.136592   \n",
       "1   azfar   andy  play4pay  0.103503 -0.047849   0.055654   0.136592   \n",
       "2   azfar   andy  play4pay  0.103503 -0.047849   0.055654   0.136592   \n",
       "3   azfar   andy  play4pay  0.103503 -0.047849   0.055654   0.136592   \n",
       "4   azfar   andy  play4pay  0.127932 -0.056047   0.071885   0.103925   \n",
       "\n",
       "   T2-NG|0+1    T2-D|1  T2-I|1+2  ...  T7-RF|VAR.1  T7-NG|VAR.1  T7-I|SD.1  \\\n",
       "0   0.240095  0.184441 -0.080906  ...     0.000125     0.000131   0.016900   \n",
       "1   0.240095  0.184441 -0.080906  ...     0.001102     0.000280   0.021553   \n",
       "2   0.240095  0.184441 -0.080906  ...     0.000771     0.000122   0.033484   \n",
       "3   0.240095  0.184441 -0.080906  ...     0.000289     0.000022   0.017208   \n",
       "4   0.231857  0.159972 -0.048710  ...     0.000801     0.007317   0.022186   \n",
       "\n",
       "   T7-PF|SD.1  T7-RF|SD.1  T7-NG|SD.1  T8-I|0+7.1  T8-PF|0+7.1  T8-RF|0+7.1  \\\n",
       "0    0.005756    0.011193    0.011463    0.744166     0.872098     0.911203   \n",
       "1    0.005081    0.033199    0.016728    0.728093     0.848707     0.880007   \n",
       "2    0.016773    0.027768    0.011057    0.647457     0.807363     0.807494   \n",
       "3    0.038917    0.016993    0.004716    0.679519     0.839164     0.864348   \n",
       "4    0.079417    0.028307    0.085538    0.832086     0.935589     0.984156   \n",
       "\n",
       "   T8-NG|0+7.1  \n",
       "0     1.039135  \n",
       "1     1.000621  \n",
       "2     0.967400  \n",
       "3     1.023993  \n",
       "4     1.087659  \n",
       "\n",
       "[5 rows x 443 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# import unseen data\n",
    "pred_df = pd.read_csv(ACTUAL_DATASET_PATH)\n",
    "pred_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "837b9992",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAEXCAYAAABGeIg9AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAArDklEQVR4nO3de3hM974/8PfkhghV6Yzatt2qW0qPS7UIp0kjkkgyI3IjRNKoS7pLDkokIohUUVTautftaOkhcqNBaKWcXXGrKjvuRVIRciVyket8f3+o+YmVxCTMjJO8X8/jecx3rVmfz1qT5D1rzay1ZEIIASIioscYGboBIiJ68TAciIhIguFAREQSDAciIpJgOBARkQTDgYiIJBgO1ChVVVVhy5Yt8PDwgJubG1xcXLBs2TKUl5cDAEJDQ7Fp0yad1V+1ahXGjx8vGU9NTYW1tbWmjyedOHECSqUSAPDVV18hISFBMk9+fj66d+8OADh06BAWLlwIADh8+DC++uqr57QG1NSZGLoBIl2IiIhAQUEBtm7dilatWqGkpAQzZ87EnDlzsGzZMp3XHzlyJNavX4/bt2+jffv2mvHo6Gh4e3vDzMzsqcuYOnXqU+ext7eHvb09AODf//43CgoKGt400WMYDtTo3Lx5Ez/88AN++eUXWFhYAADMzc2xYMECnDlzRjJ/TEwMdu7ciYqKChQUFGDixIkYM2YMcnJyEBISgrt37wIAbG1tMW3atFrHH6dQKDBkyBDExcVh8uTJAIDi4mLs378fu3fvxs8//4z169ejvLwc+fn5GDFihGQZoaGh6Nq1K8aPH4+DBw8iKioKLVq0wFtvvaWZJy4uDgcOHMDHH3+MHTt2oKqqCq1atcK5c+cwbNgwjBo1CgCwdu1a3L17F2FhYc9lG1Pjx8NK1OhcuHABXbp00QTDI3K5HI6OjtXGiouLsWvXLnzzzTdISEhAVFSUZs8iOjoaf//73xEfH4/t27cjPT0dhYWFtY4/ydfXF3FxcXh0EYK9e/eif//+aN++PTZv3owlS5YgLi4OO3fuxDfffIP8/Pwa1yc3NxdhYWFYuXIl4uLi0KFDB8k8vXv3ho+PD1xcXDB9+nT4+vpi165dAAC1Wo1du3bBx8en/huTmizuOVCjY2RkBLVardW8LVu2xLp163DkyBGkpaXh0qVLKCkpAQC89957mDRpEm7fvo1BgwZhxowZaNWqVa3jT+rfvz9atGiB48ePw9raGjt37sSMGTMgk8mwbt06HD58GImJibh27RqEEHjw4EGNPZ4+fRrdunVDly5dAACjRo3CihUr6lwvOzs7LFy4EJcuXUJWVhb+/ve/44033tBqmxAB3HOgRqhXr164fv06ioqKqo1nZWVh0qRJKC0t1YzduXMHI0aMwK1bt9CvX79qh3Z69eqFQ4cOYdSoUbh16xa8vb3x22+/1Tpek9GjRyMmJgYXL15ESUkJBg0ahJKSEri7u+P8+fPo0aMHZs2aBRMTE9R2mTOZTFZtmonJ09/TGRsbw8fHBzExMYiNjeVeA9Ub9xyo0WnXrh1UKhXCwsKwaNEiWFhYoKioCBEREWjTpg2aN2+umTc1NRVt27bFxx9/DJlMhrVr1wJ4+G2nqKgoCCEQHBwMe3t7XL58GWlpaUhOTq5x/O2335b04ubmhrVr18Lc3BxjxowBAKSnp6OoqAjTpk2DmZkZdu/ejfLy8lr3dt555x3MmTMHly5dgpWVFeLi4mqcz9jYGJWVlZrH3t7e8PT0hLGxMb744osGb09qmhgO1CjNnz8fa9asgY+PD4yNjVFeXo6hQ4ciKCio2nyDBw9GTEwMhg0bhhYtWqBXr15o27Yt0tPT8cEHHyA0NBRKpRJmZmbo3r07lEolCgoKahyviYWFBRwcHLBnzx6EhIQAALp37473338fzs7OaN26Nf7xj3+gS5cuSE9Pr/FbTG3btsXy5csxc+ZMmJqa4t13362xlrW1NYKCgmBqaoq5c+fC0tISb731Fjp37gxTU9Nn3KLU1Mh4yW6ixik/Px9eXl7Yvn17ta/TEmmDnzkQNULR0dFwcXGBv78/g4EahHsOREQkwT0HIiKSYDgQEZEEw4GIiCQYDkREJNFoznO4e7cYajU/Wyci0oaRkQwvv9yy1umNJhzUasFwICJ6TnhYiYiIJBgOREQkwXAgIiIJhgMREUnoNBxWrVoFV1dXuLq6YunSpQCAlJQUqFQqODo6IioqSjPvxYsX4enpCScnJ8yZM6fapYeJiEi/dBYOKSkp+OWXXxAfH4+EhAScP38eiYmJCAsLw5o1a7Bv3z6kpqbiyJEjAIDg4GDMnTsXBw4cgBAC0dHRumqNiIieQmfhIJfLERoaCjMzM5iamqJz585IS0vDa6+9ho4dO8LExAQqlQpJSUm4desWSktL0adPHwCAh4cHkpKSdNUaERE9hc7Oc+jatavm/2lpadi3bx/8/Pwgl8s14wqFAllZWcjOzq42LpfLkZWVVa96lpYPbyYvKqsgMzF+xu5rV9fy1ZXlMDKR3qzlealt+VWV5TDWYd3all9ZVQ4TY93VratGeVUFzIx1dwOb2pZfXlUJM2Pdnh5UW43yqiqYGevuZ7u25ZdXqWFmrNuPJ2urUVUlYGws01nd2pavrhQwMtFd3dqWLyrVkJnodltrW0PnJ8FdvXoVgYGBCAkJgYmJCW7cuFFt+pP3x318vD7y8oqgVgvI5a2Qs3bbM/VcF/k/xyInp7DmafJWuLbSTWe1OwftrrG2XN4KiZuddVZX+eH+WutGfe+ks7oAMH3MgVprO+/+SGd197utq7Wua+x6ndUFgL2egbXWHh6zW2d193i51VrXOzZVZ3UBYJfnW7XWjo7N1VndkZ6v1Fr30pr6vUGtD6uP29VaN+vLkzqrCwDtpvVHTk4hjIxkmjfVNdFpRJ0+fRoBAQGYMWMG3N3d0a5dO+Tm/v8XOjs7GwqFQjKek5MDhUKhy9aIiKgOOguH27dvY/LkyVi+fDlcXV0BAL1798aNGzeQnp6OqqoqJCYmwsbGBh06dECzZs1w+vRpAEBCQgJsbGx01RoRET2Fzg4rbdq0CWVlZViyZIlmzMfHB0uWLEFQUBDKyspga2uLYcOGAQCWL1+O8PBwFBcXo0ePHvD399dVa0RE9BQ6C4fw8HCEh4fXOG3Pnj2SMSsrK8TExOiqHSIiqgeeIU1ERBIMByIikmA4EBGRBMOBiIgkGA5ERCTBcCAiIgmGAxERSTAciIhIguFAREQSDAciIpJgOBARkQTDgYiIJBgOREQkwXAgIiIJhgMREUkwHIiISEJnN/t5pKioCD4+Pli3bh2uXbuGFStWaKZlZWWhd+/eWL9+PVatWoXY2Fi0bt0aADBy5Ej4+vrquj0iIqqBTsPh7NmzCA8PR1paGgDA1tYWtra2AICcnByMHj0as2fPBgCkpqZixYoV6Nu3ry5bIiIiLej0sFJ0dDTmz58PhUIhmbZ06VL4+Pjg9ddfB/AwHDZs2ACVSoXIyEiUlZXpsjUiIqqDTsPhs88+wzvvvCMZT0tLw8mTJ+Hv7w8AKC4uxptvvomQkBDEx8fj/v37WLNmjS5bIyKiOuj8M4ea7Ny5E2PGjIGZmRkAoGXLltiwYYNm+ocffoiwsDBMnz5d62VaWlo89z5rI5e30lutF6U217lp1OY6N/662tY2SDgcOnQImzZt0jzOzMxESkoKvLy8AABCCJiY1K+1vLwiqNVCLxs8J6ewxnFD1W7MdQ1Zm+usv7qGrN3U6j6qbWQkq/NNtd6/ypqfn4/S0lJ07NhRM9a8eXMsW7YMN2/ehBAC27dvh4ODg75bIyKiv+h9zyEjIwOvvvpqtbG2bdsiMjIS//znP1FRUYG3334b48aN03drRET0F72EQ3Jysub/vXr1QnR0tGQeJycnODk56aMdIiJ6Cp4hTUREEgwHIiKSYDgQEZEEw4GIiCQYDkREJMFwICIiCYYDERFJMByIiEiC4UBERBIMByIikmA4EBGRBMOBiIgkGA5ERCTBcCAiIgmGAxERSTAciIhIguFAREQSOg+HoqIiKJVKZGRkAABmz54NR0dHuLm5wc3NDT/++CMAICUlBSqVCo6OjoiKitJ1W0REVAed3ib07NmzCA8PR1pammYsNTUV27Ztg0Kh0IyVlpYiLCwM3333Hdq3b4/AwEAcOXIEtra2umyPiIhqodM9h+joaMyfP18TBCUlJcjMzMTcuXOhUqnw9ddfQ61W49y5c3jttdfQsWNHmJiYQKVSISkpSZetERFRHXS65/DZZ59Ve5yXl4eBAwciMjIS5ubmCAwMRExMDMzNzSGXyzXzKRQKZGVl6bI1IiKqg07D4UkdO3bE6tWrNY/9/PyQkJCAYcOGSeaVyWT1WralpcUz96ctubyV3mq9KLW5zk2jNte58dfVtrZew+Hy5ctIS0uDk5MTAEAIARMTE7Rr1w65ubma+bKzs6t9JqGNvLwiqNVCLxs8J6ewxnFD1W7MdQ1Zm+usv7qGrN3U6j6qbWQkq/NNtV6/yiqEwKJFi1BQUICKigrs3LkTDg4O6N27N27cuIH09HRUVVUhMTERNjY2+myNiIgeo9c9BysrK0yaNAmjR49GZWUlHB0doVQqAQBLlixBUFAQysrKYGtrW+OhJiIi0g+9hENycrLm/76+vvD19ZXMY21tjT179uijHSIiegqeIU1ERBIMByIikmA4EBGRBMOBiIgkGA5ERCTBcCAiIgmGAxERSTAciIhIguFAREQSDAciIpJgOBARkQTDgYiIJBgOREQkwXAgIiIJhgMREUkwHIiISILhQEREEjoPh6KiIiiVSmRkZAAAdu7cCaVSCZVKhdmzZ6O8vBwAsGrVKtjZ2cHNzQ1ubm7Yvn27rlsjIqJa6PQ2oWfPnkV4eDjS0tIAADdu3MCmTZsQFxeHli1bIjQ0FN9//z0CAgKQmpqKFStWoG/fvrpsiYiItKDTPYfo6GjMnz8fCoUCAGBmZoaIiAhYWFhAJpOhW7duyMzMBACkpqZiw4YNUKlUiIyMRFlZmS5bIyKiOug0HD777DO88847mscdOnTAoEGDAAD5+fnYvn077O3tUVxcjDfffBMhISGIj4/H/fv3sWbNGl22RkREddDpYaXaZGVlYcKECfD09MSAAQMAABs2bNBM//DDDxEWFobp06drvUxLS4vn3mdt5PJWeqv1otTmOjeN2lznxl9X29p6D4dr165h4sSJGDt2LD788EMAQGZmJlJSUuDl5QUAEELAxKR+reXlFUGtFnrZ4Dk5hTWOG6p2Y65ryNpcZ/3VNWTtplb3UW0jI1mdb6r1+lXWoqIijB8/HlOnTtUEAwA0b94cy5Ytw82bNyGEwPbt2+Hg4KDP1oiI6DF63XOIiYlBbm4uNm/ejM2bNwMAhgwZgqlTpyIyMhL//Oc/UVFRgbfffhvjxo3TZ2tERPQYvYRDcnIyACAgIAABAQE1zuPk5AQnJyd9tENERE/BM6SJiEiC4UBERBIMByIiktAqHLKysiRjf/zxx3NvhoiIXgx1hsO9e/dw7949TJw4EQUFBZrHubm5+Pjjj/XVIxER6Vmd31aaMWMGjh49CgCaM5kBwMTEBEOHDtVtZ0REZDB1hsOmTZsAALNnz8bixYv10hARERmeVuc5LF68GLdu3UJBQQGEEJrxnj176qwxIiIyHK3CYfny5fjuu+9gaWmpGZPJZDh06JDOGiMiIsPRKhz27duHgwcPol27drruh4iIXgBafZW1ffv2DAYioiZEqz0Ha2trLF26FPb29mjevLlmnJ85EBE1TlqFQ1xcHAAgKSlJM8bPHIiIGi+twuHRVVWJiKhp0CoctmzZUuM477lARNQ4aRUOV65c0fy/vLwcp0+frnbGNBERNS5anwT3uPz8fMyaNUsnDRERkeE16JLdbdu2xa1bt546X1FREZRKJTIyMgAAKSkpUKlUcHR0RFRUlGa+ixcvwtPTE05OTpgzZw4qKysb0hYRET0n9f7MQQiB1NTUamdL1+Ts2bMIDw9HWloaAKC0tBRhYWH47rvv0L59ewQGBuLIkSOwtbVFcHAwFi5ciD59+iAsLAzR0dEYM2ZMw9eKiIieiVZ7DleuXNH8u3r1Ktq3b4/ly5fX+Zzo6GjMnz8fCoUCAHDu3Dm89tpr6NixI0xMTKBSqZCUlIRbt26htLQUffr0AQB4eHhU+8osERHpX70+c7h16xYqKyvx2muvPfU5n332WbXH2dnZkMvlmscKhQJZWVmScblcXuPNhYiISH+0Cof09HR8/PHHyM7Ohlqtxssvv4z169ejc+fOWhd6/Gquj8hkslrH68vS0qLez2koubyV3mq9KLW5zk2jNte58dfVtrZW4RAZGYkJEybA3d0dABAbG4sFCxbg22+/1bqZdu3aITc3V/M4OzsbCoVCMp6Tk6M5FFUfeXlFUKuFXjZ4Tk5hjeOGqt2Y6xqyNtdZf3UNWbup1X1U28hIVuebaq0+c8jLy9MEAwB4enri7t279Wqmd+/euHHjBtLT01FVVYXExETY2NigQ4cOaNasGU6fPg0ASEhIgI2NTb2WTUREz5dWew5VVVW4d+8e2rRpA+DheQ711axZMyxZsgRBQUEoKyuDra0thg0bBuDh/SLCw8NRXFyMHj16wN/fv97LJyKi50ercBg7dixGjRoFZ2dnAMD+/fvxwQcfaFXg8esyWVtbY8+ePZJ5rKysEBMTo9XyiIhI97Q6rGRrawsAqKiowPXr15GVlQUHBwedNkZERIaj1Z5DaGgofH194e/vj7KyMvzP//wPwsLCsGHDBl33R0REBqDVnsPdu3c1nwM0a9YMAQEByMnJ0WljRERkOFqFQ1VVVbUT03Jzc2s8P4GIiBoHrQ4rBQQEYMSIEXjvvfcgk8mQkpLCq7ISETViWoWDl5cX3nrrLRw/fhzGxsYYP348unXrpuveiIjIQLQKB+Dh102trKx02QsREb0gGnQ/ByIiatwYDkREJMFwICIiCYYDERFJMByIiEiC4UBERBIMByIikmA4EBGRBMOBiIgkGA5ERCSh9eUznpddu3Zh27ZtmscZGRlwc3PDgwcPcPr0abRo0QIAMGXKFN5QiIjIQPQeDt7e3vD29gYAXL16FZMnT8aUKVPwwQcfYNu2bVAoFPpuiYiInmDQw0oRERGYPn06mjdvjszMTMydOxcqlQpff/011Gq1IVsjImrSDBYOKSkpKC0thbOzM/Ly8jBw4EAsWrQI0dHR+PXXXxETE2Oo1oiImjy9H1Z6ZMeOHRg3bhwAoGPHjli9erVmmp+fHxISEjBy5Eitl2dpafHce6yNXN5Kb7VelNpc56ZRm+vc+OtqW9sg4VBeXo5Tp05hyZIlAIDLly8jLS0NTk5OAAAhBExM6tdaXl4R1Gqhlw2ek1NY47ihajfmuoaszXXWX11D1m5qdR/VNjKS1fmm2iCHlS5fvozXX38d5ubmAB6GwaJFi1BQUICKigrs3LmT31QiIjIgg+w53Lx5E6+++qrmsZWVFSZNmoTRo0ejsrISjo6OUCqVhmiNiIhgoHBwcXGBi4tLtTFfX1/4+voaoh0iInoCz5AmIiIJhgMREUkwHIiISILhQEREEgwHIiKSYDgQEZEEw4GIiCQYDkREJMFwICIiCYYDERFJMByIiEiC4UBERBIMByIikmA4EBGRBMOBiIgkGA5ERCTBcCAiIgmD3AnO398feXl5MDF5WD4yMhJ//vkn1q5di4qKCgQEBPCucEREBqT3cBBC4Pr16zh8+LAmHLKysjB9+nTExcXBzMwMPj4+GDBgALp06aLv9oiICAYIh+vXr0Mmk2HixInIy8vDyJEj0bJlSwwcOBBt2rQBADg5OSEpKQlTpkzRd3tERAQDhMP9+/dhbW2NiIgIlJaWwt/fH87OzpDL5Zp5FAoFzp07V6/lWlpaPO9WayWXt9JbrRelNte5adTmOjf+utrW1ns49O3bF3379gUAmJubw8vLC4sXL8ZHH31UbT6ZTFav5eblFUGtFnrZ4Dk5hTWOG6p2Y65ryNpcZ/3VNWTtplb3UW0jI1mdb6r1/m2lX3/9FceOHdM8FkKgQ4cOyM3N1YxlZ2dDoVDouzUiIvqL3sOhsLAQS5cuRVlZGYqKihAfH49ly5bh2LFjyM/Px4MHD3Dw4EHY2NjouzUiIvqL3g8r2dnZ4ezZsxgxYgTUajXGjBmDfv36Yfr06fD390dFRQW8vLzQq1cvfbdGRER/Mch5DtOmTcO0adOqjalUKqhUKkO0Q0RET+AZ0kREJMFwICIiCYYDERFJMByIiEiC4UBERBIMByIikmA4EBGRBMOBiIgkGA5ERCTBcCAiIgmGAxERSTAciIhIguFAREQSDAciIpJgOBARkQTDgYiIJBgOREQkYZA7wa1atQr79+8HANja2mLWrFmYPXs2Tp8+jRYtWgAApkyZAgcHB0O0R0TU5Ok9HFJSUvDLL78gPj4eMpkMEyZMwI8//ojU1FRs27YNCoVC3y0REdET9H5YSS6XIzQ0FGZmZjA1NUXnzp2RmZmJzMxMzJ07FyqVCl9//TXUarW+WyMior/ofc+ha9eumv+npaVh3759+P7773Hy5ElERkbC3NwcgYGBiImJwciRI7VerqWlhS7arZFc3kpvtV6U2lznplGb69z462pb2yCfOQDA1atXERgYiJCQELzxxhtYvXq1Zpqfnx8SEhLqFQ55eUVQq4VeNnhOTmGN44aq3ZjrGrI211l/dQ1Zu6nVfVTbyEhW55tqg3xb6fTp0wgICMCMGTPg7u6Oy5cv48CBA5rpQgiYmBgst4iImjy9h8Pt27cxefJkLF++HK6urgAehsGiRYtQUFCAiooK7Ny5k99UIiIyIL2/Pd+0aRPKysqwZMkSzZiPjw8mTZqE0aNHo7KyEo6OjlAqlfpujYiI/qL3cAgPD0d4eHiN03x9ffXcDRER1YRnSBMRkQTDgYiIJBgOREQkwXAgIiIJhgMREUkwHIiISILhQEREEgwHIiKSYDgQEZEEw4GIiCQYDkREJMFwICIiCYYDERFJMByIiEiC4UBERBIMByIikmA4EBGRxAsVDj/88ANcXFzg4OCA7du3G7odIqImS++3Ca1NVlYWoqKiEBcXBzMzM/j4+GDAgAHo0qWLoVsjImpyXphwSElJwcCBA9GmTRsAgJOTE5KSkjBlyhStnm9kJPv//2/VUhct1ljrSSatFAap3cLCMHVbt2yn07p11Va0sDRMXXMLndatu3YLg9SVm5vqtG5dtc3NdXuAo7a6pq0MU9eotZlO6z6qXdffMQCQCSGEzjvRwvr161FSUoLp06cDAHbt2oVz587h008/NXBnRERNzwvzmUNNGSWT1Z1sRESkGy9MOLRr1w65ubmax9nZ2VAodHuohIiIavbChMOgQYNw7Ngx5Ofn48GDBzh48CBsbGwM3RYRUZP0wnwg3a5dO0yfPh3+/v6oqKiAl5cXevXqZei2iIiapBfmA2kiInpxvDCHlYiI6MXBcCAiIgmGAxERSTAciIhIokmHQ2ZmJoYNGwYPDw8UFRXppIafnx9OnDjx1PkmTpyIrKwsnfTwf0FGRgaGDBlS47Tu3bs/9fknTpyAn5/f827rmdW1Xo1ZaGgo4uLiDN1GkxMXF4fQ0NDnsqwX5qushnDy5En07NkTX3zxhaFbwYYNGwzdAhGRRqMLh8rKSkRERODq1avIzc1Fp06doFQqsWXLFgCAWq3GlStXsGvXLnz55ZcoKSnBvHnzMHnyZISFhaGwsBA5OTlwdXXFzJkzERcXh/j4eNy7dw92dnb45JNPaq0thMDy5cvx008/wdjYGKNGjQLw8DpRn3/+OQoKCjBnzhwMGTIEoaGhuHfvHtLT0xEcHIyFCxfi22+/RVFREebNm4fKyko0a9YMixcvxuuvv97g7VFTT1u2bEFycjKMjIxw8uRJfPPNN9i4cWO9lvvFF1/gwIEDePnllyGXyzFkyBAYGRlh69atUKvV6NmzJ+bPn49mzZph4MCB6NmzJ3JzcxETE4MFCxZUe31WrVpVbdkZGRkIDg5GSUkJevfuXe913rp1K3766SfMmTMHn3/+Oe7du4fmzZtj7ty56NGjB0JDQ2FhYYHz588jKysLkydPhqenZ73rPEkIgSVLluDw4cNQKBRo27YtbG1tNdMPHDiA1atX47//+7/Rtm3bZ673uDt37mDmzJkoKSmBkZERwsPDMX/+fM30GzduYOrUqRg/fnyDa0yZMgVKpRLDhg0DAHh4eCA0NBRRUVEoLS1FQUEBgoOD4ezsXO15CQkJNf5cPIvg4GC88847mt8xPz8/zJw5E19++aXk9X7eaqr93nvvITExEUZGRujVqxciIyMbvPya/obNnj0bM2bMQNeuXXHx4kVYWlriq6++Qps2bZCQkIC1a9fCwsICHTp0gLm5OY4dO4avvvoKO3bsAADEx8fj999/x4IFC7RvRDQyJ0+eFBEREUIIIaqqqsTYsWNFUlKSZvqnn36qmR4bGytCQkKEEEJs3LhRxMXFCSGEuH//vujbt6/Iy8sTsbGxwsHBQVRUVDy19r59+4SPj48oKysTRUVFYvjw4cLJyUksWLBACCFEcnKy8PDwEEIIERISoqkthBB2dnbi5s2bIjQ0VOzbt08IIcTevXtFfHz8M22PmnpydHQUKSkpQgghQkNDxd69e+u1zEOHDonRo0eLsrIyce/ePWFnZye2bdsmRo8eLUpLS4UQQixfvlysXr1aCCFEt27dxPHjx4UQtb8+N2/eFHZ2dkIIISZNmiSio6OFEELEx8eLbt26PbWn48ePi7Fjx4qYmBgxZswYUVxcLEaNGiXOnz8vhBDi6tWrwtHRUQjxcNtPnjxZqNVqcenSJdG/f/96rX9t9u7dK3x9fUV5ebnIyckRgwYNErGxscLOzk7861//EiqVSuTk5DyXWk9auXKl2LBhgxDi4bbYuHGjZtqBAweEh4eH5rVpqIMHD4qgoCAhhBA3btwQLi4uIigoSPzxxx9CCCFSUlKEUqkUQjzcxrGxseLKlSu1/lw8i2PHjokxY8YIIYTIyMgQLi4utb7ez1tNtQcMGCDKy8tFVVWVmDdvnrhz506Dl1/T78imTZtE9+7dNes3ZcoU8e2334o7d+6IwYMHi5ycHFFRUSE+/PBDERISItRqtRgyZIhIT08XQgjh5+cnfv/993r10ej2HN599120adMG27dvx/Xr15GWloaSkhIAQExMDC5cuICtW7dKnjd+/HgcP34cmzZtwtWrV1FRUYEHDx4AAHr06AETk6dvqlOnTsHZ2RlmZmYwMzPD7t274efnh6FDhwIAunTpgrt372rmr+kMcFtbW0RGRuJf//oX7Ozs4OTk1KDtUFdP8fHx2LNnD/r06YPjx4/X790EHl5e/fFlDh06FEIIpKenY+TIkQCAioqKau/aHu0B1PX6PHLy5EnNob7hw4cjPDxcq76uXLmCefPmYcWKFRBCIDU1FbNnz9ZMLykp0Wz/wYMHQyaToVu3brh371691r82p06dgqOjI0xNTfHKK69oPmu4e/cugoKCEBQUhFdeeeW51HqStbU1goKCcPHiRdja2mLs2LEAgEuXLuHzzz/Hd99998zv1m1tbfHpp5+iqKgIiYmJUKlUGDduHH7++WckJSXh7NmzKC4urvacEydO1Plz0VADBgzA3LlzkZGRgd27d8PZ2Rnr1q2r8fV++eWXn7leXbXd3Nxw5swZeHl5wd7eHr6+vmjXruGXsq/td8TS0lKz7bp27YqCggKcOXMGffv21fxcqVQqHD9+HDKZDO7u7tizZw88PDyQl5dX773wRhcOhw4dwtdffw1/f394eHjg7t27EELgt99+w7p167Bjxw6YmkqvTb9kyRLcvHkTSqUSQ4cORUpKiuZKsc2bN9eq9pMBkpGRgZKSEhgbGwOQXmW2puUOGzYMffv2xc8//4ytW7fiyJEjWLhwoVb1te3JyckJUVFROHDgAGxsbGBmVr/rxxsZGUGtVlcbq6qqgrOzs+YPeXFxMaqqqjTTH61rba/Pkx6NyWQyra/O27JlSyxatAiLFi3CoEGDNGH4yJ07dzT3C3n0h/J5Xvm3efPm1dbl0baXyWRYvXo1Zs6cCVdX12f6w1Gbfv36Ye/evTh8+DD27duH+Ph4fPHFF/iv//ovLFq0CH/729+euYaZmRnef/99JCcnIykpCevXr8eYMWMwYMAADBgwANbW1pg5c2a15zzt56KhZDIZRowYgb179yIpKQnr1q3D5s2ba329n6cna2/cuBETJ07E77//jv/93//FhAkTsHz5cvTv379By6/pd+Rvf/tbtXCXyWQQQkAmk1X7XXz8993d3R0TJkyAmZkZ3Nzc6t1Ho/u20rFjx+Ds7AxPT0+88sorOHXqFDIzMzFz5kysWLGi1nduR48exfjx4+Hs7Izbt28jKytL8gfwad599138+OOPmr2OCRMm1PsbSNOmTcO5c+fg4+ODqVOn4sKFC/V6vrY92djYYMWKFfDw8Kj3MgcPHoyDBw+ivLwcRUVFOHz4MAoLC/Hjjz8iLy8PQghERETUuIdW0+vz5B+LQYMGYc+ePQCgqaONDh06wN7eHv3798fq1avx+uuva/5YHD16FL6+vvVe1/oYPHgw9u/fj/LychQWFuLIkSMAgDZt2sDa2hqjR49+pqCvy9KlS7F79264u7tj3rx5uHDhAqZOnQo/Pz8MGDDgudVxc3PDli1b8NJLL6Fly5ZIS0vD1KlTYWtri6NHj0peywEDBmj1c9EQHh4e2LFjB1599VV06NBBr6/347VNTU3h7OyMbt26YerUqRg8eDAuX77c4GVr8zvySL9+/XD27FnN36t9+/ZppnXo0AGvvvoqduzY0aBwaHR7Dt7e3pg5cyaSkpJgZmaGPn364NatWyguLkZERIRmIwcGBlZ7XmBgIGbNmoXWrVvD0tISb731FjIyMupV28HBAampqfDw8IBarYa/vz/2799fr2V89NFHmDNnDtasWQNjY+Nn/lpaTT116tQJrq6u+O233xr0ga+trS1+++03uLu746WXXoJCocAbb7yBKVOm4IMPPoBarcabb76JSZMmSZ5b0+vz5HaeN28egoODsWPHDvzHf/wHWras3539Zs2aBaVSiZUrVyIqKgobN26EqakpoqKidHqPkP/8z//EhQsX4O7ujtatW0Mul1ebPmnSJAwfPhyHDh2Cvb39c63t5+eHGTNmID4+HsbGxpg9ezbCw8Px4MEDxMbGQgiBQYMGISQk5Jnq9OvXD4WFhfDx8UGbNm3g7e0NV1dXWFhYoE+fPigtLa12mNDKykqrn4uGaN++Pdq3bw93d3cAwLJlyxAREaGX1/vx2m3btoWPjw+8vLzQokWLaj01RE2/I7V9Hf6VV15BeHg4AgIC0KJFC8ltlV1cXHDw4MEG7a3ywntNUFVVFaKiomBpaYlx48bV+/lnzpxBWloa3N3dUVFRgVGjRmHRokWwsrLSQbf/d4WGhqJ///4N2jujugkhkJ2dDT8/PyQmJtb70Oj/1dr1UVlZiVmzZmHYsGFwdHSs9/Mb3WElejpPT0+cP38eo0ePbtDzO3XqhMTERAwfPhweHh5wdXVlMJBeHThwAG5ubvjkk0/0/sfZkLW1JYTAe++9B5lMpvlCTH1xz4GIiCS450BERBIMByIikmA4EBGRBMOBqAa///47/Pz8oFKpoFQqMWHCBFy9erXO56xcubLWa+pMnDgRf/zxR4N6uXnzJoKCghr0XKKGanTnORA9q/LycgQGBmLz5s3o2bMnAGD37t2YOHEiDh06pDnjvT6e5aq7mZmZuHHjRoOfT9QQ3HMgesKDBw9QWFhY7WSu4cOHY+7cuTh27BiUSqVm/MSJE9UeX7t2Db6+vlAqlQgODtbcJ2TIkCH497//DQBITk6Gt7c3RowYAR8fH5w5cwbAw++lL168GE5OTnBxccGcOXNQXl6O8PBw/Pnnn890RVWi+mI4ED3hpZdeQnBwMCZMmAB7e3sEBwcjNjYWgwYNqvG6XI/7888/sXLlSvzwww8QQmDt2rXVpqelpSEqKgrffPMNEhIS8OmnnyIoKAglJSX4/vvvcf78eezevRuJiYkoLi7Gvn37sHDhQvzjH//Apk2bdLnaRNXwsBJRDcaNGwdvb2+cOnUKp06dwoYNG7BhwwYEBwfX+TwHBwfNvRo8PT2xdOnSatOPHj2K7OxsBAQEaMZkMhn+/PNPpKSkwM3NTXORwi+//BIAtLqTINHzxnAgesLp06dx5swZTJgwAXZ2dpqbPKlUKly6dKnalVcrKiqqPffxzyOEEJKr4qrValhbW2v+8APA7du3oVAoJPPm5ubW++KPRM8LDysRPaFt27ZYu3Ytfv31V81YTk4OHjx4gKFDhyIzM1NzldGffvqp2nOTk5NRUFCAqqoq7Ny5EzY2NtWmDxw4EEePHsW1a9cAAEeOHMHw4cNRVlYGa2trJCYmory8HGq1GhEREdi7dy+MjY0lIUSka9xzIHpCp06dsHr1akRFReHOnTto1qwZWrVqhcjISFhZWcHHxweenp6Qy+V4//33qz23c+fOCAwMxP3799GvXz/JFUi7du2KyMhIfPLJJ5o9i7Vr18Lc3Bw+Pj64desWPDw8IIRA//794efnh+LiYhgbG8PLywu7du3S6ZVliR7htZWIdEwIgYEDB+L7779H586dDd0OkVZ4WIlIh7KysmBra4uePXuiU6dOhm6HSGvccyAiIgnuORARkQTDgYiIJBgOREQkwXAgIiIJhgMREUkwHIiISOL/AV/9WMU5sf0LAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "################################\n",
    "# # check for class validity # #\n",
    "################################\n",
    "sns.set_theme(style=\"darkgrid\")\n",
    "ax = sns.countplot(x=\"Subject\", data=pred_df).set_title(\"Class Validity\")\n",
    "\n",
    "# remove missing values if available\n",
    "pred_df = pred_df.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "5b061d06",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['andy', 'azfar', 'chris', 'cy', 'gerald', 'ken', 'qk', 'sz', 'vale', 'ye', 'ys']\n"
     ]
    }
   ],
   "source": [
    "pred_dataset = pred_df.values\n",
    "actual_np = pred_dataset[:,CLASSES_COL_NUM]\n",
    "\n",
    "# # divide data into features X\n",
    "# pred_row = pred_dataset[:,3:].astype(float)\n",
    "\n",
    "FEATURES_COL_NUM = 3\n",
    "\n",
    "########################\n",
    "# # predict all rows # #\n",
    "########################\n",
    "pred_row=pred_df.iloc[:,FEATURES_COL_NUM:]\n",
    "\n",
    "#################################\n",
    "# # predict more than one row # #\n",
    "#################################\n",
    "\n",
    "# pred_row=pred_df.iloc[46:54,FEATURES_COL_NUM:]\n",
    "# print(pred_row)\n",
    "\n",
    "############################\n",
    "# # predict a single row # #\n",
    "############################\n",
    "\n",
    "# pred_row=pred_df.iloc[11:12,FEATURES_COL_NUM:]\n",
    "\n",
    "##################\n",
    "# # shape data # #\n",
    "##################\n",
    "pred_row = pred_row.values.tolist()\n",
    "pred_arr = np.asarray(pred_row, dtype=np.float32)\n",
    "pred_arr = np.reshape(pred_arr, (pred_arr.shape[0], TIMESTEPS, pred_arr.shape[1]))\n",
    "\n",
    "print(class_list)\n",
    "Y = np.asarray(class_list)\n",
    "Y = Y.reshape(-1, 1)\n",
    "lb = LabelBinarizer().fit(Y)\n",
    "Y = lb.transform(Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "ad543869",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of the Model is: 568/2200 (25.818181818181817%)\n"
     ]
    }
   ],
   "source": [
    "####################################\n",
    "# # get prediction and its label # #\n",
    "####################################\n",
    "\n",
    "pred_proba = model.predict(pred_arr)\n",
    "\n",
    "# pred_proba = np.sum(pred_proba, axis=0)\n",
    "# pred_proba = np.reshape(pred_proba, (1, 9))\n",
    "\n",
    "pred = lb.inverse_transform(pred_proba)\n",
    "acc = np.max(pred_proba, axis=1)\n",
    "\n",
    "match_list = []\n",
    "\n",
    "for _ in range(actual_np.size):\n",
    "    if pred[_] == actual_np[_]:\n",
    "        match_list.append(\"match\")\n",
    "    else:\n",
    "        match_list.append(\"nope\")\n",
    "\n",
    "match_count = match_list.count(\"match\")\n",
    "total_count = len(match_list)\n",
    "\n",
    "match_header = str(match_count) + \"/\" + str(total_count) + \" (\" + str(match_count/total_count*100)+ \"%)\"\n",
    "match_np = np.array(match_list)\n",
    "\n",
    "exported_output = np.column_stack((actual_np, pred))\n",
    "exported_output = np.column_stack((exported_output, acc))\n",
    "exported_output = np.column_stack((exported_output, match_np))\n",
    "\n",
    "pd.set_option('display.max_rows', None)\n",
    "pd.set_option('display.max_columns', None)\n",
    "\n",
    "df = pd.DataFrame(data=exported_output, index=None, columns=['Actual', 'Prediction', 'Accuracy', match_header])\n",
    "# print(df)\n",
    "df.to_csv(RESULT_PATH)\n",
    "\n",
    "print(\"Accuracy of the Model is: \" + str(match_header))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "25dda603",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Execution time: 00:00:24\n"
     ]
    }
   ],
   "source": [
    "# print elapsed time\n",
    "elapsed_time = time.time() - start_time\n",
    "print('Execution time:', time.strftime(\"%H:%M:%S\", time.gmtime(elapsed_time)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "798d2798",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3ee8af20",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a3741765",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.5"
  },
  "vscode": {
   "interpreter": {
    "hash": "bab3aeda7df4185df76f985d1c9cc9a70e60e13cb7e8d1e31aa28fc24afc1062"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}