3_1_individual_modeling.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 50
    },
    "colab_type": "code",
    "id": "-hIM5G4oM5jh",
    "outputId": "a80cde99-d4e2-47fa-a3a5-15926e74f175"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/statsmodels/tools/_testing.py:19: FutureWarning: pandas.util.testing is deprecated. Use the functions in the public API at pandas.testing instead.\n",
      "  import pandas.util.testing as tm\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from tqdm.notebook import tqdm\n",
    "import re\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 98
    },
    "colab_type": "code",
    "id": "Rg4Z4VFdO9AW",
    "outputId": "b8ae2761-f7e9-4aef-9d97-b3e1ba1de777"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Archive:  /content/drive/My Drive/AAIC Course/Personal case study 2  - Google Quest/google-quest-challenge.zip\n",
      "  inflating: sample_submission.csv   \n",
      "  inflating: test.csv                \n",
      "  inflating: train.csv               \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "((6079, 41), (476, 11))"
      ]
     },
     "execution_count": 3,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "!unzip '/content/drive/My Drive/AAIC Course/Personal case study 2  - Google Quest/google-quest-challenge.zip'\n",
    "df_train = pd.read_csv('train.csv')\n",
    "df_test = pd.read_csv('test.csv')\n",
    "df_train.shape, df_test.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "TLEozz70PP46"
   },
   "source": [
    "# 3. Data Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 50
    },
    "colab_type": "code",
    "id": "BqbK7HdKPWhw",
    "outputId": "43f3da49-a1a9-488a-88f0-e5fadb2d4ccc"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
      "[nltk_data]   Unzipping corpora/stopwords.zip.\n"
     ]
    }
   ],
   "source": [
    "# defining a function to remove stop_words\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "nltk.download('stopwords')\n",
    "stop_words = set(stopwords.words('english'))\n",
    "stop_words.remove('no'); stop_words.remove('not'); stop_words.remove('nor')\n",
    " \n",
    "def stopwrd_removal(sent):\n",
    "  lst = []\n",
    "  for wrd in sent.split():\n",
    "    if wrd not in stop_words:\n",
    "      lst.append(wrd)\n",
    "  return \" \".join(lst)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "hoVwES0MPefG"
   },
   "outputs": [],
   "source": [
    "def text_preprocessor(column, remove_stopwords = False, remove_specialchar = False):\n",
    "  \"\"\"pass any column with Text in it from df_train | Note: returns nothing makes inplace changes in df_train\"\"\"\n",
    "  # 1. remove html tags, html urls, replace html comparison operators\n",
    "  # text = df_train[column].values\n",
    "  df_train[column] = [re.sub('<.*?>', ' ', i) for i in df_train[column].values]\n",
    "  df_train[column] = df_train[column].str.replace('&lt;', '<')\\\n",
    "                                          .str.replace('&gt;', '>')\\\n",
    "                                          .str.replace('&le;', '<=' )\\\n",
    "                                          .str.replace('&ge;', '>=')\n",
    " \n",
    "  # 2. remove latex i,e., if there is any formulas or latex we have to remove it\n",
    "  df_train[column] = [re.sub('\\$.*?\\$', ' ', i) for i in df_train[column].values]\n",
    " \n",
    "  # 3. all lowercase \n",
    "  df_train[column] = df_train[column].str.lower()\n",
    " \n",
    "  # 4. decontractions\n",
    "  df_train[column] = df_train[column].str.replace(\"won't\", \"will not\").str.replace(\"can\\'t\", \"can not\").str.replace(\"n\\'t\", \" not\").str.replace(\"\\'re\", \" are\").str.\\\n",
    "                                                replace(\"\\'s\", \" is\").str.replace(\"\\'d\", \" would\").str.replace(\"\\'ll\", \" will\").str.\\\n",
    "                                                replace(\"\\'t\", \" not\").str.replace(\"\\'ve\", \" have\").str.replace(\"\\'m\", \" am\")\n",
    "  \n",
    "  # 5. removing non-english or hebrew characters\n",
    "  df_train[column] = [i.encode(\"ascii\", \"ignore\").decode() for i in df_train[column].values]\n",
    " \n",
    "  # 6. remove all special-characters other than alpha-numericals\n",
    "  if remove_specialchar == True:\n",
    "    df_train[column] = [re.sub('[^A-Za-z0-9]+', ' ', i) for i in df_train[column].values]\n",
    " \n",
    "  # # 7. separating special chars from alphanumerics\n",
    "  # all_sc = [re.findall('[^ A-Za-z0-9]', i) for i in df_train[column].values]\n",
    "  # special_char = np.unique([j for i in all_sc for j in i])\n",
    "  # replace_char = [' '+i+' ' for i in special_char]\n",
    "  # for i,j in zip(special_char, replace_char):\n",
    "  #  df_train[column] = df_train[column].str.replace(i, j)\n",
    " \n",
    "  # 8. Stop_word removal\n",
    "  if remove_stopwords == True:\n",
    "    df_train[column] = [stopwrd_removal(i) for i in df_train[column].values]\n",
    " \n",
    "  # 9. remove all white-space i.e., \\n, \\t, and extra_spaces\n",
    "  df_train[column] = df_train[column].str.replace(\"\\n\", \" \").str.replace(\"\\t\", \" \").str.rstrip()\n",
    "  df_train[column] = [re.sub('  +', ' ', i) for i in df_train[column].values]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "IQ85mE8zjfSK"
   },
   "outputs": [],
   "source": [
    "# 1. text preprocessing\n",
    "df_train['clean_title'] = df_train['question_title']\n",
    "df_train['clean_body'] = df_train['question_body']\n",
    "df_train['clean_answer'] = df_train['answer']\n",
    "text_preprocessor('clean_title',  remove_stopwords = False, remove_specialchar = False)\n",
    "text_preprocessor('clean_body',  remove_stopwords = False, remove_specialchar = False)\n",
    "text_preprocessor('clean_answer',  remove_stopwords = False, remove_specialchar = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "onEsk2ueQk_s"
   },
   "source": [
    "# 4. Train_test_split - Random split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 33
    },
    "colab_type": "code",
    "id": "Etu9lo1iSwj0",
    "outputId": "90f7c242-601e-4abd-87a6-f581d50d73c8"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "30"
      ]
     },
     "execution_count": 7,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 1. setting up target features\n",
    "question_tar = ['question_asker_intent_understanding',\n",
    "       'question_body_critical', 'question_conversational',\n",
    "       'question_expect_short_answer', 'question_fact_seeking',\n",
    "       'question_has_commonly_accepted_answer',\n",
    "       'question_interestingness_others', 'question_interestingness_self',\n",
    "       'question_multi_intent', 'question_not_really_a_question',\n",
    "       'question_opinion_seeking', 'question_type_choice',\n",
    "       'question_type_compare', 'question_type_consequence',\n",
    "       'question_type_definition', 'question_type_entity',\n",
    "       'question_type_instructions', 'question_type_procedure',\n",
    "       'question_type_reason_explanation', 'question_type_spelling',\n",
    "       'question_well_written']\n",
    "       \n",
    "answer_tar = ['answer_helpful',\n",
    "       'answer_level_of_information', 'answer_plausible', 'answer_relevance',\n",
    "       'answer_satisfaction', 'answer_type_instructions',\n",
    "       'answer_type_procedure', 'answer_type_reason_explanation',\n",
    "       'answer_well_written']\n",
    " \n",
    "tar_features = question_tar + answer_tar\n",
    "len(tar_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 33
    },
    "colab_type": "code",
    "id": "PpP3IoR0Qta-",
    "outputId": "b9b5add9-2775-4245-a4dc-95f52b5fbd27"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((5349, 3), (730, 3), (5349, 30), (730, 30))"
      ]
     },
     "execution_count": 8,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 2. splitting dataset train_test_split\n",
    "from sklearn.model_selection import train_test_split\n",
    " \n",
    "X_train, X_cv, y_train, y_cv = train_test_split(df_train[['clean_title', 'clean_body', 'clean_answer']], df_train[tar_features], test_size = 0.12, random_state = 42)\n",
    "X_train.shape, X_cv.shape, y_train.shape, y_cv.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 33
    },
    "colab_type": "code",
    "id": "C43FZkxzXKWu",
    "outputId": "4c800362-a7a2-4c28-f4ca-6a7081a7358f"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5349, 5349, 730, 730)"
      ]
     },
     "execution_count": 9,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 3. creating training features : title + body = title_body | answer_train | title + body + answer = title_body_answer\n",
    "title_train = X_train['clean_title'].values\n",
    "body_train = X_train['clean_body'].values\n",
    "answer_train = X_train['clean_answer'].values\n",
    " \n",
    "title_cv = X_cv['clean_title'].values\n",
    "body_cv = X_cv['clean_body'].values\n",
    "answer_cv = X_cv['clean_answer'].values\n",
    " \n",
    "# train data\n",
    "title_body_train = [i+' '+j for i,j in zip(title_train, body_train)]\n",
    "y_train_ques = y_train[question_tar].values\n",
    "y_train_ans = y_train[answer_tar].values\n",
    " \n",
    "# cv data\n",
    "title_body_cv = [i+' '+j for i,j in zip(title_cv, body_cv)]\n",
    "y_cv_ques = y_cv[question_tar].values\n",
    "y_cv_ans = y_cv[answer_tar].values\n",
    " \n",
    "len(title_body_train), len(answer_train), len(title_body_cv), len(answer_cv)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "1nRC3ble_z3B"
   },
   "source": [
    "# 4.1. Sentence Embedding : BERT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 570
    },
    "colab_type": "code",
    "id": "Vjqn1vExAbT8",
    "outputId": "88311b52-3578-490e-9e71-f06b8695359e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting transformers\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/27/3c/91ed8f5c4e7ef3227b4119200fc0ed4b4fd965b1f0172021c25701087825/transformers-3.0.2-py3-none-any.whl (769kB)\n",
      "\u001b[K     |████████████████████████████████| 778kB 2.7MB/s \n",
      "\u001b[?25hCollecting sacremoses\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n",
      "\u001b[K     |████████████████████████████████| 890kB 49.8MB/s \n",
      "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers) (4.41.1)\n",
      "Requirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers) (3.0.12)\n",
      "Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from transformers) (20.4)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (2019.12.20)\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers) (1.18.5)\n",
      "Collecting sentencepiece!=0.1.92\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d4/a4/d0a884c4300004a78cca907a6ff9a5e9fe4f090f5d95ab341c53d28cbc58/sentencepiece-0.1.91-cp36-cp36m-manylinux1_x86_64.whl (1.1MB)\n",
      "\u001b[K     |████████████████████████████████| 1.1MB 48.3MB/s \n",
      "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers) (2.23.0)\n",
      "Requirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers) (0.7)\n",
      "Collecting tokenizers==0.8.1.rc1\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/40/d0/30d5f8d221a0ed981a186c8eb986ce1c94e3a6e87f994eae9f4aa5250217/tokenizers-0.8.1rc1-cp36-cp36m-manylinux1_x86_64.whl (3.0MB)\n",
      "\u001b[K     |████████████████████████████████| 3.0MB 44.1MB/s \n",
      "\u001b[?25hRequirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (1.15.0)\n",
      "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (7.1.2)\n",
      "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (0.16.0)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->transformers) (2.4.7)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (1.24.3)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2020.6.20)\n",
      "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (3.0.4)\n",
      "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2.10)\n",
      "Building wheels for collected packages: sacremoses\n",
      "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for sacremoses: filename=sacremoses-0.0.43-cp36-none-any.whl size=893260 sha256=57b6c263bc7fe6a54051d4d59400787222b18e9aeb68595aec2c810ac082dc1f\n",
      "  Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n",
      "Successfully built sacremoses\n",
      "Installing collected packages: sacremoses, sentencepiece, tokenizers, transformers\n",
      "Successfully installed sacremoses-0.0.43 sentencepiece-0.1.91 tokenizers-0.8.1rc1 transformers-3.0.2\n"
     ]
    }
   ],
   "source": [
    "!pip install transformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 262,
     "referenced_widgets": [
      "07820b67d0d0475486c9644a85edced2",
      "ec97992060844172a5b9a04fdfaa44c0",
      "f043a98303e84490b01d2fd44157ea56",
      "c894ae32b71843e4bcd376a8f1296659",
      "f182fc80701b4df58ccd3592546880b6",
      "0dc6097d33b74b678790542a49ecb783",
      "192cdbe6af654effb073e2172b653a49",
      "cc8670a704fb4bf282388423b35c23d2",
      "85f6a616ab024077b71db09bc1a99c94",
      "16baf2495d4e426e9e0ef51369764b86",
      "df22315806544dca9f103f6bc25d93fb",
      "1d626fc98e2749a5a1c11a873beb10cf",
      "9fc2e5520fcd4e39be45e31096842c6c",
      "58a453298c43429e8292d3fa9f3c4b11",
      "f46d8820d092457ba9c958512e713c48",
      "95cfe85ab42b46d9bffd5665fb3febd5",
      "1d2a0a1fe1634f16a13908ce185e49f4",
      "84a276e30bf1499e91a888cc735f3bf1",
      "a480a9eeca7c4dd889f9839c3195637e",
      "5dc26995ef604ce4b74ef8c83a8c228c",
      "e05140f32795468584a139c9d33ba1fd",
      "ad127bb1996f462cbf154f157ea4aed3",
      "a1ded462845e49639d05f8c7c89568cd",
      "776ff4e1e1ae48a1a42640d72a3b904d"
     ]
    },
    "colab_type": "code",
    "id": "-vamY42PAmkf",
    "outputId": "ec003856-ebe2-4f68-ac2b-fb46a12aa837"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "07820b67d0d0475486c9644a85edced2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=433.0, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "85f6a616ab024077b71db09bc1a99c94",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=231508.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1d2a0a1fe1634f16a13908ce185e49f4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=536063208.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing TFBertModel: ['mlm___cls', 'nsp___cls']\n",
      "- This IS expected if you are initializing TFBertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPretraining model).\n",
      "- This IS NOT expected if you are initializing TFBertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "All the weights of TFBertModel were initialized from the model checkpoint at bert-base-uncased.\n",
      "If your task is similar to the task the model of the ckeckpoint was trained on, you can already use TFBertModel for predictions without further training.\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "from transformers import BertTokenizer, TFBertModel, BertConfig\n",
    " \n",
    "# Load pretrained model/tokenizer\n",
    "config = BertConfig.from_pretrained('bert-base-uncased', output_hidden_states=True)\n",
    "bert_tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\n",
    "bert_model = TFBertModel.from_pretrained('bert-base-uncased', config = config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 33
    },
    "colab_type": "code",
    "id": "93WIPzIFZFNe",
    "outputId": "0e0c7db4-38ef-41b3-8743-4a61b9925e6d"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((5349, 512), (5349, 512), (5349, 512), (730, 512), (730, 512), (730, 512))"
      ]
     },
     "execution_count": 49,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Tokenizing : title_body\n",
    "title_body_train_tokens = bert_tokenizer.batch_encode_plus(title_body_train, max_length = 512, truncation = True, pad_to_max_length = True, return_tensors=\"tf\")\n",
    "title_body_cv_tokens = bert_tokenizer.batch_encode_plus(title_body_cv, max_length = 512, truncation = True, pad_to_max_length = True, return_tensors=\"tf\")\n",
    "# tokens = bert_tokenizer.batch_encode_plus(final_train, max_length = 50, truncation = True, pad_to_max_length = True)\n",
    "\n",
    "tb_train_input_ids  = np.array(title_body_train_tokens['input_ids'])\n",
    "tb_train_attn_mask = np.array(title_body_train_tokens['attention_mask'])\n",
    "tb_train_token_typ_ids = np.array(title_body_train_tokens['token_type_ids'])\n",
    " \n",
    "tb_cv_input_ids  = np.array(title_body_cv_tokens['input_ids'])\n",
    "tb_cv_attn_mask = np.array(title_body_cv_tokens['attention_mask'])\n",
    "tb_cv_token_typ_ids = np.array(title_body_cv_tokens['token_type_ids'])\n",
    "\n",
    "tb_train_input_ids.shape, tb_train_attn_mask.shape, tb_train_token_typ_ids.shape, tb_cv_input_ids.shape, tb_cv_attn_mask.shape, tb_cv_token_typ_ids.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 33
    },
    "colab_type": "code",
    "id": "kPt6fUmWAtad",
    "outputId": "753ce6fd-0ffb-4670-a8f6-1577117dcdf7"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((5349, 512), (5349, 512), (5349, 512), (730, 512), (730, 512), (730, 512))"
      ]
     },
     "execution_count": 50,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Tokenizing : answer\n",
    "ans_train_tokens = bert_tokenizer.batch_encode_plus(answer_train, max_length = 512, truncation = True, pad_to_max_length = True, return_tensors=\"tf\")\n",
    "ans_cv_tokens = bert_tokenizer.batch_encode_plus(answer_cv, max_length = 512, truncation = True, pad_to_max_length = True, return_tensors=\"tf\")\n",
    "# tokens = bert_tokenizer.batch_encode_plus(final_train, max_length = 50, truncation = True, pad_to_max_length = True)\n",
    " \n",
    "ans_train_input_ids  = np.array(ans_train_tokens['input_ids'])\n",
    "ans_train_attn_mask = np.array(ans_train_tokens['attention_mask'])\n",
    "ans_train_token_typ_ids = np.array(ans_train_tokens['token_type_ids'])\n",
    " \n",
    "ans_cv_input_ids  = np.array(ans_cv_tokens['input_ids'])\n",
    "ans_cv_attn_mask = np.array(ans_cv_tokens['attention_mask'])\n",
    "ans_cv_token_typ_ids = np.array(ans_cv_tokens['token_type_ids'])\n",
    "seq_len = ans_train_input_ids.shape[1]\n",
    " \n",
    "ans_train_input_ids.shape, ans_train_attn_mask.shape, ans_train_token_typ_ids.shape, ans_cv_input_ids.shape, ans_cv_attn_mask.shape, ans_cv_token_typ_ids.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "iXbtXZQunImM"
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "tf.compat.v1.enable_eager_execution()\n",
    "from tensorflow.keras.layers import Input, Dense, Dropout\n",
    "from tensorflow.keras.models import Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 146,
     "referenced_widgets": [
      "c761637a6a6b41d8ae2451ee3e9cb249",
      "9175f5a49ec74397971d1fcfaefda03a",
      "301af3078ebc4f6f868b22903757b273",
      "40ce8221d7b6421aacd1e1719fe89953",
      "edae9bc6832b44d1a9b74262fd264440",
      "d54a6f476a344f0b89c53a9b66a11e67",
      "e8f84bb9b0ba416ca36433b2ab542551",
      "34419669d655486ca2f5e1d90bd188cb",
      "86200091be6a4c818ba2ca9273b834f2",
      "81dc6c322356403f8e25657c8264b8d2",
      "018420f8e8b84a6e9fe879ad9b7905f8",
      "cea7e3ad706945a6a6510a294c6c7d53",
      "c612b892af2d47fca4cfafb42df21ab0",
      "10315fdabc2544eb8c7220a4c7707482",
      "cedb401a3ce74a3da07b1987625a33cd",
      "28c6e2e040f5413d9a569a2dbf34f45c"
     ]
    },
    "colab_type": "code",
    "id": "KPsBh01em1pF",
    "outputId": "05361309-79fe-467c-954a-cdf81bba4238"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c761637a6a6b41d8ae2451ee3e9cb249",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=168.0), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "(5349, 3072)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "86200091be6a4c818ba2ca9273b834f2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "(730, 3072)\n"
     ]
    }
   ],
   "source": [
    "# BERT embedings : title_body_train \n",
    "batch_size = 32\n",
    "l = tb_train_input_ids.shape[0]\n",
    "\n",
    "lst1 = []\n",
    "for i in tqdm(range((l//batch_size)+1)):\n",
    "  x = bert_model([tb_train_input_ids[i*batch_size:(i+1)*batch_size], tb_train_attn_mask[i*batch_size:(i+1)*batch_size], tb_train_token_typ_ids[i*batch_size:(i+1)*batch_size]])[2]\n",
    "\n",
    "  # getting hidden states of last 4 layers (avg of each timrstep)\n",
    "  lst = [tf.reduce_mean(i, axis = 1) for i in x[-4:]]\n",
    "\n",
    "  # concat all 4 averaged hidden_states\n",
    "  conc = tf.concat(lst, axis = 1)\n",
    "  lst1.append(conc)\n",
    "tb_BERT_train = tf.concat(lst1, axis = 0)\n",
    "print(tb_BERT_train.shape)\n",
    "\n",
    "# BERT embedings : title_body_train\n",
    "batch_size = 32\n",
    "l = ans_cv_input_ids.shape[0]\n",
    "\n",
    "lst1 = []\n",
    "for i in tqdm(range((l//batch_size)+1)):\n",
    "  x = bert_model([tb_cv_input_ids[i*batch_size:(i+1)*batch_size], tb_cv_attn_mask[i*batch_size:(i+1)*batch_size], tb_cv_token_typ_ids[i*batch_size:(i+1)*batch_size]])[2]\n",
    "  # getting hidden states of last 4 layers (avg of each timrstep)\n",
    "  lst = [tf.reduce_mean(i, axis = 1) for i in x[-4:]]\n",
    "  # lst = [i[:, 0, :] for i in x[-4:]]\n",
    "  conc = tf.concat(lst, axis = 1)\n",
    "  lst1.append(conc)\n",
    "tb_BERT_cv = tf.concat(lst1, axis = 0)\n",
    "print(tb_BERT_cv.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 146,
     "referenced_widgets": [
      "8e51481bf39342d0ae5d69aab423a06d",
      "e0f06efb9e0d4d8bb50c8db2567f18a6",
      "3b436761ce834477aef6ea24a471384f",
      "411cd0c4ddf8444f82b94598c32bc841",
      "937d3d3036e940c7bfb61d0b8844932c",
      "709146c17a7241459099b6f852aeab21",
      "42351d43cc264ba290edbe728308d54c",
      "bc9d2dab2c2849adb9063c4ac0059ef4",
      "168b77dc7a7e440a9e105f6faa1031f8",
      "d97737c2459a47bb95128a60efbbf91a",
      "cdc75a0ff6e641189c13f84ec32f5bb7",
      "34cca5b94cbd4ec39acd6d85bcdd644d",
      "628d0324f7034a36b7199ce0ea2b5537",
      "73a4fa09665d4c73b3a0055d45f898d3",
      "af0b6f34f2bb49c18a03921e7fe57b76",
      "8e6ac950fc7b44f3993cf0f5682a487e"
     ]
    },
    "colab_type": "code",
    "id": "QrVUlZt6dPEh",
    "outputId": "8a960f84-5ded-4832-846a-ef7c11c3a1e9"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8e51481bf39342d0ae5d69aab423a06d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=168.0), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "(5349, 3072)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "168b77dc7a7e440a9e105f6faa1031f8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "(730, 3072)\n"
     ]
    }
   ],
   "source": [
    "# BERT embedings : answer_train\n",
    "batch_size = 32\n",
    "l = ans_train_input_ids.shape[0]\n",
    "\n",
    "lst1 = []\n",
    "for i in tqdm(range((l//batch_size)+1)):\n",
    "  x = bert_model([ans_train_input_ids[i*batch_size:(i+1)*batch_size], ans_train_attn_mask[i*batch_size:(i+1)*batch_size], ans_train_token_typ_ids[i*batch_size:(i+1)*batch_size]])[2]\n",
    "  # getting hidden states of last 4 layers\n",
    "  lst = [i[:, 0, :] for i in x[-4:]]\n",
    "  conc = tf.concat(lst, axis = 1)\n",
    "  lst1.append(conc)\n",
    "ans_BERT_train = tf.concat(lst1, axis = 0)\n",
    "print(ans_BERT_train.shape)\n",
    "\n",
    "# cv : ans BERT embedings\n",
    "batch_size = 32\n",
    "l = ans_cv_input_ids.shape[0]\n",
    "lst1 = []\n",
    "for i in tqdm(range((l//batch_size)+1)):\n",
    "  x = bert_model([ans_cv_input_ids[i*batch_size:(i+1)*batch_size], ans_cv_attn_mask[i*batch_size:(i+1)*batch_size], ans_cv_token_typ_ids[i*batch_size:(i+1)*batch_size]])[2]\n",
    "  # getting hidden states of last 4 layers\n",
    "  lst = [i[:, 0, :] for i in x[-4:]]\n",
    "  conc = tf.concat(lst, axis = 1)\n",
    "  lst1.append(conc)\n",
    "ans_BERT_cv = tf.concat(lst1, axis = 0)\n",
    "print(ans_BERT_cv.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "FnEVo9wpOdoh"
   },
   "outputs": [],
   "source": [
    "# saving pretrained BERT embeddings\n",
    "np.savez_compressed('/content/drive/My Drive/AAIC Course/Personal case study 2  - Google Quest/bert_outputs', tb_BERT_train = tb_BERT_train.numpy(), tb_BERT_cv = tb_BERT_cv.numpy(), \n",
    "                    ans_BERT_train = ans_BERT_train.numpy(), ans_BERT_cv = ans_BERT_cv.numpy())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "qWFOCY_JXK4c"
   },
   "source": [
    "### predictor Model :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "NsFUMQAVN8At"
   },
   "outputs": [],
   "source": [
    "# loading pretrained BERT embeddings\n",
    "tb_BERT_train = np.load('/content/drive/My Drive/AAIC Course/Personal case study 2  - Google Quest/bert_outputs.npz')['tb_BERT_train']\n",
    "tb_BERT_cv = np.load('/content/drive/My Drive/AAIC Course/Personal case study 2  - Google Quest/bert_outputs.npz')['tb_BERT_cv']\n",
    "ans_BERT_train = np.load('/content/drive/My Drive/AAIC Course/Personal case study 2  - Google Quest/bert_outputs.npz')['ans_BERT_train']\n",
    "ans_BERT_cv = np.load('/content/drive/My Drive/AAIC Course/Personal case study 2  - Google Quest/bert_outputs.npz')['ans_BERT_cv']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 651
    },
    "colab_type": "code",
    "id": "Qtf4hzeT5Wy5",
    "outputId": "f03d1479-bbc9-45c5-c756-92d8274e2ef3"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"functional_1\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "tb_bert_out (InputLayer)        [(None, 3072)]       0                                            \n",
      "__________________________________________________________________________________________________\n",
      "ans_bert_out (InputLayer)       [(None, 3072)]       0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dropout (Dropout)               (None, 3072)         0           tb_bert_out[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dropout_3 (Dropout)             (None, 3072)         0           ans_bert_out[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dense (Dense)                   (None, 1024)         3146752     dropout[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_3 (Dense)                 (None, 2048)         6293504     dropout_3[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)             (None, 1024)         0           dense[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "dropout_4 (Dropout)             (None, 2048)         0           dense_3[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 512)          524800      dropout_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_4 (Dense)                 (None, 512)          1049088     dropout_4[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dropout_2 (Dropout)             (None, 512)          0           dense_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dropout_5 (Dropout)             (None, 512)          0           dense_4[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 21)           10773       dropout_2[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_5 (Dense)                 (None, 9)            4617        dropout_5[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_concat (TensorFlowO [(None, 30)]         0           dense_2[0][0]                    \n",
      "                                                                 dense_5[0][0]                    \n",
      "==================================================================================================\n",
      "Total params: 11,029,534\n",
      "Trainable params: 11,029,534\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "tf.keras.backend.clear_session()\n",
    "seed = 42\n",
    "\n",
    "# *-----------------title_body-----------------*\n",
    "tb_input = Input(name = 'tb_bert_out', shape = (768*4,), dtype = 'float32')\n",
    "tb_dropout_1 = Dropout(rate = 0.2, seed = seed)(tb_input)\n",
    " \n",
    "tb_dense_1 = Dense(units = 1024, activation = 'relu', kernel_initializer = tf.keras.initializers.he_normal(seed = seed))(tb_dropout_1)\n",
    "tb_dropout_2 = Dropout(rate = 0.2, seed = seed)(tb_dense_1)\n",
    " \n",
    "tb_dense_2 = Dense(units = 512, activation = 'relu', kernel_initializer = tf.keras.initializers.he_normal(seed = seed))(tb_dropout_2)\n",
    "tb_dropout_3 = Dropout(rate = 0.1, seed = seed)(tb_dense_2)\n",
    "\n",
    "tb_out = Dense(units = 21, activation = 'sigmoid', kernel_initializer = tf.keras.initializers.he_normal(seed = seed))(tb_dropout_3)\n",
    " \n",
    "# *-----------------answer_model-----------------*\n",
    "ans_input = Input(name = 'ans_bert_out', shape = (768*4,), dtype = 'float32')\n",
    "# concat = tf.concat([tb_out, ans_input], axis = -1) \n",
    "ans_dropout_1 = Dropout(rate = 0.2, seed = seed)(ans_input)\n",
    " \n",
    "ans_dense_1 = Dense(units = 2048, activation = 'relu', kernel_initializer = tf.keras.initializers.he_normal(seed = seed))(ans_dropout_1)\n",
    "ans_dropout_2 = Dropout(rate = 0.2, seed = seed)(ans_dense_1)\n",
    " \n",
    "ans_dense_2 = Dense(units = 512, activation = 'relu', kernel_initializer = tf.keras.initializers.he_normal(seed = seed))(ans_dropout_2)\n",
    "ans_dropout_3 = Dropout(rate = 0.1, seed = seed)(ans_dense_2)\n",
    "\n",
    "ans_out = Dense(units = 9, activation = 'sigmoid', kernel_initializer = tf.keras.initializers.he_normal(seed = seed))(ans_dropout_3)\n",
    "\n",
    "# *--------------------concat--------------------*\n",
    "out = tf.concat([tb_out, ans_out], axis = -1) \n",
    " \n",
    "model = Model(inputs = [tb_input, ans_input], outputs = out)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Fya9yY3TvaUn"
   },
   "outputs": [],
   "source": [
    "# post processing : binning\n",
    "def return_bins(arr):\n",
    "  val = np.unique(arr)\n",
    "  bins = []\n",
    "  for i in range(len(val)):\n",
    "    if i > 0:\n",
    "      bins.append((val[i-1] + val[i])/2)\n",
    "  return bins\n",
    "  \n",
    "unique_val_30 = [np.unique(df_train[tar_features].values[:, i]) for i in range(30)]\n",
    "bins_30 = [return_bins(df_train[tar_features].values[:, i]) for i in range(30)]\n",
    " \n",
    "def binned_out(y_pred):\n",
    "  col = y_pred.shape[1]\n",
    "  final_pred = np.zeros(y_pred.shape)\n",
    "  for i in range(col):\n",
    "    idx = np.digitize(y_pred[:, i], bins_30[i])\n",
    "    final_pred[:, i] = unique_val_30[i][idx]\n",
    "  return final_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 33
    },
    "colab_type": "code",
    "id": "fGH2enZEvcfy",
    "outputId": "1d9ba423-9974-412d-886d-c25bfa0f9ae9"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`write_grads` will be ignored in TensorFlow 2.0 for the `TensorBoard` Callback.\n"
     ]
    }
   ],
   "source": [
    "# Defining callbacks\n",
    "!rm -r '/content/saved models'\n",
    "!rm -r '/content/logs'\n",
    "!mkdir '/content/saved models'\n",
    "!mkdir '/content/logs/'\n",
    " \n",
    "# tensorboard callback\n",
    "import datetime\n",
    "log_dir=\"logs/\" + datetime.datetime.now().strftime(\"%Y-%m-%d %H_%M_%S\")\n",
    "tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir = log_dir, histogram_freq=1, write_graph=True, write_grads=True)\n",
    " \n",
    "# spearman function\n",
    "from scipy.stats import pearsonr, spearmanr\n",
    "def compute_spearman(y_true, y_pred, final_pred):\n",
    "  col = y_true.shape[1]\n",
    "  lst = []\n",
    "  for i in range(col):\n",
    "    # p = round(spearmanr(y_true[:, i], y_pred[:, i])[0], 5)\n",
    "    p = round(spearmanr(y_true[:, i], final_pred[:, i])[0], 5)\n",
    "    p = round(p, 5)\n",
    "    # if np.isnan(p):\n",
    "    #   p = round(spearmanr(y_true[:, i], y_pred[:, i])[0], 5)\n",
    "    lst.append(p)\n",
    "  return np.array(lst), round(np.nanmean(lst), 5)\n",
    " \n",
    "# Custom spearman metric\n",
    "class print_spearman(tf.keras.callbacks.Callback):\n",
    "    def __init__(self, train_data, validation_data):\n",
    "        super(tf.keras.callbacks.Callback, self).__init__()\n",
    "        self.x, self.y = train_data\n",
    "        self.val_x, self.val_y = validation_data\n",
    "    \n",
    "    def on_train_begin(self, logs={}):\n",
    "        self.all_feat_spearman = []\n",
    "        self.spearman_dict = {'train_spearman' :[], 'val_spearman' :[]}\n",
    " \n",
    "    def on_epoch_end(self, epoch, logs={}):\n",
    "        self.epoch = epoch\n",
    "        # 1. Test_set evaluation\n",
    "        print('\\nspearman :')\n",
    "        y_pred = self.model.predict(x = self.x)\n",
    "        y_pred_val = self.model.predict(x = self.val_x)\n",
    " \n",
    "        final_pred = binned_out(y_pred)\n",
    "        final_pred_val = binned_out(y_pred_val)\n",
    " \n",
    "        train_spear_lst, train_spearman = compute_spearman(self.y, y_pred, final_pred)\n",
    "        val_spear_lst, val_spearman = compute_spearman(self.val_y, y_pred_val, final_pred_val)\n",
    " \n",
    "        self.all_feat_spearman.append({'train_spearman' : train_spear_lst, 'val_spearman' : val_spear_lst})\n",
    " \n",
    "        self.spearman_dict['train_spearman'].append(train_spearman)\n",
    "        self.spearman_dict['val_spearman'].append(val_spearman)\n",
    "        prev_epoch_lr  = tf.keras.backend.eval(self.model.optimizer.lr)\n",
    "        print(\"train_spearman : {} | val_spearman : {} | Learning_Rate : {}\".format(train_spearman, val_spearman, round(prev_epoch_lr, 6)))\n",
    "        # print('train_spear_lst : ', train_spear_lst, '\\n' 'val_spear_lst :', val_spear_lst)\n",
    " \n",
    "reduce_lr = tf.keras.callbacks.ReduceLROnPlateau(monitor= 'val_loss', factor=np.sqrt(0.1), patience=4, verbose=1)\n",
    " \n",
    "checkpt = tf.keras.callbacks.ModelCheckpoint('/content/saved models/weights.-{epoch:03d}-{val_loss:.5f}', monitor='val_loss', verbose=1, save_best_only=False, save_weights_only=True)\n",
    " \n",
    "print_spearman_fn = print_spearman(train_data = ([tb_BERT_train, ans_BERT_train], y_train.values),\n",
    "                                 validation_data = ([tb_BERT_cv, ans_BERT_cv], y_cv.values))\n",
    "callbacks = [print_spearman_fn, reduce_lr, checkpt, tensorboard_callback]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "m9XVw-UtOvlr",
    "outputId": "b6c39074-bed0-446e-e707-2dc36c208292"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/40\n",
      " 2/84 [..............................] - ETA: 2s - loss: 0.7038 - root_mean_squared_error: 0.4161WARNING:tensorflow:Callbacks method `on_train_batch_end` is slow compared to the batch time (batch time: 0.0080s vs `on_train_batch_end` time: 0.0614s). Check your callbacks.\n",
      "84/84 [==============================] - ETA: 0s - loss: 0.4389 - root_mean_squared_error: 0.2535\n",
      "spearman :\n",
      "train_spearman : 0.28278 | val_spearman : 0.28746 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00001: saving model to /content/saved models/weights.-001-0.39854\n",
      "84/84 [==============================] - 2s 24ms/step - loss: 0.4389 - root_mean_squared_error: 0.2535 - val_loss: 0.3985 - val_root_mean_squared_error: 0.2234\n",
      "Epoch 2/40\n",
      "76/84 [==========================>...] - ETA: 0s - loss: 0.4059 - root_mean_squared_error: 0.2309\n",
      "spearman :\n",
      "train_spearman : 0.32832 | val_spearman : 0.32837 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00002: saving model to /content/saved models/weights.-002-0.39364\n",
      "84/84 [==============================] - 2s 23ms/step - loss: 0.4057 - root_mean_squared_error: 0.2307 - val_loss: 0.3936 - val_root_mean_squared_error: 0.2200\n",
      "Epoch 3/40\n",
      "76/84 [==========================>...] - ETA: 0s - loss: 0.3988 - root_mean_squared_error: 0.2261\n",
      "spearman :\n",
      "train_spearman : 0.3562 | val_spearman : 0.35006 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00003: saving model to /content/saved models/weights.-003-0.38994\n",
      "84/84 [==============================] - 2s 21ms/step - loss: 0.3993 - root_mean_squared_error: 0.2259 - val_loss: 0.3899 - val_root_mean_squared_error: 0.2167\n",
      "Epoch 4/40\n",
      "78/84 [==========================>...] - ETA: 0s - loss: 0.3938 - root_mean_squared_error: 0.2215\n",
      "spearman :\n",
      "train_spearman : 0.374 | val_spearman : 0.35836 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00004: saving model to /content/saved models/weights.-004-0.38757\n",
      "84/84 [==============================] - 2s 23ms/step - loss: 0.3936 - root_mean_squared_error: 0.2215 - val_loss: 0.3876 - val_root_mean_squared_error: 0.2153\n",
      "Epoch 5/40\n",
      "76/84 [==========================>...] - ETA: 0s - loss: 0.3906 - root_mean_squared_error: 0.2192\n",
      "spearman :\n",
      "train_spearman : 0.38748 | val_spearman : 0.3623 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00005: saving model to /content/saved models/weights.-005-0.38593\n",
      "84/84 [==============================] - 2s 21ms/step - loss: 0.3905 - root_mean_squared_error: 0.2192 - val_loss: 0.3859 - val_root_mean_squared_error: 0.2136\n",
      "Epoch 6/40\n",
      "75/84 [=========================>....] - ETA: 0s - loss: 0.3884 - root_mean_squared_error: 0.2177\n",
      "spearman :\n",
      "train_spearman : 0.39565 | val_spearman : 0.37429 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00006: saving model to /content/saved models/weights.-006-0.38497\n",
      "84/84 [==============================] - 2s 24ms/step - loss: 0.3881 - root_mean_squared_error: 0.2173 - val_loss: 0.3850 - val_root_mean_squared_error: 0.2129\n",
      "Epoch 7/40\n",
      "74/84 [=========================>....] - ETA: 0s - loss: 0.3843 - root_mean_squared_error: 0.2148\n",
      "spearman :\n",
      "train_spearman : 0.40851 | val_spearman : 0.3862 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00007: saving model to /content/saved models/weights.-007-0.38665\n",
      "84/84 [==============================] - 2s 23ms/step - loss: 0.3856 - root_mean_squared_error: 0.2153 - val_loss: 0.3867 - val_root_mean_squared_error: 0.2143\n",
      "Epoch 8/40\n",
      "84/84 [==============================] - ETA: 0s - loss: 0.3841 - root_mean_squared_error: 0.2140\n",
      "spearman :\n",
      "train_spearman : 0.40533 | val_spearman : 0.38297 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00008: saving model to /content/saved models/weights.-008-0.38379\n",
      "84/84 [==============================] - 2s 22ms/step - loss: 0.3841 - root_mean_squared_error: 0.2140 - val_loss: 0.3838 - val_root_mean_squared_error: 0.2122\n",
      "Epoch 9/40\n",
      "78/84 [==========================>...] - ETA: 0s - loss: 0.3822 - root_mean_squared_error: 0.2124\n",
      "spearman :\n",
      "train_spearman : 0.41247 | val_spearman : 0.39166 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00009: saving model to /content/saved models/weights.-009-0.38326\n",
      "84/84 [==============================] - 2s 21ms/step - loss: 0.3823 - root_mean_squared_error: 0.2127 - val_loss: 0.3833 - val_root_mean_squared_error: 0.2114\n",
      "Epoch 10/40\n",
      "76/84 [==========================>...] - ETA: 0s - loss: 0.3810 - root_mean_squared_error: 0.2116\n",
      "spearman :\n",
      "train_spearman : 0.41713 | val_spearman : 0.38642 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00010: saving model to /content/saved models/weights.-010-0.38311\n",
      "84/84 [==============================] - 2s 23ms/step - loss: 0.3809 - root_mean_squared_error: 0.2115 - val_loss: 0.3831 - val_root_mean_squared_error: 0.2114\n",
      "Epoch 11/40\n",
      "77/84 [==========================>...] - ETA: 0s - loss: 0.3794 - root_mean_squared_error: 0.2100\n",
      "spearman :\n",
      "train_spearman : 0.42478 | val_spearman : 0.38485 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00011: saving model to /content/saved models/weights.-011-0.38103\n",
      "84/84 [==============================] - 2s 21ms/step - loss: 0.3789 - root_mean_squared_error: 0.2098 - val_loss: 0.3810 - val_root_mean_squared_error: 0.2101\n",
      "Epoch 12/40\n",
      "79/84 [===========================>..] - ETA: 0s - loss: 0.3779 - root_mean_squared_error: 0.2094\n",
      "spearman :\n",
      "train_spearman : 0.43166 | val_spearman : 0.39216 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00012: saving model to /content/saved models/weights.-012-0.38110\n",
      "84/84 [==============================] - 2s 22ms/step - loss: 0.3781 - root_mean_squared_error: 0.2093 - val_loss: 0.3811 - val_root_mean_squared_error: 0.2100\n",
      "Epoch 13/40\n",
      "78/84 [==========================>...] - ETA: 0s - loss: 0.3770 - root_mean_squared_error: 0.2082\n",
      "spearman :\n",
      "train_spearman : 0.43582 | val_spearman : 0.37873 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00013: saving model to /content/saved models/weights.-013-0.38113\n",
      "84/84 [==============================] - 2s 22ms/step - loss: 0.3769 - root_mean_squared_error: 0.2083 - val_loss: 0.3811 - val_root_mean_squared_error: 0.2100\n",
      "Epoch 14/40\n",
      "84/84 [==============================] - ETA: 0s - loss: 0.3753 - root_mean_squared_error: 0.2071\n",
      "spearman :\n",
      "train_spearman : 0.44369 | val_spearman : 0.38182 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00014: saving model to /content/saved models/weights.-014-0.38363\n",
      "84/84 [==============================] - 2s 21ms/step - loss: 0.3753 - root_mean_squared_error: 0.2071 - val_loss: 0.3836 - val_root_mean_squared_error: 0.2114\n",
      "Epoch 15/40\n",
      "77/84 [==========================>...] - ETA: 0s - loss: 0.3747 - root_mean_squared_error: 0.2065\n",
      "spearman :\n",
      "train_spearman : 0.44933 | val_spearman : 0.38474 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00015: saving model to /content/saved models/weights.-015-0.38071\n",
      "84/84 [==============================] - 2s 24ms/step - loss: 0.3746 - root_mean_squared_error: 0.2065 - val_loss: 0.3807 - val_root_mean_squared_error: 0.2097\n",
      "Epoch 16/40\n",
      "79/84 [===========================>..] - ETA: 0s - loss: 0.3725 - root_mean_squared_error: 0.2047\n",
      "spearman :\n",
      "train_spearman : 0.4563 | val_spearman : 0.38179 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00016: saving model to /content/saved models/weights.-016-0.38133\n",
      "84/84 [==============================] - 2s 22ms/step - loss: 0.3727 - root_mean_squared_error: 0.2049 - val_loss: 0.3813 - val_root_mean_squared_error: 0.2098\n",
      "Epoch 17/40\n",
      "75/84 [=========================>....] - ETA: 0s - loss: 0.3710 - root_mean_squared_error: 0.2039\n",
      "spearman :\n",
      "train_spearman : 0.46085 | val_spearman : 0.3842 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00017: saving model to /content/saved models/weights.-017-0.38210\n",
      "84/84 [==============================] - 2s 21ms/step - loss: 0.3719 - root_mean_squared_error: 0.2042 - val_loss: 0.3821 - val_root_mean_squared_error: 0.2105\n",
      "Epoch 18/40\n",
      "75/84 [=========================>....] - ETA: 0s - loss: 0.3702 - root_mean_squared_error: 0.2036\n",
      "spearman :\n",
      "train_spearman : 0.46847 | val_spearman : 0.385 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00018: saving model to /content/saved models/weights.-018-0.38240\n",
      "84/84 [==============================] - 2s 22ms/step - loss: 0.3708 - root_mean_squared_error: 0.2034 - val_loss: 0.3824 - val_root_mean_squared_error: 0.2107\n",
      "Epoch 19/40\n",
      "75/84 [=========================>....] - ETA: 0s - loss: 0.3695 - root_mean_squared_error: 0.2021\n",
      "spearman :\n",
      "train_spearman : 0.47365 | val_spearman : 0.38302 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00019: ReduceLROnPlateau reducing learning rate to 3.1622775802825264e-05.\n",
      "\n",
      "Epoch 00019: saving model to /content/saved models/weights.-019-0.38166\n",
      "84/84 [==============================] - 2s 22ms/step - loss: 0.3697 - root_mean_squared_error: 0.2024 - val_loss: 0.3817 - val_root_mean_squared_error: 0.2100\n",
      "Epoch 20/40\n",
      "75/84 [=========================>....] - ETA: 0s - loss: 0.3674 - root_mean_squared_error: 0.2005\n",
      "spearman :\n",
      "train_spearman : 0.48182 | val_spearman : 0.38811 | Learning_Rate : 3.199999991920777e-05\n",
      "\n",
      "Epoch 00020: saving model to /content/saved models/weights.-020-0.38025\n",
      "84/84 [==============================] - 2s 23ms/step - loss: 0.3675 - root_mean_squared_error: 0.2005 - val_loss: 0.3802 - val_root_mean_squared_error: 0.2091\n",
      "Epoch 21/40\n",
      "75/84 [=========================>....] - ETA: 0s - loss: 0.3664 - root_mean_squared_error: 0.1997\n",
      "spearman :\n",
      "train_spearman : 0.4873 | val_spearman : 0.39135 | Learning_Rate : 3.199999991920777e-05\n",
      "\n",
      "Epoch 00021: saving model to /content/saved models/weights.-021-0.37994\n",
      "84/84 [==============================] - 6s 67ms/step - loss: 0.3662 - root_mean_squared_error: 0.1994 - val_loss: 0.3799 - val_root_mean_squared_error: 0.2089\n",
      "Epoch 22/40\n",
      "76/84 [==========================>...] - ETA: 0s - loss: 0.3655 - root_mean_squared_error: 0.1988\n",
      "spearman :\n",
      "train_spearman : 0.4886 | val_spearman : 0.38424 | Learning_Rate : 3.199999991920777e-05\n",
      "\n",
      "Epoch 00022: saving model to /content/saved models/weights.-022-0.38084\n",
      "84/84 [==============================] - 2s 26ms/step - loss: 0.3656 - root_mean_squared_error: 0.1990 - val_loss: 0.3808 - val_root_mean_squared_error: 0.2094\n",
      "Epoch 23/40\n",
      "79/84 [===========================>..] - ETA: 0s - loss: 0.3653 - root_mean_squared_error: 0.1988\n",
      "spearman :\n",
      "train_spearman : 0.49297 | val_spearman : 0.3901 | Learning_Rate : 3.199999991920777e-05\n",
      "\n",
      "Epoch 00023: saving model to /content/saved models/weights.-023-0.37998\n",
      "84/84 [==============================] - 4s 44ms/step - loss: 0.3656 - root_mean_squared_error: 0.1989 - val_loss: 0.3800 - val_root_mean_squared_error: 0.2088\n",
      "Epoch 24/40\n",
      "75/84 [=========================>....] - ETA: 0s - loss: 0.3650 - root_mean_squared_error: 0.1986\n",
      "spearman :\n",
      "train_spearman : 0.49532 | val_spearman : 0.39338 | Learning_Rate : 3.199999991920777e-05\n",
      "\n",
      "Epoch 00024: saving model to /content/saved models/weights.-024-0.37902\n",
      "84/84 [==============================] - 6s 71ms/step - loss: 0.3651 - root_mean_squared_error: 0.1985 - val_loss: 0.3790 - val_root_mean_squared_error: 0.2081\n",
      "Epoch 25/40\n",
      "84/84 [==============================] - ETA: 0s - loss: 0.3645 - root_mean_squared_error: 0.1981\n",
      "spearman :\n",
      "train_spearman : 0.49676 | val_spearman : 0.39023 | Learning_Rate : 3.199999991920777e-05\n",
      "\n",
      "Epoch 00025: saving model to /content/saved models/weights.-025-0.37952\n",
      "84/84 [==============================] - 2s 21ms/step - loss: 0.3645 - root_mean_squared_error: 0.1981 - val_loss: 0.3795 - val_root_mean_squared_error: 0.2084\n",
      "Epoch 26/40\n",
      "76/84 [==========================>...] - ETA: 0s - loss: 0.3636 - root_mean_squared_error: 0.1976\n",
      "spearman :\n",
      "train_spearman : 0.50081 | val_spearman : 0.39536 | Learning_Rate : 3.199999991920777e-05\n",
      "\n",
      "Epoch 00026: saving model to /content/saved models/weights.-026-0.38004\n",
      "84/84 [==============================] - 2s 23ms/step - loss: 0.3640 - root_mean_squared_error: 0.1977 - val_loss: 0.3800 - val_root_mean_squared_error: 0.2088\n",
      "Epoch 27/40\n",
      "75/84 [=========================>....] - ETA: 0s - loss: 0.3634 - root_mean_squared_error: 0.1972\n",
      "spearman :\n",
      "train_spearman : 0.50223 | val_spearman : 0.38971 | Learning_Rate : 3.199999991920777e-05\n",
      "\n",
      "Epoch 00027: saving model to /content/saved models/weights.-027-0.38019\n",
      "84/84 [==============================] - 2s 27ms/step - loss: 0.3637 - root_mean_squared_error: 0.1972 - val_loss: 0.3802 - val_root_mean_squared_error: 0.2090\n",
      "Epoch 28/40\n",
      "82/84 [============================>.] - ETA: 0s - loss: 0.3631 - root_mean_squared_error: 0.1969\n",
      "spearman :\n",
      "train_spearman : 0.50548 | val_spearman : 0.38976 | Learning_Rate : 3.199999991920777e-05\n",
      "\n",
      "Epoch 00028: ReduceLROnPlateau reducing learning rate to 9.999999259090306e-06.\n",
      "\n",
      "Epoch 00028: saving model to /content/saved models/weights.-028-0.38102\n",
      "84/84 [==============================] - 6s 76ms/step - loss: 0.3630 - root_mean_squared_error: 0.1969 - val_loss: 0.3810 - val_root_mean_squared_error: 0.2096\n",
      "Epoch 29/40\n",
      "76/84 [==========================>...] - ETA: 0s - loss: 0.3624 - root_mean_squared_error: 0.1961\n",
      "spearman :\n",
      "train_spearman : 0.5081 | val_spearman : 0.39394 | Learning_Rate : 9.999999747378752e-06\n",
      "\n",
      "Epoch 00029: saving model to /content/saved models/weights.-029-0.37906\n",
      "84/84 [==============================] - 2s 23ms/step - loss: 0.3626 - root_mean_squared_error: 0.1965 - val_loss: 0.3791 - val_root_mean_squared_error: 0.2081\n",
      "Epoch 30/40\n",
      "74/84 [=========================>....] - ETA: 0s - loss: 0.3629 - root_mean_squared_error: 0.1961\n",
      "spearman :\n",
      "train_spearman : 0.50983 | val_spearman : 0.39371 | Learning_Rate : 9.999999747378752e-06\n",
      "\n",
      "Epoch 00030: saving model to /content/saved models/weights.-030-0.37909\n",
      "84/84 [==============================] - 2s 22ms/step - loss: 0.3620 - root_mean_squared_error: 0.1958 - val_loss: 0.3791 - val_root_mean_squared_error: 0.2082\n",
      "Epoch 31/40\n",
      "83/84 [============================>.] - ETA: 0s - loss: 0.3614 - root_mean_squared_error: 0.1955\n",
      "spearman :\n",
      "train_spearman : 0.51016 | val_spearman : 0.39106 | Learning_Rate : 9.999999747378752e-06\n",
      "\n",
      "Epoch 00031: saving model to /content/saved models/weights.-031-0.37980\n",
      "84/84 [==============================] - 2s 28ms/step - loss: 0.3614 - root_mean_squared_error: 0.1954 - val_loss: 0.3798 - val_root_mean_squared_error: 0.2086\n",
      "Epoch 32/40\n",
      "84/84 [==============================] - ETA: 0s - loss: 0.3618 - root_mean_squared_error: 0.1959\n",
      "spearman :\n",
      "train_spearman : 0.51144 | val_spearman : 0.39124 | Learning_Rate : 9.999999747378752e-06\n",
      "\n",
      "Epoch 00032: ReduceLROnPlateau reducing learning rate to 3.162277292675049e-06.\n",
      "\n",
      "Epoch 00032: saving model to /content/saved models/weights.-032-0.38013\n",
      "84/84 [==============================] - 7s 81ms/step - loss: 0.3618 - root_mean_squared_error: 0.1959 - val_loss: 0.3801 - val_root_mean_squared_error: 0.2089\n",
      "Epoch 33/40\n",
      "75/84 [=========================>....] - ETA: 0s - loss: 0.3622 - root_mean_squared_error: 0.1960\n",
      "spearman :\n",
      "train_spearman : 0.51203 | val_spearman : 0.39335 | Learning_Rate : 3.000000106112566e-06\n",
      "\n",
      "Epoch 00033: saving model to /content/saved models/weights.-033-0.37965\n",
      "84/84 [==============================] - 2s 23ms/step - loss: 0.3615 - root_mean_squared_error: 0.1955 - val_loss: 0.3797 - val_root_mean_squared_error: 0.2085\n",
      "Epoch 34/40\n",
      "81/84 [===========================>..] - ETA: 0s - loss: 0.3612 - root_mean_squared_error: 0.1951\n",
      "spearman :\n",
      "train_spearman : 0.51241 | val_spearman : 0.39363 | Learning_Rate : 3.000000106112566e-06\n",
      "\n",
      "Epoch 00034: saving model to /content/saved models/weights.-034-0.37952\n",
      "84/84 [==============================] - 2s 22ms/step - loss: 0.3612 - root_mean_squared_error: 0.1954 - val_loss: 0.3795 - val_root_mean_squared_error: 0.2085\n",
      "Epoch 35/40\n",
      "76/84 [==========================>...] - ETA: 0s - loss: 0.3613 - root_mean_squared_error: 0.1951\n",
      "spearman :\n",
      "train_spearman : 0.51223 | val_spearman : 0.39201 | Learning_Rate : 3.000000106112566e-06\n",
      "\n",
      "Epoch 00035: saving model to /content/saved models/weights.-035-0.37972\n",
      "84/84 [==============================] - 6s 74ms/step - loss: 0.3610 - root_mean_squared_error: 0.1951 - val_loss: 0.3797 - val_root_mean_squared_error: 0.2086\n",
      "Epoch 36/40\n",
      "80/84 [===========================>..] - ETA: 0s - loss: 0.3608 - root_mean_squared_error: 0.1951\n",
      "spearman :\n",
      "train_spearman : 0.51314 | val_spearman : 0.39401 | Learning_Rate : 3.000000106112566e-06\n",
      "\n",
      "Epoch 00036: ReduceLROnPlateau reducing learning rate to 9.999999115286567e-07.\n",
      "\n",
      "Epoch 00036: saving model to /content/saved models/weights.-036-0.37954\n",
      "84/84 [==============================] - 2s 23ms/step - loss: 0.3610 - root_mean_squared_error: 0.1951 - val_loss: 0.3795 - val_root_mean_squared_error: 0.2085\n",
      "Epoch 37/40\n",
      "81/84 [===========================>..] - ETA: 0s - loss: 0.3616 - root_mean_squared_error: 0.1956\n",
      "spearman :\n",
      "train_spearman : 0.51296 | val_spearman : 0.39329 | Learning_Rate : 9.999999974752427e-07\n",
      "\n",
      "Epoch 00037: saving model to /content/saved models/weights.-037-0.37959\n",
      "84/84 [==============================] - 2s 22ms/step - loss: 0.3614 - root_mean_squared_error: 0.1955 - val_loss: 0.3796 - val_root_mean_squared_error: 0.2085\n",
      "Epoch 38/40\n",
      "83/84 [============================>.] - ETA: 0s - loss: 0.3611 - root_mean_squared_error: 0.1951\n",
      "spearman :\n",
      "train_spearman : 0.51317 | val_spearman : 0.39275 | Learning_Rate : 9.999999974752427e-07\n",
      "\n",
      "Epoch 00038: saving model to /content/saved models/weights.-038-0.37959\n",
      "84/84 [==============================] - 3s 35ms/step - loss: 0.3611 - root_mean_squared_error: 0.1951 - val_loss: 0.3796 - val_root_mean_squared_error: 0.2085\n",
      "Epoch 39/40\n",
      "77/84 [==========================>...] - ETA: 0s - loss: 0.3612 - root_mean_squared_error: 0.1950\n",
      "spearman :\n",
      "train_spearman : 0.51327 | val_spearman : 0.39294 | Learning_Rate : 9.999999974752427e-07\n",
      "\n",
      "Epoch 00039: saving model to /content/saved models/weights.-039-0.37965\n",
      "84/84 [==============================] - 5s 64ms/step - loss: 0.3608 - root_mean_squared_error: 0.1949 - val_loss: 0.3796 - val_root_mean_squared_error: 0.2085\n",
      "Epoch 40/40\n",
      "77/84 [==========================>...] - ETA: 0s - loss: 0.3610 - root_mean_squared_error: 0.1949\n",
      "spearman :\n",
      "train_spearman : 0.51342 | val_spearman : 0.39215 | Learning_Rate : 9.999999974752427e-07\n",
      "\n",
      "Epoch 00040: ReduceLROnPlateau reducing learning rate to 3.1622772926750485e-07.\n",
      "\n",
      "Epoch 00040: saving model to /content/saved models/weights.-040-0.37968\n",
      "84/84 [==============================] - 2s 23ms/step - loss: 0.3610 - root_mean_squared_error: 0.1951 - val_loss: 0.3797 - val_root_mean_squared_error: 0.2086\n"
     ]
    }
   ],
   "source": [
    "# BERT_model : training a model\n",
    "tf.keras.backend.clear_session()\n",
    "opt = tf.keras.optimizers.Adam(learning_rate = 0.0001)\n",
    "rmse = tf.keras.metrics.RootMeanSquaredError()\n",
    " \n",
    "model.compile(loss = 'binary_crossentropy', optimizer = opt,  metrics = [rmse])\n",
    "history = model.fit(x = [tb_BERT_train, ans_BERT_train], y =  y_train.values,\n",
    "                    validation_data = ([tb_BERT_cv, ans_BERT_cv], y_cv.values),\n",
    "                    batch_size = 64, epochs = 40, callbacks = callbacks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 33
    },
    "colab_type": "code",
    "id": "87-lOm04O2Li",
    "outputId": "80730d04-9954-44cb-cb09-067afa9b72fe"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.39536, 25)"
      ]
     },
     "execution_count": 23,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# best_results : (val_spearman, epoch_no)\n",
    "max(print_spearman_fn.spearman_dict['val_spearman']), np.argmax(print_spearman_fn.spearman_dict['val_spearman'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "zgSO3OsO8J0Q"
   },
   "source": [
    "# 4.2. USE Embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "yhfrVy3JBjH1"
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "import tensorflow_hub as hub\n",
    "from tensorflow.keras.layers import Input, Softmax, GRU, LSTM, RNN, Embedding, Dense, RepeatVector, TimeDistributed, Bidirectional, Concatenate\n",
    "from tensorflow.keras.models import Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 82
    },
    "colab_type": "code",
    "id": "n-aQpp3S8MAZ",
    "outputId": "b64e3ff8-803a-4e27-b0a2-9645e0d2e06b"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:absl:Using /tmp/tfhub_modules to cache modules.\n",
      "INFO:absl:Downloading TF-Hub Module 'https://tfhub.dev/google/universal-sentence-encoder-large/5'.\n",
      "INFO:absl:Downloaded https://tfhub.dev/google/universal-sentence-encoder-large/5, Total size: 577.10MB\n",
      "INFO:absl:Downloaded TF-Hub Module 'https://tfhub.dev/google/universal-sentence-encoder-large/5'.\n"
     ]
    }
   ],
   "source": [
    "# loading USE model\n",
    "use_model = hub.load(\"https://tfhub.dev/google/universal-sentence-encoder-large/5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 113,
     "referenced_widgets": [
      "b6fa98b87b32493aa7c57db485c0b29a",
      "44546631613c4ef88aca301ade538c1d",
      "e652ecc7474249d7bc60f2687e0a5e40",
      "f40413e300bb4099870dbca3b5495af2",
      "16dde04360234e8b9e17471341047a08",
      "82ce03604a4f41a698f77ff22a758707",
      "99762cc3d82e4237b7f8b43f69c04452",
      "595bd7256a014f9abc143feb67ec7851",
      "d83706549c114e2381d0211f346f66f8",
      "f28025790a204cdd877493c5e712dde0",
      "cd61a31aa67048ccb483eec1793f9cf9",
      "6a7f283458e44839978e11960dbe592a",
      "6c81df840b5048169571d2e189ff9a5d",
      "dcadc26b893d49ea98ac4e9bba54dec9",
      "cbc0d08cea85421c947d1b3f9179e01f",
      "6d363da8804848df920c9863c0b58871"
     ]
    },
    "colab_type": "code",
    "id": "_6P_HI9CDUIV",
    "outputId": "0a743cdd-13da-43c9-a293-870da02ecaf5"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b6fa98b87b32493aa7c57db485c0b29a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=84.0), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d83706549c114e2381d0211f346f66f8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=12.0), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# title_body\n",
    "tb_USE_train = np.zeros((len(title_body_train), 512))\n",
    "batch_size = 32\n",
    "for i in tqdm(range(int(len(title_body_train)/batch_size)+1)):\n",
    "  tb_USE_train[i*batch_size : (i+1)*batch_size] = use_model(title_body_train[i*batch_size : (i+1)*batch_size]).numpy()\n",
    "print(tb_USE_train.shape)\n",
    "\n",
    "tb_USE_cv = np.zeros((len(title_body_cv), 512))\n",
    "batch_size = 64\n",
    "for i in tqdm(range(int(len(title_body_cv)/batch_size)+1)):\n",
    "  tb_USE_cv[i*batch_size : (i+1)*batch_size] = use_model(title_body_cv[i*batch_size : (i+1)*batch_size]).numpy()\n",
    "print(tb_USE_cv.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 146,
     "referenced_widgets": [
      "4d58ab0e0305477188be681484d928aa",
      "7107da192fb942c3a59eb40c806476c8",
      "f13802026eaa4a5b8e7e3097e3c5b84a",
      "56c07f7825d843598400abf0498aac40",
      "13e45ca5a45148f9b913ca06a977bf49",
      "a6a69668740148558c5a8f61b3fc8fc8",
      "2bb3a1b6466144728401886f51721bdc",
      "48edaebcf82147319f24b3cc1a32c76c",
      "ac44a30beb2a45c29130c7f7cad7b8ae",
      "a1cf58380f654e5b9aadcd13e49d0d33",
      "0072d96c78a04a6c94a66d66fd322f9f",
      "ea4b6964735144b88aa7bd5b1b8c7636",
      "24c6bace57ae41a88310b32174c7162e",
      "1545fb47849e467188b7dd6455df0584",
      "10549e4d4ffe448badf1768aa93142ec",
      "6771fee3e62a4a2baff8a192189ebebe"
     ]
    },
    "colab_type": "code",
    "id": "2ssPuJrJFXmt",
    "outputId": "fc56c06c-0dbf-43f3-f39b-93674a182b24"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4d58ab0e0305477188be681484d928aa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=168.0), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "(5349, 512)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ac44a30beb2a45c29130c7f7cad7b8ae",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "(730, 512)\n"
     ]
    }
   ],
   "source": [
    "# answer\n",
    "ans_USE_train = np.zeros((len(answer_train), 512))\n",
    "batch_size = 32\n",
    "for i in tqdm(range(int(len(answer_train)/batch_size)+1)):\n",
    "  ans_USE_train[i*batch_size : (i+1)*batch_size] = use_model(answer_train[i*batch_size : (i+1)*batch_size]).numpy()\n",
    "print(ans_USE_train.shape)\n",
    "\n",
    "ans_USE_cv = np.zeros((len(answer_cv), 512))\n",
    "batch_size = 32\n",
    "for i in tqdm(range(int(len(answer_cv)/batch_size)+1)):\n",
    "  ans_USE_cv[i*batch_size : (i+1)*batch_size] = use_model(answer_cv[i*batch_size : (i+1)*batch_size]).numpy()\n",
    "print(ans_USE_cv.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "k8NR9jDwXjt_"
   },
   "outputs": [],
   "source": [
    "# saving pretrained USE embeddings\n",
    "np.savez_compressed('/content/drive/My Drive/AAIC Course/Personal case study 2  - Google Quest/use_outputs', tb_USE_train = tb_USE_train, tb_USE_cv = tb_USE_cv, \n",
    "                    ans_USE_train = ans_USE_train, ans_USE_cv = ans_USE_cv)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "kA_c_SUlX97r"
   },
   "source": [
    "### Modeling :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "GLfGTWGWRGzD"
   },
   "outputs": [],
   "source": [
    "# laoding pretrained USE embeddings\n",
    "tb_USE_train = np.load('/content/drive/My Drive/AAIC Course/Personal case study 2  - Google Quest/use_outputs.npz')['tb_USE_train']\n",
    "tb_USE_cv = np.load('/content/drive/My Drive/AAIC Course/Personal case study 2  - Google Quest/use_outputs.npz')['tb_USE_cv']\n",
    "ans_USE_train = np.load('/content/drive/My Drive/AAIC Course/Personal case study 2  - Google Quest/use_outputs.npz')['ans_USE_train']\n",
    "ans_USE_cv = np.load('/content/drive/My Drive/AAIC Course/Personal case study 2  - Google Quest/use_outputs.npz')['ans_USE_cv']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 700
    },
    "colab_type": "code",
    "id": "v3MJ0K9_G8IA",
    "outputId": "ee414094-65f1-4181-f233-6d8362cfe3b5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"functional_1\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "tb_use_out (InputLayer)         [(None, 512)]        0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dropout (Dropout)               (None, 512)          0           tb_use_out[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense (Dense)                   (None, 1536)         787968      dropout[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)             (None, 1536)         0           dense[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 768)          1180416     dropout_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dropout_2 (Dropout)             (None, 768)          0           dense_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 21)           16149       dropout_2[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "ans_use_out (InputLayer)        [(None, 512)]        0                                            \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_concat (TensorFlowO [(None, 533)]        0           dense_2[0][0]                    \n",
      "                                                                 ans_use_out[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dropout_3 (Dropout)             (None, 533)          0           tf_op_layer_concat[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "dense_3 (Dense)                 (None, 1024)         546816      dropout_3[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dropout_4 (Dropout)             (None, 1024)         0           dense_3[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_4 (Dense)                 (None, 512)          524800      dropout_4[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dropout_5 (Dropout)             (None, 512)          0           dense_4[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_5 (Dense)                 (None, 9)            4617        dropout_5[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_concat_1 (TensorFlo [(None, 30)]         0           dense_2[0][0]                    \n",
      "                                                                 dense_5[0][0]                    \n",
      "==================================================================================================\n",
      "Total params: 3,060,766\n",
      "Trainable params: 3,060,766\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "tf.keras.backend.clear_session()\n",
    "seed = 42\n",
    "\n",
    "# *-----------------title_body-----------------*\n",
    "tb_input = Input(name = 'tb_use_out', shape = (512,), dtype = 'float32')\n",
    "tb_dropout_1 = Dropout(rate = 0.2, seed = seed)(tb_input)\n",
    " \n",
    "tb_dense_1 = Dense(units = 1024, activation = 'relu', kernel_initializer = tf.keras.initializers.he_normal(seed = seed))(tb_dropout_1)\n",
    "tb_dropout_2 = Dropout(rate = 0.2, seed = seed)(tb_dense_1)\n",
    " \n",
    "tb_dense_2 = Dense(units = 512, activation = 'relu', kernel_initializer = tf.keras.initializers.he_normal(seed = seed))(tb_dropout_2)\n",
    "tb_dropout_3 = Dropout(rate = 0.2, seed = seed)(tb_dense_2)\n",
    "\n",
    "tb_out = Dense(units = 21, activation = 'sigmoid', kernel_initializer = tf.keras.initializers.he_normal(seed = seed))(tb_dropout_3)\n",
    " \n",
    "# *-----------------answer_model-----------------*\n",
    "ans_input = Input(name = 'ans_use_out', shape = (512,), dtype = 'float32')\n",
    "concat = tf.concat([tb_out, ans_input], axis = -1) \n",
    "ans_dropout_1 = Dropout(rate = 0.2, seed = seed)(concat)\n",
    " \n",
    "ans_dense_1 = Dense(units = 1024, activation = 'relu', kernel_initializer = tf.keras.initializers.he_normal(seed = seed))(ans_dropout_1)\n",
    "ans_dropout_2 = Dropout(rate = 0.2, seed = seed)(ans_dense_1)\n",
    " \n",
    "ans_dense_2 = Dense(units = 512, activation = 'relu', kernel_initializer = tf.keras.initializers.he_normal(seed = seed))(ans_dropout_2)\n",
    "ans_dropout_3 = Dropout(rate = 0.2, seed = seed)(ans_dense_2)\n",
    "\n",
    "ans_out = Dense(units = 9, activation = 'sigmoid', kernel_initializer = tf.keras.initializers.he_normal(seed = seed))(ans_dropout_3)\n",
    "\n",
    "# *--------------------concat--------------------*\n",
    "out = tf.concat([tb_out, ans_out], axis = -1) \n",
    " \n",
    "model = Model(inputs = [tb_input, ans_input], outputs = out)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 33
    },
    "colab_type": "code",
    "id": "9oLSZpbr0a0H",
    "outputId": "7afc8889-ec78-4625-ff29-f6e7bf64148f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`write_grads` will be ignored in TensorFlow 2.0 for the `TensorBoard` Callback.\n"
     ]
    }
   ],
   "source": [
    "# Defining callbacks\n",
    "!rm -r '/content/saved models'\n",
    "!rm -r '/content/logs'\n",
    "!mkdir '/content/saved models'\n",
    "!mkdir '/content/logs/'\n",
    " \n",
    "# tensorboard callback\n",
    "import datetime\n",
    "log_dir=\"logs/\" + datetime.datetime.now().strftime(\"%Y-%m-%d %H_%M_%S\")\n",
    "tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir = log_dir, histogram_freq=1, write_graph=True, write_grads=True)\n",
    " \n",
    "reduce_lr = tf.keras.callbacks.ReduceLROnPlateau(monitor= 'val_loss', factor=np.sqrt(0.1), patience=7, verbose=1)\n",
    " \n",
    "checkpt = tf.keras.callbacks.ModelCheckpoint('/content/saved models/weights.-{epoch:03d}-{val_loss:.5f}', monitor='val_loss', verbose=1, save_best_only=False, save_weights_only=True)\n",
    " \n",
    "print_spearman_fn = print_spearman(train_data = ([tb_USE_train, ans_USE_train], y_train.values),\n",
    "                                 validation_data = ([tb_USE_cv, ans_USE_cv], y_cv.values))\n",
    "callbacks = [print_spearman_fn, reduce_lr, checkpt, tensorboard_callback]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "hOZfE5tSRv8a",
    "outputId": "5d51483f-1574-4d64-9800-4d323015d868"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/50\n",
      " 2/84 [..............................] - ETA: 3s - loss: 0.6932 - root_mean_squared_error: 0.4166WARNING:tensorflow:Callbacks method `on_train_batch_end` is slow compared to the batch time (batch time: 0.0085s vs `on_train_batch_end` time: 0.0669s). Check your callbacks.\n",
      "75/84 [=========================>....] - ETA: 0s - loss: 0.5626 - root_mean_squared_error: 0.3341\n",
      "spearman :\n",
      "train_spearman : 0.04865 | val_spearman : 0.05349 | Learning_Rate : 7.999999797903001e-05\n",
      "\n",
      "Epoch 00001: saving model to /content/saved models/weights.-001-0.45263\n",
      "84/84 [==============================] - 1s 13ms/step - loss: 0.5521 - root_mean_squared_error: 0.3271 - val_loss: 0.4526 - val_root_mean_squared_error: 0.2563\n",
      "Epoch 2/50\n",
      "75/84 [=========================>....] - ETA: 0s - loss: 0.4317 - root_mean_squared_error: 0.2472\n",
      "spearman :\n",
      "train_spearman : 0.16816 | val_spearman : 0.16506 | Learning_Rate : 7.999999797903001e-05\n",
      "\n",
      "Epoch 00002: saving model to /content/saved models/weights.-002-0.41570\n",
      "84/84 [==============================] - 1s 10ms/step - loss: 0.4309 - root_mean_squared_error: 0.2466 - val_loss: 0.4157 - val_root_mean_squared_error: 0.2363\n",
      "Epoch 3/50\n",
      "76/84 [==========================>...] - ETA: 0s - loss: 0.4146 - root_mean_squared_error: 0.2364\n",
      "spearman :\n",
      "train_spearman : 0.20048 | val_spearman : 0.21808 | Learning_Rate : 7.999999797903001e-05\n",
      "\n",
      "Epoch 00003: saving model to /content/saved models/weights.-003-0.40312\n",
      "84/84 [==============================] - 1s 10ms/step - loss: 0.4139 - root_mean_squared_error: 0.2361 - val_loss: 0.4031 - val_root_mean_squared_error: 0.2267\n",
      "Epoch 4/50\n",
      "76/84 [==========================>...] - ETA: 0s - loss: 0.4048 - root_mean_squared_error: 0.2291\n",
      "spearman :\n",
      "train_spearman : 0.26575 | val_spearman : 0.26134 | Learning_Rate : 7.999999797903001e-05\n",
      "\n",
      "Epoch 00004: saving model to /content/saved models/weights.-004-0.39406\n",
      "84/84 [==============================] - 1s 10ms/step - loss: 0.4045 - root_mean_squared_error: 0.2287 - val_loss: 0.3941 - val_root_mean_squared_error: 0.2194\n",
      "Epoch 5/50\n",
      "76/84 [==========================>...] - ETA: 0s - loss: 0.3977 - root_mean_squared_error: 0.2233\n",
      "spearman :\n",
      "train_spearman : 0.30975 | val_spearman : 0.3101 | Learning_Rate : 7.999999797903001e-05\n",
      "\n",
      "Epoch 00005: saving model to /content/saved models/weights.-005-0.38857\n",
      "84/84 [==============================] - 1s 10ms/step - loss: 0.3972 - root_mean_squared_error: 0.2231 - val_loss: 0.3886 - val_root_mean_squared_error: 0.2150\n",
      "Epoch 6/50\n",
      "75/84 [=========================>....] - ETA: 0s - loss: 0.3916 - root_mean_squared_error: 0.2188\n",
      "spearman :\n",
      "train_spearman : 0.33698 | val_spearman : 0.3342 | Learning_Rate : 7.999999797903001e-05\n",
      "\n",
      "Epoch 00006: saving model to /content/saved models/weights.-006-0.38476\n",
      "84/84 [==============================] - 1s 11ms/step - loss: 0.3919 - root_mean_squared_error: 0.2189 - val_loss: 0.3848 - val_root_mean_squared_error: 0.2119\n",
      "Epoch 7/50\n",
      "80/84 [===========================>..] - ETA: 0s - loss: 0.3884 - root_mean_squared_error: 0.2162\n",
      "spearman :\n",
      "train_spearman : 0.334 | val_spearman : 0.33703 | Learning_Rate : 7.999999797903001e-05\n",
      "\n",
      "Epoch 00007: saving model to /content/saved models/weights.-007-0.38196\n",
      "84/84 [==============================] - 1s 11ms/step - loss: 0.3882 - root_mean_squared_error: 0.2160 - val_loss: 0.3820 - val_root_mean_squared_error: 0.2096\n",
      "Epoch 8/50\n",
      "75/84 [=========================>....] - ETA: 0s - loss: 0.3848 - root_mean_squared_error: 0.2134\n",
      "spearman :\n",
      "train_spearman : 0.36633 | val_spearman : 0.34047 | Learning_Rate : 7.999999797903001e-05\n",
      "\n",
      "Epoch 00008: saving model to /content/saved models/weights.-008-0.38008\n",
      "84/84 [==============================] - 1s 10ms/step - loss: 0.3849 - root_mean_squared_error: 0.2133 - val_loss: 0.3801 - val_root_mean_squared_error: 0.2080\n",
      "Epoch 9/50\n",
      "76/84 [==========================>...] - ETA: 0s - loss: 0.3831 - root_mean_squared_error: 0.2120\n",
      "spearman :\n",
      "train_spearman : 0.37998 | val_spearman : 0.35627 | Learning_Rate : 7.999999797903001e-05\n",
      "\n",
      "Epoch 00009: saving model to /content/saved models/weights.-009-0.37779\n",
      "84/84 [==============================] - 1s 10ms/step - loss: 0.3829 - root_mean_squared_error: 0.2119 - val_loss: 0.3778 - val_root_mean_squared_error: 0.2062\n",
      "Epoch 10/50\n",
      "75/84 [=========================>....] - ETA: 0s - loss: 0.3815 - root_mean_squared_error: 0.2103\n",
      "spearman :\n",
      "train_spearman : 0.39721 | val_spearman : 0.37235 | Learning_Rate : 7.999999797903001e-05\n",
      "\n",
      "Epoch 00010: saving model to /content/saved models/weights.-010-0.37676\n",
      "84/84 [==============================] - 1s 10ms/step - loss: 0.3809 - root_mean_squared_error: 0.2102 - val_loss: 0.3768 - val_root_mean_squared_error: 0.2053\n",
      "Epoch 11/50\n",
      "76/84 [==========================>...] - ETA: 0s - loss: 0.3787 - root_mean_squared_error: 0.2086\n",
      "spearman :\n",
      "train_spearman : 0.40684 | val_spearman : 0.38228 | Learning_Rate : 7.999999797903001e-05\n",
      "\n",
      "Epoch 00011: saving model to /content/saved models/weights.-011-0.37556\n",
      "84/84 [==============================] - 1s 10ms/step - loss: 0.3788 - root_mean_squared_error: 0.2086 - val_loss: 0.3756 - val_root_mean_squared_error: 0.2044\n",
      "Epoch 12/50\n",
      "83/84 [============================>.] - ETA: 0s - loss: 0.3773 - root_mean_squared_error: 0.2073\n",
      "spearman :\n",
      "train_spearman : 0.41609 | val_spearman : 0.38803 | Learning_Rate : 7.999999797903001e-05\n",
      "\n",
      "Epoch 00012: saving model to /content/saved models/weights.-012-0.37464\n",
      "84/84 [==============================] - 1s 11ms/step - loss: 0.3772 - root_mean_squared_error: 0.2073 - val_loss: 0.3746 - val_root_mean_squared_error: 0.2036\n",
      "Epoch 13/50\n",
      "74/84 [=========================>....] - ETA: 0s - loss: 0.3770 - root_mean_squared_error: 0.2070\n",
      "spearman :\n",
      "train_spearman : 0.42325 | val_spearman : 0.39397 | Learning_Rate : 7.999999797903001e-05\n",
      "\n",
      "Epoch 00013: saving model to /content/saved models/weights.-013-0.37423\n",
      "84/84 [==============================] - 1s 10ms/step - loss: 0.3764 - root_mean_squared_error: 0.2067 - val_loss: 0.3742 - val_root_mean_squared_error: 0.2034\n",
      "Epoch 14/50\n",
      "75/84 [=========================>....] - ETA: 0s - loss: 0.3752 - root_mean_squared_error: 0.2052\n",
      "spearman :\n",
      "train_spearman : 0.43058 | val_spearman : 0.39814 | Learning_Rate : 7.999999797903001e-05\n",
      "\n",
      "Epoch 00014: saving model to /content/saved models/weights.-014-0.37342\n",
      "84/84 [==============================] - 1s 10ms/step - loss: 0.3745 - root_mean_squared_error: 0.2053 - val_loss: 0.3734 - val_root_mean_squared_error: 0.2028\n",
      "Epoch 15/50\n",
      "82/84 [============================>.] - ETA: 0s - loss: 0.3736 - root_mean_squared_error: 0.2047\n",
      "spearman :\n",
      "train_spearman : 0.43644 | val_spearman : 0.40486 | Learning_Rate : 7.999999797903001e-05\n",
      "\n",
      "Epoch 00015: saving model to /content/saved models/weights.-015-0.37323\n",
      "84/84 [==============================] - 1s 10ms/step - loss: 0.3738 - root_mean_squared_error: 0.2047 - val_loss: 0.3732 - val_root_mean_squared_error: 0.2026\n",
      "Epoch 16/50\n",
      "76/84 [==========================>...] - ETA: 0s - loss: 0.3726 - root_mean_squared_error: 0.2042\n",
      "spearman :\n",
      "train_spearman : 0.44264 | val_spearman : 0.40517 | Learning_Rate : 7.999999797903001e-05\n",
      "\n",
      "Epoch 00016: saving model to /content/saved models/weights.-016-0.37291\n",
      "84/84 [==============================] - 1s 10ms/step - loss: 0.3728 - root_mean_squared_error: 0.2039 - val_loss: 0.3729 - val_root_mean_squared_error: 0.2023\n",
      "Epoch 17/50\n",
      "76/84 [==========================>...] - ETA: 0s - loss: 0.3719 - root_mean_squared_error: 0.2034\n",
      "spearman :\n",
      "train_spearman : 0.44801 | val_spearman : 0.4055 | Learning_Rate : 7.999999797903001e-05\n",
      "\n",
      "Epoch 00017: saving model to /content/saved models/weights.-017-0.37236\n",
      "84/84 [==============================] - 1s 11ms/step - loss: 0.3718 - root_mean_squared_error: 0.2032 - val_loss: 0.3724 - val_root_mean_squared_error: 0.2020\n",
      "Epoch 18/50\n",
      "82/84 [============================>.] - ETA: 0s - loss: 0.3714 - root_mean_squared_error: 0.2027\n",
      "spearman :\n",
      "train_spearman : 0.45044 | val_spearman : 0.40578 | Learning_Rate : 7.999999797903001e-05\n",
      "\n",
      "Epoch 00018: saving model to /content/saved models/weights.-018-0.37206\n",
      "84/84 [==============================] - 1s 11ms/step - loss: 0.3711 - root_mean_squared_error: 0.2026 - val_loss: 0.3721 - val_root_mean_squared_error: 0.2018\n",
      "Epoch 19/50\n",
      "75/84 [=========================>....] - ETA: 0s - loss: 0.3712 - root_mean_squared_error: 0.2020\n",
      "spearman :\n",
      "train_spearman : 0.45573 | val_spearman : 0.41072 | Learning_Rate : 7.999999797903001e-05\n",
      "\n",
      "Epoch 00019: saving model to /content/saved models/weights.-019-0.37149\n",
      "84/84 [==============================] - 1s 10ms/step - loss: 0.3704 - root_mean_squared_error: 0.2020 - val_loss: 0.3715 - val_root_mean_squared_error: 0.2014\n",
      "Epoch 20/50\n",
      "75/84 [=========================>....] - ETA: 0s - loss: 0.3694 - root_mean_squared_error: 0.2010\n",
      "spearman :\n",
      "train_spearman : 0.45917 | val_spearman : 0.41031 | Learning_Rate : 7.999999797903001e-05\n",
      "\n",
      "Epoch 00020: saving model to /content/saved models/weights.-020-0.37154\n",
      "84/84 [==============================] - 1s 10ms/step - loss: 0.3694 - root_mean_squared_error: 0.2013 - val_loss: 0.3715 - val_root_mean_squared_error: 0.2013\n",
      "Epoch 21/50\n",
      "82/84 [============================>.] - ETA: 0s - loss: 0.3683 - root_mean_squared_error: 0.2004\n",
      "spearman :\n",
      "train_spearman : 0.46326 | val_spearman : 0.4118 | Learning_Rate : 7.999999797903001e-05\n",
      "\n",
      "Epoch 00021: saving model to /content/saved models/weights.-021-0.37208\n",
      "84/84 [==============================] - 1s 10ms/step - loss: 0.3683 - root_mean_squared_error: 0.2004 - val_loss: 0.3721 - val_root_mean_squared_error: 0.2017\n",
      "Epoch 22/50\n",
      "74/84 [=========================>....] - ETA: 0s - loss: 0.3683 - root_mean_squared_error: 0.2005\n",
      "spearman :\n",
      "train_spearman : 0.46608 | val_spearman : 0.41287 | Learning_Rate : 7.999999797903001e-05\n",
      "\n",
      "Epoch 00022: saving model to /content/saved models/weights.-022-0.37105\n",
      "84/84 [==============================] - 1s 11ms/step - loss: 0.3681 - root_mean_squared_error: 0.2001 - val_loss: 0.3711 - val_root_mean_squared_error: 0.2010\n",
      "Epoch 23/50\n",
      "79/84 [===========================>..] - ETA: 0s - loss: 0.3677 - root_mean_squared_error: 0.1996\n",
      "spearman :\n",
      "train_spearman : 0.4685 | val_spearman : 0.4146 | Learning_Rate : 7.999999797903001e-05\n",
      "\n",
      "Epoch 00023: saving model to /content/saved models/weights.-023-0.37097\n",
      "84/84 [==============================] - 1s 10ms/step - loss: 0.3675 - root_mean_squared_error: 0.1997 - val_loss: 0.3710 - val_root_mean_squared_error: 0.2009\n",
      "Epoch 24/50\n",
      "83/84 [============================>.] - ETA: 0s - loss: 0.3667 - root_mean_squared_error: 0.1990\n",
      "spearman :\n",
      "train_spearman : 0.47194 | val_spearman : 0.41732 | Learning_Rate : 7.999999797903001e-05\n",
      "\n",
      "Epoch 00024: saving model to /content/saved models/weights.-024-0.37086\n",
      "84/84 [==============================] - 1s 10ms/step - loss: 0.3668 - root_mean_squared_error: 0.1992 - val_loss: 0.3709 - val_root_mean_squared_error: 0.2007\n",
      "Epoch 25/50\n",
      "79/84 [===========================>..] - ETA: 0s - loss: 0.3667 - root_mean_squared_error: 0.1987\n",
      "spearman :\n",
      "train_spearman : 0.47624 | val_spearman : 0.41893 | Learning_Rate : 7.999999797903001e-05\n",
      "\n",
      "Epoch 00025: saving model to /content/saved models/weights.-025-0.37025\n",
      "84/84 [==============================] - 1s 10ms/step - loss: 0.3663 - root_mean_squared_error: 0.1987 - val_loss: 0.3702 - val_root_mean_squared_error: 0.2003\n",
      "Epoch 26/50\n",
      "76/84 [==========================>...] - ETA: 0s - loss: 0.3659 - root_mean_squared_error: 0.1982\n",
      "spearman :\n",
      "train_spearman : 0.47857 | val_spearman : 0.41713 | Learning_Rate : 7.999999797903001e-05\n",
      "\n",
      "Epoch 00026: saving model to /content/saved models/weights.-026-0.37052\n",
      "84/84 [==============================] - 1s 10ms/step - loss: 0.3656 - root_mean_squared_error: 0.1983 - val_loss: 0.3705 - val_root_mean_squared_error: 0.2005\n",
      "Epoch 27/50\n",
      "73/84 [=========================>....] - ETA: 0s - loss: 0.3655 - root_mean_squared_error: 0.1978\n",
      "spearman :\n",
      "train_spearman : 0.48062 | val_spearman : 0.42146 | Learning_Rate : 7.999999797903001e-05\n",
      "\n",
      "Epoch 00027: saving model to /content/saved models/weights.-027-0.37029\n",
      "84/84 [==============================] - 1s 10ms/step - loss: 0.3649 - root_mean_squared_error: 0.1976 - val_loss: 0.3703 - val_root_mean_squared_error: 0.2004\n",
      "Epoch 28/50\n",
      "71/84 [========================>.....] - ETA: 0s - loss: 0.3645 - root_mean_squared_error: 0.1968\n",
      "spearman :\n",
      "train_spearman : 0.48476 | val_spearman : 0.42224 | Learning_Rate : 7.999999797903001e-05\n",
      "\n",
      "Epoch 00028: saving model to /content/saved models/weights.-028-0.36996\n",
      "84/84 [==============================] - 1s 11ms/step - loss: 0.3645 - root_mean_squared_error: 0.1972 - val_loss: 0.3700 - val_root_mean_squared_error: 0.2001\n",
      "Epoch 29/50\n",
      "75/84 [=========================>....] - ETA: 0s - loss: 0.3640 - root_mean_squared_error: 0.1971\n",
      "spearman :\n",
      "train_spearman : 0.48811 | val_spearman : 0.4231 | Learning_Rate : 7.999999797903001e-05\n",
      "\n",
      "Epoch 00029: saving model to /content/saved models/weights.-029-0.36995\n",
      "84/84 [==============================] - 1s 10ms/step - loss: 0.3642 - root_mean_squared_error: 0.1970 - val_loss: 0.3700 - val_root_mean_squared_error: 0.2001\n",
      "Epoch 30/50\n",
      "78/84 [==========================>...] - ETA: 0s - loss: 0.3633 - root_mean_squared_error: 0.1963\n",
      "spearman :\n",
      "train_spearman : 0.48936 | val_spearman : 0.42233 | Learning_Rate : 7.999999797903001e-05\n",
      "\n",
      "Epoch 00030: saving model to /content/saved models/weights.-030-0.37000\n",
      "84/84 [==============================] - 1s 11ms/step - loss: 0.3636 - root_mean_squared_error: 0.1965 - val_loss: 0.3700 - val_root_mean_squared_error: 0.2002\n",
      "Epoch 31/50\n",
      "74/84 [=========================>....] - ETA: 0s - loss: 0.3628 - root_mean_squared_error: 0.1957\n",
      "spearman :\n",
      "train_spearman : 0.49227 | val_spearman : 0.42306 | Learning_Rate : 7.999999797903001e-05\n",
      "\n",
      "Epoch 00031: saving model to /content/saved models/weights.-031-0.36978\n",
      "84/84 [==============================] - 1s 10ms/step - loss: 0.3627 - root_mean_squared_error: 0.1957 - val_loss: 0.3698 - val_root_mean_squared_error: 0.1999\n",
      "Epoch 32/50\n",
      "76/84 [==========================>...] - ETA: 0s - loss: 0.3628 - root_mean_squared_error: 0.1952\n",
      "spearman :\n",
      "train_spearman : 0.49376 | val_spearman : 0.42497 | Learning_Rate : 7.999999797903001e-05\n",
      "\n",
      "Epoch 00032: saving model to /content/saved models/weights.-032-0.36934\n",
      "84/84 [==============================] - 1s 10ms/step - loss: 0.3626 - root_mean_squared_error: 0.1956 - val_loss: 0.3693 - val_root_mean_squared_error: 0.1997\n",
      "Epoch 33/50\n",
      "76/84 [==========================>...] - ETA: 0s - loss: 0.3626 - root_mean_squared_error: 0.1953\n",
      "spearman :\n",
      "train_spearman : 0.49697 | val_spearman : 0.42585 | Learning_Rate : 7.999999797903001e-05\n",
      "\n",
      "Epoch 00033: saving model to /content/saved models/weights.-033-0.36993\n",
      "84/84 [==============================] - 1s 10ms/step - loss: 0.3626 - root_mean_squared_error: 0.1957 - val_loss: 0.3699 - val_root_mean_squared_error: 0.2001\n",
      "Epoch 34/50\n",
      "74/84 [=========================>....] - ETA: 0s - loss: 0.3619 - root_mean_squared_error: 0.1948\n",
      "spearman :\n",
      "train_spearman : 0.49876 | val_spearman : 0.42937 | Learning_Rate : 7.999999797903001e-05\n",
      "\n",
      "Epoch 00034: saving model to /content/saved models/weights.-034-0.37013\n",
      "84/84 [==============================] - 1s 11ms/step - loss: 0.3616 - root_mean_squared_error: 0.1948 - val_loss: 0.3701 - val_root_mean_squared_error: 0.2002\n",
      "Epoch 35/50\n",
      "72/84 [========================>.....] - ETA: 0s - loss: 0.3610 - root_mean_squared_error: 0.1940\n",
      "spearman :\n",
      "train_spearman : 0.50215 | val_spearman : 0.42978 | Learning_Rate : 7.999999797903001e-05\n",
      "\n",
      "Epoch 00035: saving model to /content/saved models/weights.-035-0.36960\n",
      "84/84 [==============================] - 1s 11ms/step - loss: 0.3609 - root_mean_squared_error: 0.1942 - val_loss: 0.3696 - val_root_mean_squared_error: 0.1998\n",
      "Epoch 36/50\n",
      "74/84 [=========================>....] - ETA: 0s - loss: 0.3603 - root_mean_squared_error: 0.1940\n",
      "spearman :\n",
      "train_spearman : 0.50513 | val_spearman : 0.42786 | Learning_Rate : 7.999999797903001e-05\n",
      "\n",
      "Epoch 00036: saving model to /content/saved models/weights.-036-0.36953\n",
      "84/84 [==============================] - 1s 10ms/step - loss: 0.3608 - root_mean_squared_error: 0.1942 - val_loss: 0.3695 - val_root_mean_squared_error: 0.1997\n",
      "Epoch 37/50\n",
      "81/84 [===========================>..] - ETA: 0s - loss: 0.3604 - root_mean_squared_error: 0.1941\n",
      "spearman :\n",
      "train_spearman : 0.4949 | val_spearman : 0.42903 | Learning_Rate : 7.999999797903001e-05\n",
      "\n",
      "Epoch 00037: saving model to /content/saved models/weights.-037-0.36933\n",
      "84/84 [==============================] - 1s 10ms/step - loss: 0.3605 - root_mean_squared_error: 0.1939 - val_loss: 0.3693 - val_root_mean_squared_error: 0.1995\n",
      "Epoch 38/50\n",
      "79/84 [===========================>..] - ETA: 0s - loss: 0.3592 - root_mean_squared_error: 0.1933\n",
      "spearman :\n",
      "train_spearman : 0.49607 | val_spearman : 0.4275 | Learning_Rate : 7.999999797903001e-05\n",
      "\n",
      "Epoch 00038: saving model to /content/saved models/weights.-038-0.36889\n",
      "84/84 [==============================] - 1s 10ms/step - loss: 0.3595 - root_mean_squared_error: 0.1931 - val_loss: 0.3689 - val_root_mean_squared_error: 0.1992\n",
      "Epoch 39/50\n",
      "76/84 [==========================>...] - ETA: 0s - loss: 0.3597 - root_mean_squared_error: 0.1933\n",
      "spearman :\n",
      "train_spearman : 0.49899 | val_spearman : 0.42851 | Learning_Rate : 7.999999797903001e-05\n",
      "\n",
      "Epoch 00039: saving model to /content/saved models/weights.-039-0.36880\n",
      "84/84 [==============================] - 1s 11ms/step - loss: 0.3594 - root_mean_squared_error: 0.1929 - val_loss: 0.3688 - val_root_mean_squared_error: 0.1992\n",
      "Epoch 40/50\n",
      "75/84 [=========================>....] - ETA: 0s - loss: 0.3587 - root_mean_squared_error: 0.1919\n",
      "spearman :\n",
      "train_spearman : 0.50118 | val_spearman : 0.4302 | Learning_Rate : 7.999999797903001e-05\n",
      "\n",
      "Epoch 00040: saving model to /content/saved models/weights.-040-0.36916\n",
      "84/84 [==============================] - 1s 11ms/step - loss: 0.3589 - root_mean_squared_error: 0.1926 - val_loss: 0.3692 - val_root_mean_squared_error: 0.1994\n",
      "Epoch 41/50\n",
      "79/84 [===========================>..] - ETA: 0s - loss: 0.3587 - root_mean_squared_error: 0.1923\n",
      "spearman :\n",
      "train_spearman : 0.50309 | val_spearman : 0.43018 | Learning_Rate : 7.999999797903001e-05\n",
      "\n",
      "Epoch 00041: saving model to /content/saved models/weights.-041-0.36900\n",
      "84/84 [==============================] - 1s 10ms/step - loss: 0.3586 - root_mean_squared_error: 0.1922 - val_loss: 0.3690 - val_root_mean_squared_error: 0.1993\n",
      "Epoch 42/50\n",
      "72/84 [========================>.....] - ETA: 0s - loss: 0.3578 - root_mean_squared_error: 0.1919\n",
      "spearman :\n",
      "train_spearman : 0.50535 | val_spearman : 0.43262 | Learning_Rate : 7.999999797903001e-05\n",
      "\n",
      "Epoch 00042: saving model to /content/saved models/weights.-042-0.36904\n",
      "84/84 [==============================] - 1s 10ms/step - loss: 0.3579 - root_mean_squared_error: 0.1916 - val_loss: 0.3690 - val_root_mean_squared_error: 0.1993\n",
      "Epoch 43/50\n",
      "74/84 [=========================>....] - ETA: 0s - loss: 0.3577 - root_mean_squared_error: 0.1913\n",
      "spearman :\n",
      "train_spearman : 0.50807 | val_spearman : 0.43046 | Learning_Rate : 7.999999797903001e-05\n",
      "\n",
      "Epoch 00043: saving model to /content/saved models/weights.-043-0.36922\n",
      "84/84 [==============================] - 1s 10ms/step - loss: 0.3577 - root_mean_squared_error: 0.1915 - val_loss: 0.3692 - val_root_mean_squared_error: 0.1995\n",
      "Epoch 44/50\n",
      "76/84 [==========================>...] - ETA: 0s - loss: 0.3568 - root_mean_squared_error: 0.1905\n",
      "spearman :\n",
      "train_spearman : 0.51077 | val_spearman : 0.43133 | Learning_Rate : 7.999999797903001e-05\n",
      "\n",
      "Epoch 00044: saving model to /content/saved models/weights.-044-0.36904\n",
      "84/84 [==============================] - 1s 10ms/step - loss: 0.3570 - root_mean_squared_error: 0.1909 - val_loss: 0.3690 - val_root_mean_squared_error: 0.1993\n",
      "Epoch 45/50\n",
      "75/84 [=========================>....] - ETA: 0s - loss: 0.3573 - root_mean_squared_error: 0.1912\n",
      "spearman :\n",
      "train_spearman : 0.5139 | val_spearman : 0.43263 | Learning_Rate : 7.999999797903001e-05\n",
      "\n",
      "Epoch 00045: saving model to /content/saved models/weights.-045-0.36828\n",
      "84/84 [==============================] - 1s 10ms/step - loss: 0.3568 - root_mean_squared_error: 0.1908 - val_loss: 0.3683 - val_root_mean_squared_error: 0.1988\n",
      "Epoch 46/50\n",
      "78/84 [==========================>...] - ETA: 0s - loss: 0.3569 - root_mean_squared_error: 0.1910\n",
      "spearman :\n",
      "train_spearman : 0.51537 | val_spearman : 0.43412 | Learning_Rate : 7.999999797903001e-05\n",
      "\n",
      "Epoch 00046: saving model to /content/saved models/weights.-046-0.36873\n",
      "84/84 [==============================] - 1s 10ms/step - loss: 0.3570 - root_mean_squared_error: 0.1909 - val_loss: 0.3687 - val_root_mean_squared_error: 0.1992\n",
      "Epoch 47/50\n",
      "69/84 [=======================>......] - ETA: 0s - loss: 0.3557 - root_mean_squared_error: 0.1899\n",
      "spearman :\n",
      "train_spearman : 0.51813 | val_spearman : 0.43176 | Learning_Rate : 7.999999797903001e-05\n",
      "\n",
      "Epoch 00047: saving model to /content/saved models/weights.-047-0.36903\n",
      "84/84 [==============================] - 1s 10ms/step - loss: 0.3559 - root_mean_squared_error: 0.1898 - val_loss: 0.3690 - val_root_mean_squared_error: 0.1994\n",
      "Epoch 48/50\n",
      "75/84 [=========================>....] - ETA: 0s - loss: 0.3549 - root_mean_squared_error: 0.1894\n",
      "spearman :\n",
      "train_spearman : 0.5203 | val_spearman : 0.43176 | Learning_Rate : 7.999999797903001e-05\n",
      "\n",
      "Epoch 00048: saving model to /content/saved models/weights.-048-0.36898\n",
      "84/84 [==============================] - 1s 10ms/step - loss: 0.3555 - root_mean_squared_error: 0.1895 - val_loss: 0.3690 - val_root_mean_squared_error: 0.1993\n",
      "Epoch 49/50\n",
      "74/84 [=========================>....] - ETA: 0s - loss: 0.3548 - root_mean_squared_error: 0.1889\n",
      "spearman :\n",
      "train_spearman : 0.52245 | val_spearman : 0.43661 | Learning_Rate : 7.999999797903001e-05\n",
      "\n",
      "Epoch 00049: saving model to /content/saved models/weights.-049-0.36893\n",
      "84/84 [==============================] - 1s 10ms/step - loss: 0.3546 - root_mean_squared_error: 0.1886 - val_loss: 0.3689 - val_root_mean_squared_error: 0.1992\n",
      "Epoch 50/50\n",
      "76/84 [==========================>...] - ETA: 0s - loss: 0.3543 - root_mean_squared_error: 0.1887\n",
      "spearman :\n",
      "train_spearman : 0.52512 | val_spearman : 0.43424 | Learning_Rate : 7.999999797903001e-05\n",
      "\n",
      "Epoch 00050: saving model to /content/saved models/weights.-050-0.36888\n",
      "84/84 [==============================] - 1s 10ms/step - loss: 0.3546 - root_mean_squared_error: 0.1887 - val_loss: 0.3689 - val_root_mean_squared_error: 0.1992\n"
     ]
    }
   ],
   "source": [
    "# LSTM : training a model\n",
    "tf.keras.backend.clear_session()\n",
    "opt = tf.keras.optimizers.Adam(learning_rate = 0.00008)\n",
    "rmse = tf.keras.metrics.RootMeanSquaredError()\n",
    "\n",
    "model.compile(loss = 'binary_crossentropy', optimizer = opt,  metrics = [rmse])\n",
    "history = model.fit(x = [tb_USE_train, ans_USE_train], y =  y_train.values,\n",
    "                    validation_data = ([tb_USE_cv, ans_USE_cv], y_cv.values),\n",
    "                    batch_size = 64, epochs = 50, callbacks = callbacks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 33
    },
    "colab_type": "code",
    "id": "Pw2UMhwZRzVP",
    "outputId": "67a8e0e5-61be-4e82-ea6f-591f0ef14c1f"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.43661, 48)"
      ]
     },
     "execution_count": 34,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# best_results : (val_spearman, epoch_no)\n",
    "max(print_spearman_fn.spearman_dict['val_spearman']), np.argmax(print_spearman_fn.spearman_dict['val_spearman'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "nC7qxGeu9BLT"
   },
   "source": [
    "# 4.3. Sentence Embedding : RoBERTa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 570
    },
    "colab_type": "code",
    "id": "p0Enx-MN9BLj",
    "outputId": "6245fbdc-a28a-42a5-a358-dc3e83dbbe2d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting transformers\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/27/3c/91ed8f5c4e7ef3227b4119200fc0ed4b4fd965b1f0172021c25701087825/transformers-3.0.2-py3-none-any.whl (769kB)\n",
      "\u001b[K     |████████████████████████████████| 778kB 2.8MB/s \n",
      "\u001b[?25hCollecting sacremoses\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n",
      "\u001b[K     |████████████████████████████████| 890kB 14.3MB/s \n",
      "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers) (3.0.12)\n",
      "Collecting tokenizers==0.8.1.rc1\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/40/d0/30d5f8d221a0ed981a186c8eb986ce1c94e3a6e87f994eae9f4aa5250217/tokenizers-0.8.1rc1-cp36-cp36m-manylinux1_x86_64.whl (3.0MB)\n",
      "\u001b[K     |████████████████████████████████| 3.0MB 20.3MB/s \n",
      "\u001b[?25hCollecting sentencepiece!=0.1.92\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d4/a4/d0a884c4300004a78cca907a6ff9a5e9fe4f090f5d95ab341c53d28cbc58/sentencepiece-0.1.91-cp36-cp36m-manylinux1_x86_64.whl (1.1MB)\n",
      "\u001b[K     |████████████████████████████████| 1.1MB 37.7MB/s \n",
      "\u001b[?25hRequirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from transformers) (20.4)\n",
      "Requirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers) (0.7)\n",
      "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers) (4.41.1)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (2019.12.20)\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers) (1.18.5)\n",
      "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers) (2.23.0)\n",
      "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (1.15.0)\n",
      "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (7.1.2)\n",
      "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (0.16.0)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->transformers) (2.4.7)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2020.6.20)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (1.24.3)\n",
      "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (3.0.4)\n",
      "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2.10)\n",
      "Building wheels for collected packages: sacremoses\n",
      "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for sacremoses: filename=sacremoses-0.0.43-cp36-none-any.whl size=893260 sha256=a6d1c1aca66db3ca7511a0e432fccbe854e961b27b22d704870ee4ed800338d0\n",
      "  Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n",
      "Successfully built sacremoses\n",
      "Installing collected packages: sacremoses, tokenizers, sentencepiece, transformers\n",
      "Successfully installed sacremoses-0.0.43 sentencepiece-0.1.91 tokenizers-0.8.1rc1 transformers-3.0.2\n"
     ]
    }
   ],
   "source": [
    "!pip install transformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 118
    },
    "colab_type": "code",
    "id": "iQ9PYY1A9BL4",
    "outputId": "5e4139ea-c343-4b3e-8d28-d8a7bedf966e"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:transformers.modeling_tf_utils:Some weights of the model checkpoint at roberta-base were not used when initializing TFRobertaModel: ['lm_head']\n",
      "- This IS expected if you are initializing TFRobertaModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPretraining model).\n",
      "- This IS NOT expected if you are initializing TFRobertaModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "WARNING:transformers.modeling_tf_utils:All the weights of TFRobertaModel were initialized from the model checkpoint at roberta-base.\n",
      "If your task is similar to the task the model of the ckeckpoint was trained on, you can already use TFRobertaModel for predictions without further training.\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "from transformers import RobertaConfig, RobertaTokenizer, TFRobertaModel\n",
    " \n",
    "# Load pretrained model/tokenizer\n",
    "config = RobertaConfig.from_pretrained('roberta-base', output_hidden_states=True)\n",
    "roberta_tokenizer = RobertaTokenizer.from_pretrained('roberta-base')\n",
    "roberta_model = TFRobertaModel.from_pretrained('roberta-base', config = config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 50
    },
    "colab_type": "code",
    "id": "TLTW2MgUMDB2",
    "outputId": "5a6a8f81-74e2-45c6-a38a-022bdcd4414d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5349, 512) (5349, 512) (730, 512) (730, 512)\n",
      "(5349, 300) (5349, 300) (730, 300) (730, 300)\n"
     ]
    }
   ],
   "source": [
    "# Tokenizing : title_body\n",
    "title_body_train_tokens = roberta_tokenizer.batch_encode_plus(title_body_train, max_length = 512, truncation = True, pad_to_max_length = True, return_tensors=\"tf\")\n",
    "title_body_cv_tokens = roberta_tokenizer.batch_encode_plus(title_body_cv, max_length = 512, truncation = True, pad_to_max_length = True, return_tensors=\"tf\")\n",
    "# tokens = bert_tokenizer.batch_encode_plus(final_train, max_length = 50, truncation = True, pad_to_max_length = True)\n",
    "\n",
    "tb_train_input_ids  = np.array(title_body_train_tokens['input_ids'])\n",
    "tb_train_attn_mask = np.array(title_body_train_tokens['attention_mask'])\n",
    " \n",
    "tb_cv_input_ids  = np.array(title_body_cv_tokens['input_ids'])\n",
    "tb_cv_attn_mask = np.array(title_body_cv_tokens['attention_mask'])\n",
    "\n",
    "print(tb_train_input_ids.shape, tb_train_attn_mask.shape, tb_cv_input_ids.shape, tb_cv_attn_mask.shape)\n",
    "\n",
    "# Tokenizing : answer\n",
    "ans_train_tokens = roberta_tokenizer.batch_encode_plus(answer_train, max_length = 300, truncation = True, pad_to_max_length = True, return_tensors=\"tf\")\n",
    "ans_cv_tokens = roberta_tokenizer.batch_encode_plus(answer_cv, max_length = 300, truncation = True, pad_to_max_length = True, return_tensors=\"tf\")\n",
    "# tokens = bert_tokenizer.batch_encode_plus(final_train, max_length = 50, truncation = True, pad_to_max_length = True)\n",
    " \n",
    "ans_train_input_ids  = np.array(ans_train_tokens['input_ids'])\n",
    "ans_train_attn_mask = np.array(ans_train_tokens['attention_mask'])\n",
    " \n",
    "ans_cv_input_ids  = np.array(ans_cv_tokens['input_ids'])\n",
    "ans_cv_attn_mask = np.array(ans_cv_tokens['attention_mask'])\n",
    "seq_len = ans_train_input_ids.shape[1]\n",
    " \n",
    "print(ans_train_input_ids.shape, ans_train_attn_mask.shape, ans_cv_input_ids.shape, ans_cv_attn_mask.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "BW3kZ5BR9BMN"
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "tf.compat.v1.enable_eager_execution()\n",
    "from tensorflow.keras.layers import Input, Dense, Dropout\n",
    "from tensorflow.keras.models import Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 146,
     "referenced_widgets": [
      "846aa2b5f5ea45149f8c924f8601f4c3",
      "7b296c3ac88147bca4b362b152d7323c",
      "931619a17d8d4b1f9abb071fb0a17b7a",
      "afc8cb74078d46399b8e4e5628942ae7",
      "88bb3253383f4ac2b6fc3102b147d1ca",
      "342b574c8e67464f8baa8fc83cc5cf56",
      "225c0b674051430c98c2bc111d5c352a",
      "d8079814c7794191b31a431d7a1e940b",
      "0c09f1af502c4211855374ceb2d9f185",
      "7caee6dc1d87471c81a7a7eac39b78de",
      "1b337bbed0d84d34940f37ef65bec3e3",
      "1844897a82f848c1885097865c6bc33d",
      "fdbd355e29164575b4140c9da8b28705",
      "d265236a86d34e898315726538d389aa",
      "fadb842104cc433fb0b7010f79cfd631",
      "3bdfc1ceea0f4b85a2c373a28517e0c9"
     ]
    },
    "colab_type": "code",
    "id": "2524QqfEO6tw",
    "outputId": "39b4cfe0-8f94-4306-ba9e-d0247162ffa7"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "846aa2b5f5ea45149f8c924f8601f4c3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=168.0), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "(5349, 3072)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0c09f1af502c4211855374ceb2d9f185",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "(730, 3072)\n"
     ]
    }
   ],
   "source": [
    "# train : title_body RoBERTa embedings\n",
    "batch_size = 32\n",
    "l = tb_train_input_ids.shape[0]\n",
    "\n",
    "lst1 = []\n",
    "for i in tqdm(range((l//batch_size)+1)):\n",
    "  x = roberta_model([tb_train_input_ids[i*batch_size:(i+1)*batch_size], tb_train_attn_mask[i*batch_size:(i+1)*batch_size]])[2]\n",
    "\n",
    "  # getting hidden states of last 4 layers (avg of each timrstep)\n",
    "  lst = [i[:, 0, :] for i in x[-4:]]\n",
    "\n",
    "  # concat all 4 averaged hidden_states\n",
    "  conc = tf.concat(lst, axis = 1)\n",
    "  lst1.append(conc)\n",
    "tb_RoBERTa_train = tf.concat(lst1, axis = 0)\n",
    "print(tb_RoBERTa_train.shape)\n",
    "\n",
    "# cv : title_body RoBERTa embedings\n",
    "batch_size = 32\n",
    "l = ans_cv_input_ids.shape[0]\n",
    "\n",
    "lst1 = []\n",
    "for i in tqdm(range((l//batch_size)+1)):\n",
    "  x = roberta_model([tb_cv_input_ids[i*batch_size:(i+1)*batch_size], tb_cv_attn_mask[i*batch_size:(i+1)*batch_size]])[2]\n",
    "  # getting hidden states of last 4 layers (avg of each timrstep)\n",
    "  lst = [i[:, 0, :] for i in x[-4:]]\n",
    "  # lst = [i[:, 0, :] for i in x[-4:]]\n",
    "  conc = tf.concat(lst, axis = 1)\n",
    "  lst1.append(conc)\n",
    "tb_RoBERTa_cv = tf.concat(lst1, axis = 0)\n",
    "print(tb_RoBERTa_cv.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 146,
     "referenced_widgets": [
      "24ee8f7fbdfe45c49ec5452a2bd39e89",
      "2c7cf14ab33f40d9be631dd56c7b1b20",
      "bd5c006a141a4e80922ee048ae04c9f7",
      "4e6eec0b36b64445b40b9753d51f677f",
      "630d5c3d71b84179a0b60fbc6c1a7b9c",
      "29f5625daa904c2fbf068770bacd99de",
      "02c40c9d125e48da9255863bcbcf1f83",
      "ba5a8b57f4994ff09454253301a194c6",
      "16ff17f3143249638e26ccce45758c7e",
      "0648904a662a4886a0f079a6cddb4c70",
      "0618f78ac6c74b35a3fe2c1fe4ca03aa",
      "492a78cbbfcb4298b2208e9b657a9a05",
      "8d78d026d9da4ae5b456e2bbfaa3169c",
      "1f5ba81b4e7d4d959a09d8ba149d5d59",
      "8120d7023386492a8a2adc64b6e2ad9e",
      "5c22116ba2dd4f30921082473d55feb2"
     ]
    },
    "colab_type": "code",
    "id": "hekWFr4PP7xj",
    "outputId": "4e67b6ec-7197-4aba-c130-60e03de26420"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "24ee8f7fbdfe45c49ec5452a2bd39e89",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=168.0), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "(5349, 3072)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "16ff17f3143249638e26ccce45758c7e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "(730, 3072)\n"
     ]
    }
   ],
   "source": [
    "# train : ans RoBERTa embedings\n",
    "batch_size = 32\n",
    "l = ans_train_input_ids.shape[0]\n",
    "\n",
    "lst1 = []\n",
    "for i in tqdm(range((l//batch_size)+1)):\n",
    "  x = roberta_model([ans_train_input_ids[i*batch_size:(i+1)*batch_size], ans_train_attn_mask[i*batch_size:(i+1)*batch_size]])[2]\n",
    "  # getting hidden states of last 4 layers (avg of each timrstep)\n",
    "  lst = [i[:, 0, :] for i in x[-4:]]\n",
    "  conc = tf.concat(lst, axis = 1)\n",
    "  lst1.append(conc)\n",
    "ans_RoBERTa_train = tf.concat(lst1, axis = 0)\n",
    "print(ans_RoBERTa_train.shape)\n",
    "\n",
    "# cv : ans BERT embedings\n",
    "batch_size = 32\n",
    "l = ans_cv_input_ids.shape[0]\n",
    "lst1 = []\n",
    "for i in tqdm(range((l//batch_size)+1)):\n",
    "  x = roberta_model([ans_cv_input_ids[i*batch_size:(i+1)*batch_size], ans_cv_attn_mask[i*batch_size:(i+1)*batch_size]])[2]\n",
    "  # getting hidden states of last 4 layers (avg of each timrstep)\n",
    "  lst = [i[:, 0, :] for i in x[-4:]]\n",
    "  conc = tf.concat(lst, axis = 1)\n",
    "  lst1.append(conc)\n",
    "ans_RoBERTa_cv = tf.concat(lst1, axis = 0)\n",
    "print(ans_RoBERTa_cv.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "m_2prHgEYIUv"
   },
   "outputs": [],
   "source": [
    "# saving roberta pretrained embeddings\n",
    "np.savez_compressed('/content/drive/My Drive/AAIC Course/Personal case study 2  - Google Quest/roberta_outputs', tb_RoBERTa_train = tb_RoBERTa_train.numpy(), tb_RoBERTa_cv = tb_RoBERTa_cv.numpy(), \n",
    "                    ans_RoBERTa_train = ans_RoBERTa_train.numpy(), ans_RoBERTa_cv = ans_RoBERTa_cv.numpy())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "EuLU1HXIYRjd"
   },
   "source": [
    "### Modelling :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "woe14ld6_ptV"
   },
   "outputs": [],
   "source": [
    "# loading roberta pretrained embeddings\n",
    "tb_RoBERTa_train = np.load('/content/drive/My Drive/AAIC Course/Personal case study 2  - Google Quest/roberta_outputs.npz')['tb_RoBERTa_train']\n",
    "tb_RoBERTa_cv = np.load('/content/drive/My Drive/AAIC Course/Personal case study 2  - Google Quest/roberta_outputs.npz')['tb_RoBERTa_cv']\n",
    "ans_RoBERTa_train = np.load('/content/drive/My Drive/AAIC Course/Personal case study 2  - Google Quest/roberta_outputs.npz')['ans_RoBERTa_train']\n",
    "ans_RoBERTa_cv = np.load('/content/drive/My Drive/AAIC Course/Personal case study 2  - Google Quest/roberta_outputs.npz')['ans_RoBERTa_cv']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 700
    },
    "colab_type": "code",
    "id": "Jc7Oucx2Ooq7",
    "outputId": "0cfbaf7c-24c4-4ac4-e30d-55c4c15d51bd"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"functional_1\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "tb_roberta_out (InputLayer)     [(None, 3072)]       0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dropout (Dropout)               (None, 3072)         0           tb_roberta_out[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dense (Dense)                   (None, 1024)         3146752     dropout[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)             (None, 1024)         0           dense[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 512)          524800      dropout_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dropout_2 (Dropout)             (None, 512)          0           dense_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 21)           10773       dropout_2[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "ans_roberta_out (InputLayer)    [(None, 3072)]       0                                            \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_concat (TensorFlowO [(None, 3093)]       0           dense_2[0][0]                    \n",
      "                                                                 ans_roberta_out[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_3 (Dropout)             (None, 3093)         0           tf_op_layer_concat[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "dense_3 (Dense)                 (None, 1024)         3168256     dropout_3[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dropout_4 (Dropout)             (None, 1024)         0           dense_3[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_4 (Dense)                 (None, 512)          524800      dropout_4[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dropout_5 (Dropout)             (None, 512)          0           dense_4[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_5 (Dense)                 (None, 9)            4617        dropout_5[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_concat_1 (TensorFlo [(None, 30)]         0           dense_2[0][0]                    \n",
      "                                                                 dense_5[0][0]                    \n",
      "==================================================================================================\n",
      "Total params: 7,379,998\n",
      "Trainable params: 7,379,998\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "tf.keras.backend.clear_session()\n",
    "seed = 42\n",
    "\n",
    "# *-----------------title_body-----------------*\n",
    "tb_input = Input(name = 'tb_roberta_out', shape = (768*4,), dtype = 'float32')\n",
    "tb_dropout_1 = Dropout(rate = 0.2, seed = seed)(tb_input)\n",
    " \n",
    "tb_dense_1 = Dense(units = 1024, activation = 'relu', kernel_initializer = tf.keras.initializers.he_normal(seed = seed))(tb_dropout_1)\n",
    "tb_dropout_2 = Dropout(rate = 0.2, seed = seed)(tb_dense_1)\n",
    " \n",
    "tb_dense_2 = Dense(units = 512, activation = 'relu', kernel_initializer = tf.keras.initializers.he_normal(seed = seed))(tb_dropout_2)\n",
    "tb_dropout_3 = Dropout(rate = 0.1, seed = seed)(tb_dense_2)\n",
    "\n",
    "tb_out = Dense(units = 21, activation = 'sigmoid', kernel_initializer = tf.keras.initializers.he_normal(seed = seed))(tb_dropout_3)\n",
    " \n",
    "# *-----------------answer_model-----------------*\n",
    "ans_input = Input(name = 'ans_roberta_out', shape = (768*4,), dtype = 'float32')\n",
    "concat = tf.concat([tb_out, ans_input], axis = -1) \n",
    "ans_dropout_1 = Dropout(rate = 0.2, seed = seed)(concat)\n",
    " \n",
    "ans_dense_1 = Dense(units = 1024, activation = 'relu', kernel_initializer = tf.keras.initializers.he_normal(seed = seed))(ans_dropout_1)\n",
    "ans_dropout_2 = Dropout(rate = 0.2, seed = seed)(ans_dense_1)\n",
    " \n",
    "ans_dense_2 = Dense(units = 512, activation = 'relu', kernel_initializer = tf.keras.initializers.he_normal(seed = seed))(ans_dropout_2)\n",
    "ans_dropout_3 = Dropout(rate = 0.1, seed = seed)(ans_dense_2)\n",
    "\n",
    "ans_out = Dense(units = 9, activation = 'sigmoid', kernel_initializer = tf.keras.initializers.he_normal(seed = seed))(ans_dropout_3)\n",
    "\n",
    "# *--------------------concat--------------------*\n",
    "out = tf.concat([tb_out, ans_out], axis = -1) \n",
    " \n",
    "model = Model(inputs = [tb_input, ans_input], outputs = out)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 33
    },
    "colab_type": "code",
    "id": "IauovVnD9BM9",
    "outputId": "b602084d-c2a1-4b48-d3ee-4893728a80c1"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`write_grads` will be ignored in TensorFlow 2.0 for the `TensorBoard` Callback.\n"
     ]
    }
   ],
   "source": [
    "# Defining callbacks\n",
    "!rm -r '/content/saved models'\n",
    "!rm -r '/content/logs'\n",
    "!mkdir '/content/saved models'\n",
    "!mkdir '/content/logs/'\n",
    " \n",
    "# tensorboard callback\n",
    "import datetime\n",
    "log_dir=\"logs/\" + datetime.datetime.now().strftime(\"%Y-%m-%d %H_%M_%S\")\n",
    "tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir = log_dir, histogram_freq=1, write_graph=True, write_grads=True)\n",
    " \n",
    "reduce_lr = tf.keras.callbacks.ReduceLROnPlateau(monitor= 'val_loss', factor=np.sqrt(0.1), patience=7, verbose=1)\n",
    " \n",
    "checkpt = tf.keras.callbacks.ModelCheckpoint('/content/saved models/weights.-{epoch:03d}-{val_loss:.5f}', monitor='val_loss', verbose=1, save_best_only=False, save_weights_only=True)\n",
    " \n",
    "print_spearman_fn = print_spearman(train_data = ([tb_RoBERTa_train, ans_RoBERTa_train], y_train.values),\n",
    "                                 validation_data = ([tb_RoBERTa_cv, ans_RoBERTa_cv], y_cv.values))\n",
    "callbacks = [print_spearman_fn, reduce_lr, checkpt, tensorboard_callback]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "IdG-CwXgAjND",
    "outputId": "7c5f4426-5537-4818-8299-641bd82d7f33"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/75\n",
      " 2/84 [..............................] - ETA: 3s - loss: 0.5097 - root_mean_squared_error: 0.2912WARNING:tensorflow:Callbacks method `on_train_batch_end` is slow compared to the batch time (batch time: 0.0082s vs `on_train_batch_end` time: 0.0669s). Check your callbacks.\n",
      "77/84 [==========================>...] - ETA: 0s - loss: 0.4304 - root_mean_squared_error: 0.2495\n",
      "spearman :\n",
      "train_spearman : 0.24809 | val_spearman : 0.24901 | Learning_Rate : 0.0010000000474974513\n",
      "\n",
      "Epoch 00001: saving model to /content/saved models/weights.-001-0.41228\n",
      "84/84 [==============================] - 2s 20ms/step - loss: 0.4291 - root_mean_squared_error: 0.2484 - val_loss: 0.4123 - val_root_mean_squared_error: 0.2344\n",
      "Epoch 2/75\n",
      "83/84 [============================>.] - ETA: 0s - loss: 0.4070 - root_mean_squared_error: 0.2320\n",
      "spearman :\n",
      "train_spearman : 0.27542 | val_spearman : 0.27254 | Learning_Rate : 0.0010000000474974513\n",
      "\n",
      "Epoch 00002: saving model to /content/saved models/weights.-002-0.40281\n",
      "84/84 [==============================] - 2s 18ms/step - loss: 0.4069 - root_mean_squared_error: 0.2320 - val_loss: 0.4028 - val_root_mean_squared_error: 0.2284\n",
      "Epoch 3/75\n",
      "82/84 [============================>.] - ETA: 0s - loss: 0.4019 - root_mean_squared_error: 0.2281\n",
      "spearman :\n",
      "train_spearman : 0.29459 | val_spearman : 0.29435 | Learning_Rate : 0.0010000000474974513\n",
      "\n",
      "Epoch 00003: saving model to /content/saved models/weights.-003-0.39780\n",
      "84/84 [==============================] - 1s 17ms/step - loss: 0.4017 - root_mean_squared_error: 0.2280 - val_loss: 0.3978 - val_root_mean_squared_error: 0.2236\n",
      "Epoch 4/75\n",
      "83/84 [============================>.] - ETA: 0s - loss: 0.3984 - root_mean_squared_error: 0.2254\n",
      "spearman :\n",
      "train_spearman : 0.31958 | val_spearman : 0.30907 | Learning_Rate : 0.0010000000474974513\n",
      "\n",
      "Epoch 00004: saving model to /content/saved models/weights.-004-0.39491\n",
      "84/84 [==============================] - 1s 17ms/step - loss: 0.3983 - root_mean_squared_error: 0.2254 - val_loss: 0.3949 - val_root_mean_squared_error: 0.2208\n",
      "Epoch 5/75\n",
      "82/84 [============================>.] - ETA: 0s - loss: 0.3936 - root_mean_squared_error: 0.2217\n",
      "spearman :\n",
      "train_spearman : 0.32488 | val_spearman : 0.31364 | Learning_Rate : 0.0010000000474974513\n",
      "\n",
      "Epoch 00005: saving model to /content/saved models/weights.-005-0.39732\n",
      "84/84 [==============================] - 1s 18ms/step - loss: 0.3939 - root_mean_squared_error: 0.2218 - val_loss: 0.3973 - val_root_mean_squared_error: 0.2230\n",
      "Epoch 6/75\n",
      "84/84 [==============================] - ETA: 0s - loss: 0.3925 - root_mean_squared_error: 0.2207\n",
      "spearman :\n",
      "train_spearman : 0.32311 | val_spearman : 0.31672 | Learning_Rate : 0.0010000000474974513\n",
      "\n",
      "Epoch 00006: saving model to /content/saved models/weights.-006-0.39043\n",
      "84/84 [==============================] - 1s 17ms/step - loss: 0.3925 - root_mean_squared_error: 0.2207 - val_loss: 0.3904 - val_root_mean_squared_error: 0.2176\n",
      "Epoch 7/75\n",
      "84/84 [==============================] - ETA: 0s - loss: 0.3899 - root_mean_squared_error: 0.2186\n",
      "spearman :\n",
      "train_spearman : 0.33097 | val_spearman : 0.31874 | Learning_Rate : 0.0010000000474974513\n",
      "\n",
      "Epoch 00007: saving model to /content/saved models/weights.-007-0.38615\n",
      "84/84 [==============================] - 1s 17ms/step - loss: 0.3899 - root_mean_squared_error: 0.2186 - val_loss: 0.3861 - val_root_mean_squared_error: 0.2144\n",
      "Epoch 8/75\n",
      "81/84 [===========================>..] - ETA: 0s - loss: 0.3884 - root_mean_squared_error: 0.2173\n",
      "spearman :\n",
      "train_spearman : 0.33808 | val_spearman : 0.32137 | Learning_Rate : 0.0010000000474974513\n",
      "\n",
      "Epoch 00008: saving model to /content/saved models/weights.-008-0.38591\n",
      "84/84 [==============================] - 1s 18ms/step - loss: 0.3883 - root_mean_squared_error: 0.2174 - val_loss: 0.3859 - val_root_mean_squared_error: 0.2145\n",
      "Epoch 9/75\n",
      "80/84 [===========================>..] - ETA: 0s - loss: 0.3878 - root_mean_squared_error: 0.2172\n",
      "spearman :\n",
      "train_spearman : 0.35321 | val_spearman : 0.33826 | Learning_Rate : 0.0010000000474974513\n",
      "\n",
      "Epoch 00009: saving model to /content/saved models/weights.-009-0.38544\n",
      "84/84 [==============================] - 1s 17ms/step - loss: 0.3877 - root_mean_squared_error: 0.2168 - val_loss: 0.3854 - val_root_mean_squared_error: 0.2139\n",
      "Epoch 10/75\n",
      "82/84 [============================>.] - ETA: 0s - loss: 0.3868 - root_mean_squared_error: 0.2160\n",
      "spearman :\n",
      "train_spearman : 0.36461 | val_spearman : 0.33674 | Learning_Rate : 0.0010000000474974513\n",
      "\n",
      "Epoch 00010: saving model to /content/saved models/weights.-010-0.38777\n",
      "84/84 [==============================] - 1s 18ms/step - loss: 0.3867 - root_mean_squared_error: 0.2160 - val_loss: 0.3878 - val_root_mean_squared_error: 0.2157\n",
      "Epoch 11/75\n",
      "80/84 [===========================>..] - ETA: 0s - loss: 0.3850 - root_mean_squared_error: 0.2148\n",
      "spearman :\n",
      "train_spearman : 0.36869 | val_spearman : 0.34319 | Learning_Rate : 0.0010000000474974513\n",
      "\n",
      "Epoch 00011: saving model to /content/saved models/weights.-011-0.38689\n",
      "84/84 [==============================] - 1s 17ms/step - loss: 0.3848 - root_mean_squared_error: 0.2145 - val_loss: 0.3869 - val_root_mean_squared_error: 0.2148\n",
      "Epoch 12/75\n",
      "81/84 [===========================>..] - ETA: 0s - loss: 0.3839 - root_mean_squared_error: 0.2136\n",
      "spearman :\n",
      "train_spearman : 0.38208 | val_spearman : 0.34869 | Learning_Rate : 0.0010000000474974513\n",
      "\n",
      "Epoch 00012: saving model to /content/saved models/weights.-012-0.38312\n",
      "84/84 [==============================] - 2s 18ms/step - loss: 0.3839 - root_mean_squared_error: 0.2137 - val_loss: 0.3831 - val_root_mean_squared_error: 0.2120\n",
      "Epoch 13/75\n",
      "80/84 [===========================>..] - ETA: 0s - loss: 0.3826 - root_mean_squared_error: 0.2130\n",
      "spearman :\n",
      "train_spearman : 0.38537 | val_spearman : 0.35262 | Learning_Rate : 0.0010000000474974513\n",
      "\n",
      "Epoch 00013: saving model to /content/saved models/weights.-013-0.38073\n",
      "84/84 [==============================] - 1s 17ms/step - loss: 0.3828 - root_mean_squared_error: 0.2129 - val_loss: 0.3807 - val_root_mean_squared_error: 0.2101\n",
      "Epoch 14/75\n",
      "82/84 [============================>.] - ETA: 0s - loss: 0.3818 - root_mean_squared_error: 0.2121\n",
      "spearman :\n",
      "train_spearman : 0.38812 | val_spearman : 0.35998 | Learning_Rate : 0.0010000000474974513\n",
      "\n",
      "Epoch 00014: saving model to /content/saved models/weights.-014-0.37961\n",
      "84/84 [==============================] - 1s 18ms/step - loss: 0.3819 - root_mean_squared_error: 0.2122 - val_loss: 0.3796 - val_root_mean_squared_error: 0.2091\n",
      "Epoch 15/75\n",
      "81/84 [===========================>..] - ETA: 0s - loss: 0.3815 - root_mean_squared_error: 0.2117\n",
      "spearman :\n",
      "train_spearman : 0.38329 | val_spearman : 0.36732 | Learning_Rate : 0.0010000000474974513\n",
      "\n",
      "Epoch 00015: saving model to /content/saved models/weights.-015-0.38146\n",
      "84/84 [==============================] - 1s 17ms/step - loss: 0.3814 - root_mean_squared_error: 0.2118 - val_loss: 0.3815 - val_root_mean_squared_error: 0.2109\n",
      "Epoch 16/75\n",
      "80/84 [===========================>..] - ETA: 0s - loss: 0.3804 - root_mean_squared_error: 0.2107\n",
      "spearman :\n",
      "train_spearman : 0.39899 | val_spearman : 0.37027 | Learning_Rate : 0.0010000000474974513\n",
      "\n",
      "Epoch 00016: saving model to /content/saved models/weights.-016-0.38077\n",
      "84/84 [==============================] - 1s 18ms/step - loss: 0.3803 - root_mean_squared_error: 0.2110 - val_loss: 0.3808 - val_root_mean_squared_error: 0.2097\n",
      "Epoch 17/75\n",
      "81/84 [===========================>..] - ETA: 0s - loss: 0.3801 - root_mean_squared_error: 0.2109\n",
      "spearman :\n",
      "train_spearman : 0.40265 | val_spearman : 0.37412 | Learning_Rate : 0.0010000000474974513\n",
      "\n",
      "Epoch 00017: saving model to /content/saved models/weights.-017-0.38264\n",
      "84/84 [==============================] - 1s 17ms/step - loss: 0.3800 - root_mean_squared_error: 0.2108 - val_loss: 0.3826 - val_root_mean_squared_error: 0.2114\n",
      "Epoch 18/75\n",
      "82/84 [============================>.] - ETA: 0s - loss: 0.3793 - root_mean_squared_error: 0.2099\n",
      "spearman :\n",
      "train_spearman : 0.41283 | val_spearman : 0.38732 | Learning_Rate : 0.0010000000474974513\n",
      "\n",
      "Epoch 00018: saving model to /content/saved models/weights.-018-0.38088\n",
      "84/84 [==============================] - 2s 18ms/step - loss: 0.3793 - root_mean_squared_error: 0.2099 - val_loss: 0.3809 - val_root_mean_squared_error: 0.2099\n",
      "Epoch 19/75\n",
      "83/84 [============================>.] - ETA: 0s - loss: 0.3782 - root_mean_squared_error: 0.2091\n",
      "spearman :\n",
      "train_spearman : 0.40996 | val_spearman : 0.37767 | Learning_Rate : 0.0010000000474974513\n",
      "\n",
      "Epoch 00019: saving model to /content/saved models/weights.-019-0.37843\n",
      "84/84 [==============================] - 1s 17ms/step - loss: 0.3780 - root_mean_squared_error: 0.2089 - val_loss: 0.3784 - val_root_mean_squared_error: 0.2082\n",
      "Epoch 20/75\n",
      "80/84 [===========================>..] - ETA: 0s - loss: 0.3778 - root_mean_squared_error: 0.2090\n",
      "spearman :\n",
      "train_spearman : 0.41353 | val_spearman : 0.37753 | Learning_Rate : 0.0010000000474974513\n",
      "\n",
      "Epoch 00020: saving model to /content/saved models/weights.-020-0.38013\n",
      "84/84 [==============================] - 2s 18ms/step - loss: 0.3779 - root_mean_squared_error: 0.2090 - val_loss: 0.3801 - val_root_mean_squared_error: 0.2096\n",
      "Epoch 21/75\n",
      "81/84 [===========================>..] - ETA: 0s - loss: 0.3770 - root_mean_squared_error: 0.2083\n",
      "spearman :\n",
      "train_spearman : 0.41164 | val_spearman : 0.37551 | Learning_Rate : 0.0010000000474974513\n",
      "\n",
      "Epoch 00021: saving model to /content/saved models/weights.-021-0.38082\n",
      "84/84 [==============================] - 1s 17ms/step - loss: 0.3774 - root_mean_squared_error: 0.2085 - val_loss: 0.3808 - val_root_mean_squared_error: 0.2103\n",
      "Epoch 22/75\n",
      "81/84 [===========================>..] - ETA: 0s - loss: 0.3762 - root_mean_squared_error: 0.2072\n",
      "spearman :\n",
      "train_spearman : 0.41029 | val_spearman : 0.37497 | Learning_Rate : 0.0010000000474974513\n",
      "\n",
      "Epoch 00022: saving model to /content/saved models/weights.-022-0.38111\n",
      "84/84 [==============================] - 2s 19ms/step - loss: 0.3761 - root_mean_squared_error: 0.2073 - val_loss: 0.3811 - val_root_mean_squared_error: 0.2101\n",
      "Epoch 23/75\n",
      "83/84 [============================>.] - ETA: 0s - loss: 0.3760 - root_mean_squared_error: 0.2073\n",
      "spearman :\n",
      "train_spearman : 0.41931 | val_spearman : 0.38045 | Learning_Rate : 0.0010000000474974513\n",
      "\n",
      "Epoch 00023: saving model to /content/saved models/weights.-023-0.37626\n",
      "84/84 [==============================] - 1s 17ms/step - loss: 0.3759 - root_mean_squared_error: 0.2072 - val_loss: 0.3763 - val_root_mean_squared_error: 0.2063\n",
      "Epoch 24/75\n",
      "84/84 [==============================] - ETA: 0s - loss: 0.3757 - root_mean_squared_error: 0.2073\n",
      "spearman :\n",
      "train_spearman : 0.42219 | val_spearman : 0.38933 | Learning_Rate : 0.0010000000474974513\n",
      "\n",
      "Epoch 00024: saving model to /content/saved models/weights.-024-0.37965\n",
      "84/84 [==============================] - 1s 17ms/step - loss: 0.3757 - root_mean_squared_error: 0.2073 - val_loss: 0.3796 - val_root_mean_squared_error: 0.2095\n",
      "Epoch 25/75\n",
      "80/84 [===========================>..] - ETA: 0s - loss: 0.3750 - root_mean_squared_error: 0.2068\n",
      "spearman :\n",
      "train_spearman : 0.41428 | val_spearman : 0.3697 | Learning_Rate : 0.0010000000474974513\n",
      "\n",
      "Epoch 00025: saving model to /content/saved models/weights.-025-0.37664\n",
      "84/84 [==============================] - 2s 19ms/step - loss: 0.3749 - root_mean_squared_error: 0.2065 - val_loss: 0.3766 - val_root_mean_squared_error: 0.2066\n",
      "Epoch 26/75\n",
      "79/84 [===========================>..] - ETA: 0s - loss: 0.3739 - root_mean_squared_error: 0.2060\n",
      "spearman :\n",
      "train_spearman : 0.43083 | val_spearman : 0.38181 | Learning_Rate : 0.0010000000474974513\n",
      "\n",
      "Epoch 00026: saving model to /content/saved models/weights.-026-0.37814\n",
      "84/84 [==============================] - 1s 18ms/step - loss: 0.3741 - root_mean_squared_error: 0.2060 - val_loss: 0.3781 - val_root_mean_squared_error: 0.2079\n",
      "Epoch 27/75\n",
      "80/84 [===========================>..] - ETA: 0s - loss: 0.3735 - root_mean_squared_error: 0.2056\n",
      "spearman :\n",
      "train_spearman : 0.43204 | val_spearman : 0.38759 | Learning_Rate : 0.0010000000474974513\n",
      "\n",
      "Epoch 00027: saving model to /content/saved models/weights.-027-0.37693\n",
      "84/84 [==============================] - 2s 19ms/step - loss: 0.3735 - root_mean_squared_error: 0.2054 - val_loss: 0.3769 - val_root_mean_squared_error: 0.2067\n",
      "Epoch 28/75\n",
      "79/84 [===========================>..] - ETA: 0s - loss: 0.3734 - root_mean_squared_error: 0.2054\n",
      "spearman :\n",
      "train_spearman : 0.4256 | val_spearman : 0.3744 | Learning_Rate : 0.0010000000474974513\n",
      "\n",
      "Epoch 00028: saving model to /content/saved models/weights.-028-0.37791\n",
      "84/84 [==============================] - 1s 17ms/step - loss: 0.3733 - root_mean_squared_error: 0.2052 - val_loss: 0.3779 - val_root_mean_squared_error: 0.2079\n",
      "Epoch 29/75\n",
      "83/84 [============================>.] - ETA: 0s - loss: 0.3729 - root_mean_squared_error: 0.2051\n",
      "spearman :\n",
      "train_spearman : 0.43245 | val_spearman : 0.39864 | Learning_Rate : 0.0010000000474974513\n",
      "\n",
      "Epoch 00029: saving model to /content/saved models/weights.-029-0.37855\n",
      "84/84 [==============================] - 2s 19ms/step - loss: 0.3730 - root_mean_squared_error: 0.2051 - val_loss: 0.3785 - val_root_mean_squared_error: 0.2087\n",
      "Epoch 30/75\n",
      "81/84 [===========================>..] - ETA: 0s - loss: 0.3734 - root_mean_squared_error: 0.2052\n",
      "spearman :\n",
      "train_spearman : 0.43881 | val_spearman : 0.39006 | Learning_Rate : 0.0010000000474974513\n",
      "\n",
      "Epoch 00030: ReduceLROnPlateau reducing learning rate to 0.00031622778103685084.\n",
      "\n",
      "Epoch 00030: saving model to /content/saved models/weights.-030-0.37745\n",
      "84/84 [==============================] - 1s 17ms/step - loss: 0.3731 - root_mean_squared_error: 0.2051 - val_loss: 0.3774 - val_root_mean_squared_error: 0.2071\n",
      "Epoch 31/75\n",
      "81/84 [===========================>..] - ETA: 0s - loss: 0.3693 - root_mean_squared_error: 0.2019\n",
      "spearman :\n",
      "train_spearman : 0.45346 | val_spearman : 0.4004 | Learning_Rate : 0.00031599999056197703\n",
      "\n",
      "Epoch 00031: saving model to /content/saved models/weights.-031-0.37335\n",
      "84/84 [==============================] - 2s 23ms/step - loss: 0.3693 - root_mean_squared_error: 0.2018 - val_loss: 0.3734 - val_root_mean_squared_error: 0.2039\n",
      "Epoch 32/75\n",
      "80/84 [===========================>..] - ETA: 0s - loss: 0.3680 - root_mean_squared_error: 0.2007\n",
      "spearman :\n",
      "train_spearman : 0.45406 | val_spearman : 0.40302 | Learning_Rate : 0.00031599999056197703\n",
      "\n",
      "Epoch 00032: saving model to /content/saved models/weights.-032-0.37387\n",
      "84/84 [==============================] - 1s 18ms/step - loss: 0.3681 - root_mean_squared_error: 0.2008 - val_loss: 0.3739 - val_root_mean_squared_error: 0.2041\n",
      "Epoch 33/75\n",
      "82/84 [============================>.] - ETA: 0s - loss: 0.3680 - root_mean_squared_error: 0.2006\n",
      "spearman :\n",
      "train_spearman : 0.45587 | val_spearman : 0.39582 | Learning_Rate : 0.00031599999056197703\n",
      "\n",
      "Epoch 00033: saving model to /content/saved models/weights.-033-0.37344\n",
      "84/84 [==============================] - 2s 18ms/step - loss: 0.3677 - root_mean_squared_error: 0.2005 - val_loss: 0.3734 - val_root_mean_squared_error: 0.2039\n",
      "Epoch 34/75\n",
      "79/84 [===========================>..] - ETA: 0s - loss: 0.3673 - root_mean_squared_error: 0.2004\n",
      "spearman :\n",
      "train_spearman : 0.45801 | val_spearman : 0.40161 | Learning_Rate : 0.00031599999056197703\n",
      "\n",
      "Epoch 00034: saving model to /content/saved models/weights.-034-0.37352\n",
      "84/84 [==============================] - 1s 17ms/step - loss: 0.3675 - root_mean_squared_error: 0.2003 - val_loss: 0.3735 - val_root_mean_squared_error: 0.2040\n",
      "Epoch 35/75\n",
      "84/84 [==============================] - ETA: 0s - loss: 0.3672 - root_mean_squared_error: 0.2002\n",
      "spearman :\n",
      "train_spearman : 0.45869 | val_spearman : 0.39922 | Learning_Rate : 0.00031599999056197703\n",
      "\n",
      "Epoch 00035: saving model to /content/saved models/weights.-035-0.37363\n",
      "84/84 [==============================] - 2s 19ms/step - loss: 0.3672 - root_mean_squared_error: 0.2002 - val_loss: 0.3736 - val_root_mean_squared_error: 0.2039\n",
      "Epoch 36/75\n",
      "76/84 [==========================>...] - ETA: 0s - loss: 0.3669 - root_mean_squared_error: 0.2003\n",
      "spearman :\n",
      "train_spearman : 0.4635 | val_spearman : 0.40161 | Learning_Rate : 0.00031599999056197703\n",
      "\n",
      "Epoch 00036: saving model to /content/saved models/weights.-036-0.37315\n",
      "84/84 [==============================] - 1s 17ms/step - loss: 0.3666 - root_mean_squared_error: 0.1996 - val_loss: 0.3732 - val_root_mean_squared_error: 0.2035\n",
      "Epoch 37/75\n",
      "80/84 [===========================>..] - ETA: 0s - loss: 0.3662 - root_mean_squared_error: 0.1993\n",
      "spearman :\n",
      "train_spearman : 0.4643 | val_spearman : 0.40149 | Learning_Rate : 0.00031599999056197703\n",
      "\n",
      "Epoch 00037: saving model to /content/saved models/weights.-037-0.37327\n",
      "84/84 [==============================] - 2s 18ms/step - loss: 0.3663 - root_mean_squared_error: 0.1994 - val_loss: 0.3733 - val_root_mean_squared_error: 0.2037\n",
      "Epoch 38/75\n",
      "78/84 [==========================>...] - ETA: 0s - loss: 0.3665 - root_mean_squared_error: 0.1996\n",
      "spearman :\n",
      "train_spearman : 0.46646 | val_spearman : 0.40258 | Learning_Rate : 0.00031599999056197703\n",
      "\n",
      "Epoch 00038: saving model to /content/saved models/weights.-038-0.37332\n",
      "84/84 [==============================] - 1s 17ms/step - loss: 0.3664 - root_mean_squared_error: 0.1994 - val_loss: 0.3733 - val_root_mean_squared_error: 0.2037\n",
      "Epoch 39/75\n",
      "84/84 [==============================] - ETA: 0s - loss: 0.3660 - root_mean_squared_error: 0.1992\n",
      "spearman :\n",
      "train_spearman : 0.46531 | val_spearman : 0.4043 | Learning_Rate : 0.00031599999056197703\n",
      "\n",
      "Epoch 00039: saving model to /content/saved models/weights.-039-0.37278\n",
      "84/84 [==============================] - 2s 18ms/step - loss: 0.3660 - root_mean_squared_error: 0.1992 - val_loss: 0.3728 - val_root_mean_squared_error: 0.2031\n",
      "Epoch 40/75\n",
      "81/84 [===========================>..] - ETA: 0s - loss: 0.3658 - root_mean_squared_error: 0.1989\n",
      "spearman :\n",
      "train_spearman : 0.46569 | val_spearman : 0.39995 | Learning_Rate : 0.00031599999056197703\n",
      "\n",
      "Epoch 00040: saving model to /content/saved models/weights.-040-0.37279\n",
      "84/84 [==============================] - 1s 18ms/step - loss: 0.3658 - root_mean_squared_error: 0.1989 - val_loss: 0.3728 - val_root_mean_squared_error: 0.2032\n",
      "Epoch 41/75\n",
      "78/84 [==========================>...] - ETA: 0s - loss: 0.3655 - root_mean_squared_error: 0.1988\n",
      "spearman :\n",
      "train_spearman : 0.46932 | val_spearman : 0.40665 | Learning_Rate : 0.00031599999056197703\n",
      "\n",
      "Epoch 00041: saving model to /content/saved models/weights.-041-0.37353\n",
      "84/84 [==============================] - 5s 55ms/step - loss: 0.3654 - root_mean_squared_error: 0.1986 - val_loss: 0.3735 - val_root_mean_squared_error: 0.2038\n",
      "Epoch 42/75\n",
      "79/84 [===========================>..] - ETA: 0s - loss: 0.3653 - root_mean_squared_error: 0.1984\n",
      "spearman :\n",
      "train_spearman : 0.46887 | val_spearman : 0.40046 | Learning_Rate : 0.00031599999056197703\n",
      "\n",
      "Epoch 00042: saving model to /content/saved models/weights.-042-0.37320\n",
      "84/84 [==============================] - 2s 24ms/step - loss: 0.3654 - root_mean_squared_error: 0.1986 - val_loss: 0.3732 - val_root_mean_squared_error: 0.2038\n",
      "Epoch 43/75\n",
      "83/84 [============================>.] - ETA: 0s - loss: 0.3653 - root_mean_squared_error: 0.1985\n",
      "spearman :\n",
      "train_spearman : 0.47193 | val_spearman : 0.40948 | Learning_Rate : 0.00031599999056197703\n",
      "\n",
      "Epoch 00043: saving model to /content/saved models/weights.-043-0.37252\n",
      "84/84 [==============================] - 2s 18ms/step - loss: 0.3651 - root_mean_squared_error: 0.1985 - val_loss: 0.3725 - val_root_mean_squared_error: 0.2029\n",
      "Epoch 44/75\n",
      "80/84 [===========================>..] - ETA: 0s - loss: 0.3651 - root_mean_squared_error: 0.1981\n",
      "spearman :\n",
      "train_spearman : 0.47382 | val_spearman : 0.40752 | Learning_Rate : 0.00031599999056197703\n",
      "\n",
      "Epoch 00044: saving model to /content/saved models/weights.-044-0.37236\n",
      "84/84 [==============================] - 1s 17ms/step - loss: 0.3651 - root_mean_squared_error: 0.1983 - val_loss: 0.3724 - val_root_mean_squared_error: 0.2028\n",
      "Epoch 45/75\n",
      "80/84 [===========================>..] - ETA: 0s - loss: 0.3648 - root_mean_squared_error: 0.1978\n",
      "spearman :\n",
      "train_spearman : 0.46327 | val_spearman : 0.40427 | Learning_Rate : 0.00031599999056197703\n",
      "\n",
      "Epoch 00045: saving model to /content/saved models/weights.-045-0.37218\n",
      "84/84 [==============================] - 1s 18ms/step - loss: 0.3646 - root_mean_squared_error: 0.1979 - val_loss: 0.3722 - val_root_mean_squared_error: 0.2027\n",
      "Epoch 46/75\n",
      "80/84 [===========================>..] - ETA: 0s - loss: 0.3644 - root_mean_squared_error: 0.1977\n",
      "spearman :\n",
      "train_spearman : 0.46435 | val_spearman : 0.40713 | Learning_Rate : 0.00031599999056197703\n",
      "\n",
      "Epoch 00046: saving model to /content/saved models/weights.-046-0.37249\n",
      "84/84 [==============================] - 5s 54ms/step - loss: 0.3642 - root_mean_squared_error: 0.1977 - val_loss: 0.3725 - val_root_mean_squared_error: 0.2029\n",
      "Epoch 47/75\n",
      "82/84 [============================>.] - ETA: 0s - loss: 0.3641 - root_mean_squared_error: 0.1974\n",
      "spearman :\n",
      "train_spearman : 0.46245 | val_spearman : 0.39554 | Learning_Rate : 0.00031599999056197703\n",
      "\n",
      "Epoch 00047: saving model to /content/saved models/weights.-047-0.37317\n",
      "84/84 [==============================] - 2s 19ms/step - loss: 0.3641 - root_mean_squared_error: 0.1975 - val_loss: 0.3732 - val_root_mean_squared_error: 0.2035\n",
      "Epoch 48/75\n",
      "84/84 [==============================] - ETA: 0s - loss: 0.3639 - root_mean_squared_error: 0.1974\n",
      "spearman :\n",
      "train_spearman : 0.47614 | val_spearman : 0.41002 | Learning_Rate : 0.00031599999056197703\n",
      "\n",
      "Epoch 00048: saving model to /content/saved models/weights.-048-0.37202\n",
      "84/84 [==============================] - 1s 17ms/step - loss: 0.3639 - root_mean_squared_error: 0.1974 - val_loss: 0.3720 - val_root_mean_squared_error: 0.2025\n",
      "Epoch 49/75\n",
      "84/84 [==============================] - ETA: 0s - loss: 0.3639 - root_mean_squared_error: 0.1973\n",
      "spearman :\n",
      "train_spearman : 0.46605 | val_spearman : 0.40166 | Learning_Rate : 0.00031599999056197703\n",
      "\n",
      "Epoch 00049: saving model to /content/saved models/weights.-049-0.37256\n",
      "84/84 [==============================] - 5s 62ms/step - loss: 0.3639 - root_mean_squared_error: 0.1973 - val_loss: 0.3726 - val_root_mean_squared_error: 0.2030\n",
      "Epoch 50/75\n",
      "82/84 [============================>.] - ETA: 0s - loss: 0.3637 - root_mean_squared_error: 0.1969\n",
      "spearman :\n",
      "train_spearman : 0.48156 | val_spearman : 0.40642 | Learning_Rate : 0.00031599999056197703\n",
      "\n",
      "Epoch 00050: saving model to /content/saved models/weights.-050-0.37253\n",
      "84/84 [==============================] - 1s 17ms/step - loss: 0.3636 - root_mean_squared_error: 0.1970 - val_loss: 0.3725 - val_root_mean_squared_error: 0.2030\n",
      "Epoch 51/75\n",
      "82/84 [============================>.] - ETA: 0s - loss: 0.3632 - root_mean_squared_error: 0.1968\n",
      "spearman :\n",
      "train_spearman : 0.47884 | val_spearman : 0.40911 | Learning_Rate : 0.00031599999056197703\n",
      "\n",
      "Epoch 00051: saving model to /content/saved models/weights.-051-0.37276\n",
      "84/84 [==============================] - 2s 18ms/step - loss: 0.3631 - root_mean_squared_error: 0.1967 - val_loss: 0.3728 - val_root_mean_squared_error: 0.2030\n",
      "Epoch 52/75\n",
      "82/84 [============================>.] - ETA: 0s - loss: 0.3629 - root_mean_squared_error: 0.1964\n",
      "spearman :\n",
      "train_spearman : 0.47076 | val_spearman : 0.40322 | Learning_Rate : 0.00031599999056197703\n",
      "\n",
      "Epoch 00052: saving model to /content/saved models/weights.-052-0.37293\n",
      "84/84 [==============================] - 1s 17ms/step - loss: 0.3629 - root_mean_squared_error: 0.1966 - val_loss: 0.3729 - val_root_mean_squared_error: 0.2034\n",
      "Epoch 53/75\n",
      "82/84 [============================>.] - ETA: 0s - loss: 0.3635 - root_mean_squared_error: 0.1971\n",
      "spearman :\n",
      "train_spearman : 0.47203 | val_spearman : 0.40844 | Learning_Rate : 0.00031599999056197703\n",
      "\n",
      "Epoch 00053: saving model to /content/saved models/weights.-053-0.37213\n",
      "84/84 [==============================] - 1s 17ms/step - loss: 0.3636 - root_mean_squared_error: 0.1971 - val_loss: 0.3721 - val_root_mean_squared_error: 0.2025\n",
      "Epoch 54/75\n",
      "82/84 [============================>.] - ETA: 0s - loss: 0.3631 - root_mean_squared_error: 0.1964\n",
      "spearman :\n",
      "train_spearman : 0.47151 | val_spearman : 0.41174 | Learning_Rate : 0.00031599999056197703\n",
      "\n",
      "Epoch 00054: saving model to /content/saved models/weights.-054-0.37279\n",
      "84/84 [==============================] - 2s 18ms/step - loss: 0.3629 - root_mean_squared_error: 0.1965 - val_loss: 0.3728 - val_root_mean_squared_error: 0.2030\n",
      "Epoch 55/75\n",
      "83/84 [============================>.] - ETA: 0s - loss: 0.3625 - root_mean_squared_error: 0.1959\n",
      "spearman :\n",
      "train_spearman : 0.47435 | val_spearman : 0.41167 | Learning_Rate : 0.00031599999056197703\n",
      "\n",
      "Epoch 00055: ReduceLROnPlateau reducing learning rate to 0.00010000000639606199.\n",
      "\n",
      "Epoch 00055: saving model to /content/saved models/weights.-055-0.37222\n",
      "84/84 [==============================] - 2s 19ms/step - loss: 0.3624 - root_mean_squared_error: 0.1960 - val_loss: 0.3722 - val_root_mean_squared_error: 0.2025\n",
      "Epoch 56/75\n",
      "80/84 [===========================>..] - ETA: 0s - loss: 0.3614 - root_mean_squared_error: 0.1950\n",
      "spearman :\n",
      "train_spearman : 0.4787 | val_spearman : 0.40719 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00056: saving model to /content/saved models/weights.-056-0.37153\n",
      "84/84 [==============================] - 5s 58ms/step - loss: 0.3613 - root_mean_squared_error: 0.1951 - val_loss: 0.3715 - val_root_mean_squared_error: 0.2020\n",
      "Epoch 57/75\n",
      "83/84 [============================>.] - ETA: 0s - loss: 0.3607 - root_mean_squared_error: 0.1946\n",
      "spearman :\n",
      "train_spearman : 0.47974 | val_spearman : 0.40985 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00057: saving model to /content/saved models/weights.-057-0.37127\n",
      "84/84 [==============================] - 2s 18ms/step - loss: 0.3607 - root_mean_squared_error: 0.1946 - val_loss: 0.3713 - val_root_mean_squared_error: 0.2017\n",
      "Epoch 58/75\n",
      "77/84 [==========================>...] - ETA: 0s - loss: 0.3605 - root_mean_squared_error: 0.1948\n",
      "spearman :\n",
      "train_spearman : 0.47904 | val_spearman : 0.40617 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00058: saving model to /content/saved models/weights.-058-0.37114\n",
      "84/84 [==============================] - 2s 19ms/step - loss: 0.3607 - root_mean_squared_error: 0.1946 - val_loss: 0.3711 - val_root_mean_squared_error: 0.2017\n",
      "Epoch 59/75\n",
      "82/84 [============================>.] - ETA: 0s - loss: 0.3607 - root_mean_squared_error: 0.1945\n",
      "spearman :\n",
      "train_spearman : 0.48059 | val_spearman : 0.41772 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00059: saving model to /content/saved models/weights.-059-0.37133\n",
      "84/84 [==============================] - 1s 18ms/step - loss: 0.3608 - root_mean_squared_error: 0.1946 - val_loss: 0.3713 - val_root_mean_squared_error: 0.2018\n",
      "Epoch 60/75\n",
      "79/84 [===========================>..] - ETA: 0s - loss: 0.3603 - root_mean_squared_error: 0.1944\n",
      "spearman :\n",
      "train_spearman : 0.47771 | val_spearman : 0.40978 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00060: saving model to /content/saved models/weights.-060-0.37134\n",
      "84/84 [==============================] - 1s 18ms/step - loss: 0.3603 - root_mean_squared_error: 0.1942 - val_loss: 0.3713 - val_root_mean_squared_error: 0.2019\n",
      "Epoch 61/75\n",
      "80/84 [===========================>..] - ETA: 0s - loss: 0.3599 - root_mean_squared_error: 0.1941\n",
      "spearman :\n",
      "train_spearman : 0.48145 | val_spearman : 0.40254 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00061: saving model to /content/saved models/weights.-061-0.37131\n",
      "84/84 [==============================] - 5s 59ms/step - loss: 0.3601 - root_mean_squared_error: 0.1941 - val_loss: 0.3713 - val_root_mean_squared_error: 0.2017\n",
      "Epoch 62/75\n",
      "73/84 [=========================>....] - ETA: 0s - loss: 0.3593 - root_mean_squared_error: 0.1932\n",
      "spearman :\n",
      "train_spearman : 0.46957 | val_spearman : 0.41316 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00062: saving model to /content/saved models/weights.-062-0.37130\n",
      "84/84 [==============================] - 1s 17ms/step - loss: 0.3596 - root_mean_squared_error: 0.1937 - val_loss: 0.3713 - val_root_mean_squared_error: 0.2019\n",
      "Epoch 63/75\n",
      "81/84 [===========================>..] - ETA: 0s - loss: 0.3601 - root_mean_squared_error: 0.1941\n",
      "spearman :\n",
      "train_spearman : 0.48331 | val_spearman : 0.41439 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00063: saving model to /content/saved models/weights.-063-0.37101\n",
      "84/84 [==============================] - 1s 17ms/step - loss: 0.3601 - root_mean_squared_error: 0.1942 - val_loss: 0.3710 - val_root_mean_squared_error: 0.2015\n",
      "Epoch 64/75\n",
      "84/84 [==============================] - ETA: 0s - loss: 0.3598 - root_mean_squared_error: 0.1938\n",
      "spearman :\n",
      "train_spearman : 0.47127 | val_spearman : 0.40978 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00064: saving model to /content/saved models/weights.-064-0.37130\n",
      "84/84 [==============================] - 2s 18ms/step - loss: 0.3598 - root_mean_squared_error: 0.1938 - val_loss: 0.3713 - val_root_mean_squared_error: 0.2018\n",
      "Epoch 65/75\n",
      "79/84 [===========================>..] - ETA: 0s - loss: 0.3598 - root_mean_squared_error: 0.1937\n",
      "spearman :\n",
      "train_spearman : 0.47202 | val_spearman : 0.4117 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00065: saving model to /content/saved models/weights.-065-0.37195\n",
      "84/84 [==============================] - 1s 17ms/step - loss: 0.3597 - root_mean_squared_error: 0.1937 - val_loss: 0.3720 - val_root_mean_squared_error: 0.2025\n",
      "Epoch 66/75\n",
      "80/84 [===========================>..] - ETA: 0s - loss: 0.3596 - root_mean_squared_error: 0.1937\n",
      "spearman :\n",
      "train_spearman : 0.4828 | val_spearman : 0.41099 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00066: saving model to /content/saved models/weights.-066-0.37124\n",
      "84/84 [==============================] - 5s 64ms/step - loss: 0.3597 - root_mean_squared_error: 0.1937 - val_loss: 0.3712 - val_root_mean_squared_error: 0.2018\n",
      "Epoch 67/75\n",
      "79/84 [===========================>..] - ETA: 0s - loss: 0.3590 - root_mean_squared_error: 0.1933\n",
      "spearman :\n",
      "train_spearman : 0.47284 | val_spearman : 0.41575 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00067: saving model to /content/saved models/weights.-067-0.37111\n",
      "84/84 [==============================] - 1s 17ms/step - loss: 0.3594 - root_mean_squared_error: 0.1936 - val_loss: 0.3711 - val_root_mean_squared_error: 0.2016\n",
      "Epoch 68/75\n",
      "82/84 [============================>.] - ETA: 0s - loss: 0.3593 - root_mean_squared_error: 0.1932\n",
      "spearman :\n",
      "train_spearman : 0.4723 | val_spearman : 0.41433 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00068: saving model to /content/saved models/weights.-068-0.37144\n",
      "84/84 [==============================] - 2s 18ms/step - loss: 0.3595 - root_mean_squared_error: 0.1934 - val_loss: 0.3714 - val_root_mean_squared_error: 0.2019\n",
      "Epoch 69/75\n",
      "79/84 [===========================>..] - ETA: 0s - loss: 0.3590 - root_mean_squared_error: 0.1930\n",
      "spearman :\n",
      "train_spearman : 0.47335 | val_spearman : 0.41187 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00069: saving model to /content/saved models/weights.-069-0.37086\n",
      "84/84 [==============================] - 2s 18ms/step - loss: 0.3593 - root_mean_squared_error: 0.1935 - val_loss: 0.3709 - val_root_mean_squared_error: 0.2015\n",
      "Epoch 70/75\n",
      "79/84 [===========================>..] - ETA: 0s - loss: 0.3589 - root_mean_squared_error: 0.1933\n",
      "spearman :\n",
      "train_spearman : 0.48346 | val_spearman : 0.40993 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00070: saving model to /content/saved models/weights.-070-0.37107\n",
      "84/84 [==============================] - 2s 20ms/step - loss: 0.3592 - root_mean_squared_error: 0.1933 - val_loss: 0.3711 - val_root_mean_squared_error: 0.2017\n",
      "Epoch 71/75\n",
      "82/84 [============================>.] - ETA: 0s - loss: 0.3594 - root_mean_squared_error: 0.1935\n",
      "spearman :\n",
      "train_spearman : 0.47303 | val_spearman : 0.41304 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00071: saving model to /content/saved models/weights.-071-0.37113\n",
      "84/84 [==============================] - 7s 80ms/step - loss: 0.3593 - root_mean_squared_error: 0.1933 - val_loss: 0.3711 - val_root_mean_squared_error: 0.2017\n",
      "Epoch 72/75\n",
      "81/84 [===========================>..] - ETA: 0s - loss: 0.3587 - root_mean_squared_error: 0.1930\n",
      "spearman :\n",
      "train_spearman : 0.47742 | val_spearman : 0.41196 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00072: saving model to /content/saved models/weights.-072-0.37113\n",
      "84/84 [==============================] - 1s 17ms/step - loss: 0.3589 - root_mean_squared_error: 0.1931 - val_loss: 0.3711 - val_root_mean_squared_error: 0.2016\n",
      "Epoch 73/75\n",
      "83/84 [============================>.] - ETA: 0s - loss: 0.3589 - root_mean_squared_error: 0.1931\n",
      "spearman :\n",
      "train_spearman : 0.48806 | val_spearman : 0.4106 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00073: saving model to /content/saved models/weights.-073-0.37125\n",
      "84/84 [==============================] - 2s 18ms/step - loss: 0.3588 - root_mean_squared_error: 0.1929 - val_loss: 0.3712 - val_root_mean_squared_error: 0.2017\n",
      "Epoch 74/75\n",
      "78/84 [==========================>...] - ETA: 0s - loss: 0.3585 - root_mean_squared_error: 0.1932\n",
      "spearman :\n",
      "train_spearman : 0.4746 | val_spearman : 0.41568 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00074: saving model to /content/saved models/weights.-074-0.37102\n",
      "84/84 [==============================] - 1s 17ms/step - loss: 0.3588 - root_mean_squared_error: 0.1930 - val_loss: 0.3710 - val_root_mean_squared_error: 0.2016\n",
      "Epoch 75/75\n",
      "82/84 [============================>.] - ETA: 0s - loss: 0.3590 - root_mean_squared_error: 0.1930\n",
      "spearman :\n",
      "train_spearman : 0.47272 | val_spearman : 0.40937 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00075: saving model to /content/saved models/weights.-075-0.37134\n",
      "84/84 [==============================] - 2s 18ms/step - loss: 0.3590 - root_mean_squared_error: 0.1930 - val_loss: 0.3713 - val_root_mean_squared_error: 0.2020\n"
     ]
    }
   ],
   "source": [
    "# LSTM : training a model\n",
    "tf.keras.backend.clear_session()\n",
    "opt = tf.keras.optimizers.Adam(learning_rate = 0.001)\n",
    "rmse = tf.keras.metrics.RootMeanSquaredError()\n",
    " \n",
    "model.compile(loss = 'binary_crossentropy', optimizer = opt,  metrics = [rmse])\n",
    "history = model.fit(x = [tb_RoBERTa_train, ans_RoBERTa_train], y =  y_train.values,\n",
    "                    validation_data = ([tb_RoBERTa_cv, ans_RoBERTa_cv], y_cv.values),\n",
    "                    batch_size = 64, epochs = 75, callbacks = callbacks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 33
    },
    "colab_type": "code",
    "id": "IyaImzUVBoEu",
    "outputId": "4815d2ca-c7e2-4544-ef52-53ad5f6c1d0f"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.41772, 58)"
      ]
     },
     "execution_count": 48,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# best_results : (val_spearman, epoch_no)\n",
    "max(print_spearman_fn.spearman_dict['val_spearman']), np.argmax(print_spearman_fn.spearman_dict['val_spearman'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "-G6cTx9aNzR1"
   },
   "source": [
    "# 4.4. Sentence Embedding : XLNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 570
    },
    "colab_type": "code",
    "id": "acFh7wVbNzR4",
    "outputId": "6245fbdc-a28a-42a5-a358-dc3e83dbbe2d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting transformers\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/27/3c/91ed8f5c4e7ef3227b4119200fc0ed4b4fd965b1f0172021c25701087825/transformers-3.0.2-py3-none-any.whl (769kB)\n",
      "\u001b[K     |████████████████████████████████| 778kB 2.8MB/s \n",
      "\u001b[?25hCollecting sacremoses\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n",
      "\u001b[K     |████████████████████████████████| 890kB 14.3MB/s \n",
      "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers) (3.0.12)\n",
      "Collecting tokenizers==0.8.1.rc1\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/40/d0/30d5f8d221a0ed981a186c8eb986ce1c94e3a6e87f994eae9f4aa5250217/tokenizers-0.8.1rc1-cp36-cp36m-manylinux1_x86_64.whl (3.0MB)\n",
      "\u001b[K     |████████████████████████████████| 3.0MB 20.3MB/s \n",
      "\u001b[?25hCollecting sentencepiece!=0.1.92\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d4/a4/d0a884c4300004a78cca907a6ff9a5e9fe4f090f5d95ab341c53d28cbc58/sentencepiece-0.1.91-cp36-cp36m-manylinux1_x86_64.whl (1.1MB)\n",
      "\u001b[K     |████████████████████████████████| 1.1MB 37.7MB/s \n",
      "\u001b[?25hRequirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from transformers) (20.4)\n",
      "Requirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers) (0.7)\n",
      "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers) (4.41.1)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (2019.12.20)\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers) (1.18.5)\n",
      "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers) (2.23.0)\n",
      "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (1.15.0)\n",
      "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (7.1.2)\n",
      "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (0.16.0)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->transformers) (2.4.7)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2020.6.20)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (1.24.3)\n",
      "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (3.0.4)\n",
      "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2.10)\n",
      "Building wheels for collected packages: sacremoses\n",
      "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for sacremoses: filename=sacremoses-0.0.43-cp36-none-any.whl size=893260 sha256=a6d1c1aca66db3ca7511a0e432fccbe854e961b27b22d704870ee4ed800338d0\n",
      "  Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n",
      "Successfully built sacremoses\n",
      "Installing collected packages: sacremoses, tokenizers, sentencepiece, transformers\n",
      "Successfully installed sacremoses-0.0.43 sentencepiece-0.1.91 tokenizers-0.8.1rc1 transformers-3.0.2\n"
     ]
    }
   ],
   "source": [
    "!pip install transformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 262,
     "referenced_widgets": [
      "4e007ce6f520428c9282c1b4d2840b88",
      "09c7528baf164d2ab097ffd86a293af1",
      "20161806a2d149f389f3ec6e43ec4f99",
      "d7c0d20837d5450f8e6e57ceedf52784",
      "5dbc3bf1b2f3414bb9c82a27ad7b23f1",
      "3652fd96bd044fea92701a751da9e6c4",
      "4cacbd4222b84b64bf2655a472c169fd",
      "9cdabd8c25f045ce89627bde52ab9a21",
      "8d6e476417c2496588818f4ab41ec649",
      "9bf02940c29f4dd0bbc0c3b3e6228070",
      "0f5270dfa6d94e2eb03335215181b182",
      "9b6821e0177f4eac9bb0ff75d7981c27",
      "b86df2b6d0554306bc570bd8d6ec7d7e",
      "b37515616973430cae58fe29b350660b",
      "bfc5f9b4f5874ba5b43180b03d7320fe",
      "238746b3bbf04e2298c2a355345fa1d3",
      "5bb660c119794b6abc5d5b0643a2cbfc",
      "db1b195f9e60412fa54a092851b8bf14",
      "701a9da19a2b419bb54e6a746cd1037c",
      "c758513fa19148eb92d089d69e745d20",
      "5eb8457f4c8a4164827f1e2309829d23",
      "30343d6c72a44bf8bb2640892c3c479d",
      "99c3f363e10e48fc84b7dc8f86df3d53",
      "ea62c2f49be941e28ca5813abdfe33da"
     ]
    },
    "colab_type": "code",
    "id": "csjpdrBoNzSI",
    "outputId": "3f1b28bd-0261-4d53-b83e-4ab5d89a5485"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4e007ce6f520428c9282c1b4d2840b88",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=760.0, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8d6e476417c2496588818f4ab41ec649",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=798011.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5bb660c119794b6abc5d5b0643a2cbfc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=565485600.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at xlnet-base-cased were not used when initializing TFXLNetModel: ['lm_loss']\n",
      "- This IS expected if you are initializing TFXLNetModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPretraining model).\n",
      "- This IS NOT expected if you are initializing TFXLNetModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "All the weights of TFXLNetModel were initialized from the model checkpoint at xlnet-base-cased.\n",
      "If your task is similar to the task the model of the ckeckpoint was trained on, you can already use TFXLNetModel for predictions without further training.\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "from transformers import XLNetConfig, XLNetTokenizer, TFXLNetModel\n",
    " \n",
    "# Load pretrained model/tokenizer\n",
    "config = XLNetConfig.from_pretrained('xlnet-base-cased', output_hidden_states=True)\n",
    "xlnet_tokenizer = XLNetTokenizer.from_pretrained('xlnet-base-cased')\n",
    "xlnet_model = TFXLNetModel.from_pretrained('xlnet-base-cased', config = config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 50
    },
    "colab_type": "code",
    "id": "XO40tarqNzSX",
    "outputId": "1dbf10e2-f238-4563-8104-22926bf44af4"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5349, 512) (5349, 512) (5349, 512) (730, 512) (730, 512) (730, 512)\n",
      "(5349, 512) (5349, 512) (5349, 512) (730, 512) (730, 512) (730, 512)\n"
     ]
    }
   ],
   "source": [
    "# Tokenizing : title_body\n",
    "title_body_train_tokens = xlnet_tokenizer.batch_encode_plus(title_body_train, max_length = 512, truncation = True, pad_to_max_length = True, return_tensors=\"tf\")\n",
    "title_body_cv_tokens = xlnet_tokenizer.batch_encode_plus(title_body_cv, max_length = 512, truncation = True, pad_to_max_length = True, return_tensors=\"tf\")\n",
    "# tokens = bert_tokenizer.batch_encode_plus(final_train, max_length = 50, truncation = True, pad_to_max_length = True)\n",
    "\n",
    "tb_train_input_ids  = np.array(title_body_train_tokens['input_ids'])\n",
    "tb_train_attn_mask = np.array(title_body_train_tokens['attention_mask'])\n",
    "tb_train_token_typ_ids = np.array(title_body_train_tokens['token_type_ids'])\n",
    " \n",
    "tb_cv_input_ids  = np.array(title_body_cv_tokens['input_ids'])\n",
    "tb_cv_attn_mask = np.array(title_body_cv_tokens['attention_mask'])\n",
    "tb_cv_token_typ_ids = np.array(title_body_cv_tokens['token_type_ids'])\n",
    "\n",
    "print(tb_train_input_ids.shape, tb_train_attn_mask.shape, tb_train_token_typ_ids.shape, tb_cv_input_ids.shape, tb_cv_attn_mask.shape, tb_cv_token_typ_ids.shape)\n",
    "\n",
    "# Tokenizing : answer\n",
    "ans_train_tokens = xlnet_tokenizer.batch_encode_plus(answer_train, max_length = 512, truncation = True, pad_to_max_length = True, return_tensors=\"tf\")\n",
    "ans_cv_tokens = xlnet_tokenizer.batch_encode_plus(answer_cv, max_length = 512, truncation = True, pad_to_max_length = True, return_tensors=\"tf\")\n",
    "ans_train_input_ids  = np.array(ans_train_tokens['input_ids'])\n",
    "ans_train_attn_mask = np.array(ans_train_tokens['attention_mask'])\n",
    "ans_train_token_typ_ids = np.array(ans_train_tokens['token_type_ids'])\n",
    " \n",
    "ans_cv_input_ids  = np.array(ans_cv_tokens['input_ids'])\n",
    "ans_cv_attn_mask = np.array(ans_cv_tokens['attention_mask'])\n",
    "ans_cv_token_typ_ids = np.array(ans_cv_tokens['token_type_ids'])\n",
    "seq_len = ans_train_input_ids.shape[1]\n",
    " \n",
    "print(ans_train_input_ids.shape, ans_train_attn_mask.shape, ans_train_token_typ_ids.shape, ans_cv_input_ids.shape, ans_cv_attn_mask.shape, ans_cv_token_typ_ids.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "lzTal6IYNzSk"
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "tf.compat.v1.enable_eager_execution()\n",
    "from tensorflow.keras.layers import Input, Dense, Dropout\n",
    "from tensorflow.keras.models import Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 146,
     "referenced_widgets": [
      "99ab339a4402497cbb80572366c49aae",
      "eefaad3cdef94003ba02408f4784f9e0",
      "62cca8a6dc4741e1a48ed4ada9b3bb06",
      "bd630f0cc09a45cbba3910b75e4c823e",
      "147d5e4ea5994be9a4e6df38a0c78fae",
      "c8f8ea592c654181ac1655d7d9cf0f84",
      "082e3889c7e24cbcaf7f8d8fd40e33e5",
      "635ee408dfe3404c99d9b08f7b7db947",
      "17832dfd8ffd4584a81c26a43667b95b",
      "799165714cfa45e4bcb44700e6750468",
      "de2ea4cf8b9b4a64948669b14624e3f1",
      "8434e38f4f6c4e6084c74d2758c3b9db",
      "f5301ead168c4027975a39dd89244805",
      "e0ae7eb7d3954636bbd95fa70697a534",
      "9da9b44a2e66438b8916d24c89937b46",
      "1659f99e09c5446bb3b980e3841886eb"
     ]
    },
    "colab_type": "code",
    "id": "Xyag1bP3-1mz",
    "outputId": "8b3f7143-7f2b-44db-b67a-0ac3b55b867a"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "99ab339a4402497cbb80572366c49aae",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=168.0), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "(5349, 3072)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "17832dfd8ffd4584a81c26a43667b95b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "(730, 3072)\n"
     ]
    }
   ],
   "source": [
    "# train : title_body XLNet embeddings\n",
    "batch_size = 32\n",
    "l = tb_train_input_ids.shape[0]\n",
    "\n",
    "lst1 = []\n",
    "for i in tqdm(range((l//batch_size)+1)):\n",
    "  x = xlnet_model([tb_train_input_ids[i*batch_size:(i+1)*batch_size], tb_train_attn_mask[i*batch_size:(i+1)*batch_size]])[1]\n",
    "\n",
    "  # getting hidden states of last 4 layers (avg of each timrstep)\n",
    "  lst = [tf.reduce_mean(i, axis = 1) for i in x[-4:]]\n",
    "\n",
    "  # concat all 4 averaged hidden_states\n",
    "  conc = tf.concat(lst, axis = 1)\n",
    "  lst1.append(conc)\n",
    "tb_XLNet_train = tf.concat(lst1, axis = 0)\n",
    "print(tb_XLNet_train.shape)\n",
    "\n",
    "# cv : title_body XLNet embeddings\n",
    "batch_size = 32\n",
    "l = tb_cv_input_ids.shape[0]\n",
    "\n",
    "lst1 = []\n",
    "for i in tqdm(range((l//batch_size)+1)):\n",
    "  x = xlnet_model([tb_cv_input_ids[i*batch_size:(i+1)*batch_size], tb_cv_attn_mask[i*batch_size:(i+1)*batch_size]])[1]\n",
    "  # getting hidden states of last 4 layers (avg of each timrstep)\n",
    "  lst = [tf.reduce_mean(i, axis = 1) for i in x[-4:]]\n",
    "  # lst = [i[:, 0, :] for i in x[-4:]]\n",
    "  conc = tf.concat(lst, axis = 1)\n",
    "  lst1.append(conc)\n",
    "tb_XLNet_cv = tf.concat(lst1, axis = 0)\n",
    "print(tb_XLNet_cv.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 146,
     "referenced_widgets": [
      "180363b7b5c04a00bb8e95c1ae645eab",
      "93fc96a9b550409986c23a0ef8ff07d7",
      "e6ea074710014546bd2dc007e53e0865",
      "9160b4ee7e0f42daa271dea4cb1a82a3",
      "7d30fdccfc8043d0ba305ab56a40c54e",
      "abbb139816074ebeaecce3f12f994198",
      "62e7d665f2df4e6a8a9a74c5e31fe2e3",
      "cecec159b882408e861c4e009b0979e3",
      "9b1a02d275ed419ea593a1a646dd982b",
      "68c7efc4afa949e583a4fc418965902b",
      "8cbba98eeb584eec9704921f7e7442c8",
      "611f465a910c4865b1f70b0051c99c8b",
      "8793c64d528f49be821e254358c2d2e3",
      "62a3f36d22c34284966620513a0a4248",
      "87c171348f974f25a139767a23748e04",
      "5adff9692eca4311a2157cac94a522d4"
     ]
    },
    "colab_type": "code",
    "id": "sQuyfTNd-1nE",
    "outputId": "f8fef907-8725-4e32-f40f-5931346b508e"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "180363b7b5c04a00bb8e95c1ae645eab",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=168.0), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "(5349, 3072)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9b1a02d275ed419ea593a1a646dd982b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "(730, 3072)\n"
     ]
    }
   ],
   "source": [
    "# train : ans XLNet embeddings\n",
    "batch_size = 32\n",
    "l = ans_train_input_ids.shape[0]\n",
    "\n",
    "lst1 = []\n",
    "for i in tqdm(range((l//batch_size)+1)):\n",
    "  x = xlnet_model([ans_train_input_ids[i*batch_size:(i+1)*batch_size], ans_train_attn_mask[i*batch_size:(i+1)*batch_size]])[1]\n",
    "  # getting hidden states of last 4 layers (avg of each timrstep)\n",
    "  lst = [tf.reduce_mean(i, axis = 1) for i in x[-4:]]\n",
    "  conc = tf.concat(lst, axis = 1)\n",
    "  lst1.append(conc)\n",
    "ans_XLNet_train = tf.concat(lst1, axis = 0)\n",
    "print(ans_XLNet_train.shape)\n",
    "\n",
    "# cv : ans XLNet embeddings\n",
    "batch_size = 32\n",
    "l = ans_cv_input_ids.shape[0]\n",
    "lst1 = []\n",
    "for i in tqdm(range((l//batch_size)+1)):\n",
    "  x = xlnet_model([ans_cv_input_ids[i*batch_size:(i+1)*batch_size], ans_cv_attn_mask[i*batch_size:(i+1)*batch_size]])[1]\n",
    "  # getting hidden states of last 4 layers (avg of each timrstep)\n",
    "  lst = [tf.reduce_mean(i, axis = 1) for i in x[-4:]]\n",
    "  conc = tf.concat(lst, axis = 1)\n",
    "  lst1.append(conc)\n",
    "ans_XLNet_cv = tf.concat(lst1, axis = 0)\n",
    "print(ans_XLNet_cv.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "7PBLIPivZMTD"
   },
   "outputs": [],
   "source": [
    "# saving pretrained xlnet embeddings\n",
    "np.savez_compressed('/content/drive/My Drive/AAIC Course/Personal case study 2  - Google Quest/xlnet_outputs', tb_XLNet_train = tb_XLNet_train.numpy(), tb_XLNet_cv = tb_XLNet_cv.numpy(), \n",
    "                    ans_XLNet_train = ans_XLNet_train.numpy(), ans_XLNet_cv = ans_XLNet_cv.numpy())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "baDXlmxnZWr5"
   },
   "source": [
    "### Modelling :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "7McdLg-BNzSq"
   },
   "outputs": [],
   "source": [
    "# loading pretrained xlnet embeddings\n",
    "tb_XLNet_train = np.load('/content/drive/My Drive/AAIC Course/Personal case study 2  - Google Quest/xlnet_outputs.npz')['tb_XLNet_train']\n",
    "tb_XLNet_cv = np.load('/content/drive/My Drive/AAIC Course/Personal case study 2  - Google Quest/xlnet_outputs.npz')['tb_XLNet_cv']\n",
    "ans_XLNet_train = np.load('/content/drive/My Drive/AAIC Course/Personal case study 2  - Google Quest/xlnet_outputs.npz')['ans_XLNet_train']\n",
    "ans_XLNet_cv = np.load('/content/drive/My Drive/AAIC Course/Personal case study 2  - Google Quest/xlnet_outputs.npz')['ans_XLNet_cv']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 651
    },
    "colab_type": "code",
    "id": "_0rEkzpONzS_",
    "outputId": "c770b8a3-8956-449a-c0af-db1d1f63ee7a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"functional_1\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "tb_roberta_out (InputLayer)     [(None, 3072)]       0                                            \n",
      "__________________________________________________________________________________________________\n",
      "ans_roberta_out (InputLayer)    [(None, 3072)]       0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dropout (Dropout)               (None, 3072)         0           tb_roberta_out[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dropout_3 (Dropout)             (None, 3072)         0           ans_roberta_out[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dense (Dense)                   (None, 1536)         4720128     dropout[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_3 (Dense)                 (None, 1024)         3146752     dropout_3[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)             (None, 1536)         0           dense[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "dropout_4 (Dropout)             (None, 1024)         0           dense_3[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 512)          786944      dropout_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_4 (Dense)                 (None, 512)          524800      dropout_4[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dropout_2 (Dropout)             (None, 512)          0           dense_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dropout_5 (Dropout)             (None, 512)          0           dense_4[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 21)           10773       dropout_2[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_5 (Dense)                 (None, 9)            4617        dropout_5[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_concat (TensorFlowO [(None, 30)]         0           dense_2[0][0]                    \n",
      "                                                                 dense_5[0][0]                    \n",
      "==================================================================================================\n",
      "Total params: 9,194,014\n",
      "Trainable params: 9,194,014\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "tf.keras.backend.clear_session()\n",
    "seed = 42\n",
    "\n",
    "# *-----------------title_body-----------------*\n",
    "tb_input = Input(name = 'tb_roberta_out', shape = (768*4,), dtype = 'float32')\n",
    "tb_dropout_1 = Dropout(rate = 0.2, seed = seed)(tb_input)\n",
    " \n",
    "tb_dense_1 = Dense(units = 1536, activation = 'relu', kernel_initializer = tf.keras.initializers.he_normal(seed = seed))(tb_dropout_1)\n",
    "tb_dropout_2 = Dropout(rate = 0.2, seed = seed)(tb_dense_1)\n",
    " \n",
    "tb_dense_2 = Dense(units = 512, activation = 'relu', kernel_initializer = tf.keras.initializers.he_normal(seed = seed))(tb_dropout_2)\n",
    "tb_dropout_3 = Dropout(rate = 0.1, seed = seed)(tb_dense_2)\n",
    "\n",
    "tb_out = Dense(units = 21, activation = 'sigmoid', kernel_initializer = tf.keras.initializers.he_normal(seed = seed))(tb_dropout_3)\n",
    " \n",
    "# *-----------------answer_model-----------------*\n",
    "ans_input = Input(name = 'ans_roberta_out', shape = (768*4,), dtype = 'float32')\n",
    "# concat = tf.concat([tb_out, ans_input], axis = -1) \n",
    "ans_dropout_1 = Dropout(rate = 0.2, seed = seed)(ans_input)\n",
    " \n",
    "ans_dense_1 = Dense(units = 1024, activation = 'relu', kernel_initializer = tf.keras.initializers.he_normal(seed = seed))(ans_dropout_1)\n",
    "ans_dropout_2 = Dropout(rate = 0.2, seed = seed)(ans_dense_1)\n",
    " \n",
    "ans_dense_2 = Dense(units = 512, activation = 'relu', kernel_initializer = tf.keras.initializers.he_normal(seed = seed))(ans_dropout_2)\n",
    "ans_dropout_3 = Dropout(rate = 0.1, seed = seed)(ans_dense_2)\n",
    "\n",
    "ans_out = Dense(units = 9, activation = 'sigmoid', kernel_initializer = tf.keras.initializers.he_normal(seed = seed))(ans_dropout_3)\n",
    "\n",
    "# *--------------------concat--------------------*\n",
    "out = tf.concat([tb_out, ans_out], axis = -1) \n",
    " \n",
    "model = Model(inputs = [tb_input, ans_input], outputs = out)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 33
    },
    "colab_type": "code",
    "id": "1-0oTkdqNzTT",
    "outputId": "895aad57-4146-4790-826a-6d5009f945d2"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`write_grads` will be ignored in TensorFlow 2.0 for the `TensorBoard` Callback.\n"
     ]
    }
   ],
   "source": [
    "# Defining callbacks\n",
    "!rm -r '/content/saved models'\n",
    "!rm -r '/content/logs'\n",
    "!mkdir '/content/saved models'\n",
    "!mkdir '/content/logs/'\n",
    " \n",
    "# tensorboard callback\n",
    "import datetime\n",
    "log_dir=\"logs/\" + datetime.datetime.now().strftime(\"%Y-%m-%d %H_%M_%S\")\n",
    "tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir = log_dir, histogram_freq=1, write_graph=True, write_grads=True)\n",
    " \n",
    "reduce_lr = tf.keras.callbacks.ReduceLROnPlateau(monitor= 'val_loss', factor=np.sqrt(0.1), patience=7, verbose=1)\n",
    " \n",
    "checkpt = tf.keras.callbacks.ModelCheckpoint('/content/saved models/weights.-{epoch:03d}-{val_loss:.5f}', monitor='val_loss', verbose=1, save_best_only=False, save_weights_only=True)\n",
    " \n",
    "print_spearman_fn = print_spearman(train_data = ([tb_XLNet_train, ans_XLNet_train], y_train.values),\n",
    "                                 validation_data = ([tb_XLNet_cv, ans_XLNet_cv], y_cv.values))\n",
    "callbacks = [print_spearman_fn, reduce_lr, checkpt, tensorboard_callback]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "YHQnjpAQDsoC",
    "outputId": "ff778f63-2dca-4eb6-a5f5-444f76703454"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/75\n",
      " 2/84 [..............................] - ETA: 3s - loss: 1.9366 - root_mean_squared_error: 0.5618WARNING:tensorflow:Callbacks method `on_train_batch_end` is slow compared to the batch time (batch time: 0.0076s vs `on_train_batch_end` time: 0.0736s). Check your callbacks.\n",
      "77/84 [==========================>...] - ETA: 0s - loss: 0.7246 - root_mean_squared_error: 0.3367\n",
      "spearman :\n",
      "train_spearman : 0.16392 | val_spearman : 0.15531 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00001: saving model to /content/saved models/weights.-001-0.42974\n",
      "84/84 [==============================] - 2s 23ms/step - loss: 0.7093 - root_mean_squared_error: 0.3333 - val_loss: 0.4297 - val_root_mean_squared_error: 0.2433\n",
      "Epoch 2/75\n",
      "84/84 [==============================] - ETA: 0s - loss: 0.4871 - root_mean_squared_error: 0.2764\n",
      "spearman :\n",
      "train_spearman : 0.17559 | val_spearman : 0.17226 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00002: saving model to /content/saved models/weights.-002-0.42308\n",
      "84/84 [==============================] - 2s 20ms/step - loss: 0.4871 - root_mean_squared_error: 0.2764 - val_loss: 0.4231 - val_root_mean_squared_error: 0.2395\n",
      "Epoch 3/75\n",
      "76/84 [==========================>...] - ETA: 0s - loss: 0.4495 - root_mean_squared_error: 0.2585\n",
      "spearman :\n",
      "train_spearman : 0.19335 | val_spearman : 0.20855 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00003: saving model to /content/saved models/weights.-003-0.41867\n",
      "84/84 [==============================] - 2s 24ms/step - loss: 0.4492 - root_mean_squared_error: 0.2582 - val_loss: 0.4187 - val_root_mean_squared_error: 0.2365\n",
      "Epoch 4/75\n",
      "76/84 [==========================>...] - ETA: 0s - loss: 0.4328 - root_mean_squared_error: 0.2490\n",
      "spearman :\n",
      "train_spearman : 0.21641 | val_spearman : 0.21935 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00004: saving model to /content/saved models/weights.-004-0.41265\n",
      "84/84 [==============================] - 2s 20ms/step - loss: 0.4323 - root_mean_squared_error: 0.2487 - val_loss: 0.4127 - val_root_mean_squared_error: 0.2328\n",
      "Epoch 5/75\n",
      "78/84 [==========================>...] - ETA: 0s - loss: 0.4224 - root_mean_squared_error: 0.2421\n",
      "spearman :\n",
      "train_spearman : 0.24054 | val_spearman : 0.24004 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00005: saving model to /content/saved models/weights.-005-0.40907\n",
      "84/84 [==============================] - 2s 20ms/step - loss: 0.4227 - root_mean_squared_error: 0.2421 - val_loss: 0.4091 - val_root_mean_squared_error: 0.2301\n",
      "Epoch 6/75\n",
      "80/84 [===========================>..] - ETA: 0s - loss: 0.4178 - root_mean_squared_error: 0.2389\n",
      "spearman :\n",
      "train_spearman : 0.2705 | val_spearman : 0.25795 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00006: saving model to /content/saved models/weights.-006-0.40627\n",
      "84/84 [==============================] - 2s 19ms/step - loss: 0.4175 - root_mean_squared_error: 0.2388 - val_loss: 0.4063 - val_root_mean_squared_error: 0.2281\n",
      "Epoch 7/75\n",
      "75/84 [=========================>....] - ETA: 0s - loss: 0.4128 - root_mean_squared_error: 0.2350\n",
      "spearman :\n",
      "train_spearman : 0.2774 | val_spearman : 0.25839 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00007: saving model to /content/saved models/weights.-007-0.40547\n",
      "84/84 [==============================] - 2s 20ms/step - loss: 0.4128 - root_mean_squared_error: 0.2355 - val_loss: 0.4055 - val_root_mean_squared_error: 0.2275\n",
      "Epoch 8/75\n",
      "78/84 [==========================>...] - ETA: 0s - loss: 0.4089 - root_mean_squared_error: 0.2328\n",
      "spearman :\n",
      "train_spearman : 0.29352 | val_spearman : 0.27519 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00008: saving model to /content/saved models/weights.-008-0.40322\n",
      "84/84 [==============================] - 2s 20ms/step - loss: 0.4090 - root_mean_squared_error: 0.2327 - val_loss: 0.4032 - val_root_mean_squared_error: 0.2261\n",
      "Epoch 9/75\n",
      "80/84 [===========================>..] - ETA: 0s - loss: 0.4055 - root_mean_squared_error: 0.2301\n",
      "spearman :\n",
      "train_spearman : 0.29857 | val_spearman : 0.27967 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00009: saving model to /content/saved models/weights.-009-0.40131\n",
      "84/84 [==============================] - 2s 19ms/step - loss: 0.4054 - root_mean_squared_error: 0.2301 - val_loss: 0.4013 - val_root_mean_squared_error: 0.2248\n",
      "Epoch 10/75\n",
      "77/84 [==========================>...] - ETA: 0s - loss: 0.4039 - root_mean_squared_error: 0.2289\n",
      "spearman :\n",
      "train_spearman : 0.3147 | val_spearman : 0.28488 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00010: saving model to /content/saved models/weights.-010-0.39933\n",
      "84/84 [==============================] - 2s 21ms/step - loss: 0.4037 - root_mean_squared_error: 0.2289 - val_loss: 0.3993 - val_root_mean_squared_error: 0.2240\n",
      "Epoch 11/75\n",
      "78/84 [==========================>...] - ETA: 0s - loss: 0.4017 - root_mean_squared_error: 0.2273\n",
      "spearman :\n",
      "train_spearman : 0.31836 | val_spearman : 0.29988 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00011: saving model to /content/saved models/weights.-011-0.39839\n",
      "84/84 [==============================] - 2s 19ms/step - loss: 0.4014 - root_mean_squared_error: 0.2270 - val_loss: 0.3984 - val_root_mean_squared_error: 0.2230\n",
      "Epoch 12/75\n",
      "79/84 [===========================>..] - ETA: 0s - loss: 0.3994 - root_mean_squared_error: 0.2256\n",
      "spearman :\n",
      "train_spearman : 0.33252 | val_spearman : 0.30488 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00012: saving model to /content/saved models/weights.-012-0.39698\n",
      "84/84 [==============================] - 2s 22ms/step - loss: 0.3990 - root_mean_squared_error: 0.2253 - val_loss: 0.3970 - val_root_mean_squared_error: 0.2222\n",
      "Epoch 13/75\n",
      "78/84 [==========================>...] - ETA: 0s - loss: 0.3972 - root_mean_squared_error: 0.2242\n",
      "spearman :\n",
      "train_spearman : 0.32226 | val_spearman : 0.29676 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00013: saving model to /content/saved models/weights.-013-0.39630\n",
      "84/84 [==============================] - 2s 20ms/step - loss: 0.3972 - root_mean_squared_error: 0.2240 - val_loss: 0.3963 - val_root_mean_squared_error: 0.2215\n",
      "Epoch 14/75\n",
      "81/84 [===========================>..] - ETA: 0s - loss: 0.3952 - root_mean_squared_error: 0.2224\n",
      "spearman :\n",
      "train_spearman : 0.32844 | val_spearman : 0.30667 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00014: saving model to /content/saved models/weights.-014-0.39772\n",
      "84/84 [==============================] - 2s 19ms/step - loss: 0.3956 - root_mean_squared_error: 0.2227 - val_loss: 0.3977 - val_root_mean_squared_error: 0.2223\n",
      "Epoch 15/75\n",
      "78/84 [==========================>...] - ETA: 0s - loss: 0.3945 - root_mean_squared_error: 0.2222\n",
      "spearman :\n",
      "train_spearman : 0.35037 | val_spearman : 0.32223 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00015: saving model to /content/saved models/weights.-015-0.39439\n",
      "84/84 [==============================] - 2s 22ms/step - loss: 0.3945 - root_mean_squared_error: 0.2221 - val_loss: 0.3944 - val_root_mean_squared_error: 0.2198\n",
      "Epoch 16/75\n",
      "80/84 [===========================>..] - ETA: 0s - loss: 0.3927 - root_mean_squared_error: 0.2204\n",
      "spearman :\n",
      "train_spearman : 0.35562 | val_spearman : 0.32489 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00016: saving model to /content/saved models/weights.-016-0.39280\n",
      "84/84 [==============================] - 2s 20ms/step - loss: 0.3927 - root_mean_squared_error: 0.2206 - val_loss: 0.3928 - val_root_mean_squared_error: 0.2189\n",
      "Epoch 17/75\n",
      "80/84 [===========================>..] - ETA: 0s - loss: 0.3911 - root_mean_squared_error: 0.2194\n",
      "spearman :\n",
      "train_spearman : 0.35127 | val_spearman : 0.32792 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00017: saving model to /content/saved models/weights.-017-0.39386\n",
      "84/84 [==============================] - 2s 21ms/step - loss: 0.3918 - root_mean_squared_error: 0.2199 - val_loss: 0.3939 - val_root_mean_squared_error: 0.2193\n",
      "Epoch 18/75\n",
      "74/84 [=========================>....] - ETA: 0s - loss: 0.3913 - root_mean_squared_error: 0.2194\n",
      "spearman :\n",
      "train_spearman : 0.36095 | val_spearman : 0.32475 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00018: saving model to /content/saved models/weights.-018-0.39335\n",
      "84/84 [==============================] - 2s 21ms/step - loss: 0.3907 - root_mean_squared_error: 0.2190 - val_loss: 0.3934 - val_root_mean_squared_error: 0.2190\n",
      "Epoch 19/75\n",
      "83/84 [============================>.] - ETA: 0s - loss: 0.3891 - root_mean_squared_error: 0.2179\n",
      "spearman :\n",
      "train_spearman : 0.35368 | val_spearman : 0.31385 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00019: saving model to /content/saved models/weights.-019-0.39362\n",
      "84/84 [==============================] - 2s 22ms/step - loss: 0.3890 - root_mean_squared_error: 0.2178 - val_loss: 0.3936 - val_root_mean_squared_error: 0.2196\n",
      "Epoch 20/75\n",
      "74/84 [=========================>....] - ETA: 0s - loss: 0.3873 - root_mean_squared_error: 0.2170\n",
      "spearman :\n",
      "train_spearman : 0.36506 | val_spearman : 0.31714 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00020: saving model to /content/saved models/weights.-020-0.39543\n",
      "84/84 [==============================] - 2s 20ms/step - loss: 0.3882 - root_mean_squared_error: 0.2171 - val_loss: 0.3954 - val_root_mean_squared_error: 0.2201\n",
      "Epoch 21/75\n",
      "84/84 [==============================] - ETA: 0s - loss: 0.3875 - root_mean_squared_error: 0.2166\n",
      "spearman :\n",
      "train_spearman : 0.38189 | val_spearman : 0.33553 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00021: saving model to /content/saved models/weights.-021-0.38991\n",
      "84/84 [==============================] - 2s 23ms/step - loss: 0.3875 - root_mean_squared_error: 0.2166 - val_loss: 0.3899 - val_root_mean_squared_error: 0.2166\n",
      "Epoch 22/75\n",
      "76/84 [==========================>...] - ETA: 0s - loss: 0.3858 - root_mean_squared_error: 0.2151\n",
      "spearman :\n",
      "train_spearman : 0.37442 | val_spearman : 0.33086 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00022: saving model to /content/saved models/weights.-022-0.39124\n",
      "84/84 [==============================] - 2s 21ms/step - loss: 0.3858 - root_mean_squared_error: 0.2152 - val_loss: 0.3912 - val_root_mean_squared_error: 0.2172\n",
      "Epoch 23/75\n",
      "75/84 [=========================>....] - ETA: 0s - loss: 0.3853 - root_mean_squared_error: 0.2150\n",
      "spearman :\n",
      "train_spearman : 0.38919 | val_spearman : 0.34534 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00023: saving model to /content/saved models/weights.-023-0.39197\n",
      "84/84 [==============================] - 2s 21ms/step - loss: 0.3855 - root_mean_squared_error: 0.2151 - val_loss: 0.3920 - val_root_mean_squared_error: 0.2180\n",
      "Epoch 24/75\n",
      "77/84 [==========================>...] - ETA: 0s - loss: 0.3840 - root_mean_squared_error: 0.2139\n",
      "spearman :\n",
      "train_spearman : 0.37957 | val_spearman : 0.33076 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00024: saving model to /content/saved models/weights.-024-0.39111\n",
      "84/84 [==============================] - 2s 20ms/step - loss: 0.3840 - root_mean_squared_error: 0.2139 - val_loss: 0.3911 - val_root_mean_squared_error: 0.2172\n",
      "Epoch 25/75\n",
      "79/84 [===========================>..] - ETA: 0s - loss: 0.3836 - root_mean_squared_error: 0.2134\n",
      "spearman :\n",
      "train_spearman : 0.4029 | val_spearman : 0.34313 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00025: saving model to /content/saved models/weights.-025-0.38994\n",
      "84/84 [==============================] - 2s 21ms/step - loss: 0.3836 - root_mean_squared_error: 0.2135 - val_loss: 0.3899 - val_root_mean_squared_error: 0.2163\n",
      "Epoch 26/75\n",
      "80/84 [===========================>..] - ETA: 0s - loss: 0.3827 - root_mean_squared_error: 0.2127\n",
      "spearman :\n",
      "train_spearman : 0.38634 | val_spearman : 0.33825 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00026: saving model to /content/saved models/weights.-026-0.38876\n",
      "84/84 [==============================] - 2s 19ms/step - loss: 0.3825 - root_mean_squared_error: 0.2126 - val_loss: 0.3888 - val_root_mean_squared_error: 0.2156\n",
      "Epoch 27/75\n",
      "75/84 [=========================>....] - ETA: 0s - loss: 0.3817 - root_mean_squared_error: 0.2120\n",
      "spearman :\n",
      "train_spearman : 0.39839 | val_spearman : 0.35286 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00027: saving model to /content/saved models/weights.-027-0.38932\n",
      "84/84 [==============================] - 2s 21ms/step - loss: 0.3819 - root_mean_squared_error: 0.2123 - val_loss: 0.3893 - val_root_mean_squared_error: 0.2160\n",
      "Epoch 28/75\n",
      "74/84 [=========================>....] - ETA: 0s - loss: 0.3803 - root_mean_squared_error: 0.2110\n",
      "spearman :\n",
      "train_spearman : 0.40627 | val_spearman : 0.35089 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00028: saving model to /content/saved models/weights.-028-0.38955\n",
      "84/84 [==============================] - 2s 26ms/step - loss: 0.3805 - root_mean_squared_error: 0.2111 - val_loss: 0.3896 - val_root_mean_squared_error: 0.2159\n",
      "Epoch 29/75\n",
      "76/84 [==========================>...] - ETA: 0s - loss: 0.3808 - root_mean_squared_error: 0.2111\n",
      "spearman :\n",
      "train_spearman : 0.4173 | val_spearman : 0.35777 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00029: saving model to /content/saved models/weights.-029-0.38751\n",
      "84/84 [==============================] - 2s 21ms/step - loss: 0.3809 - root_mean_squared_error: 0.2114 - val_loss: 0.3875 - val_root_mean_squared_error: 0.2143\n",
      "Epoch 30/75\n",
      "81/84 [===========================>..] - ETA: 0s - loss: 0.3782 - root_mean_squared_error: 0.2092\n",
      "spearman :\n",
      "train_spearman : 0.40867 | val_spearman : 0.35134 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00030: saving model to /content/saved models/weights.-030-0.39060\n",
      "84/84 [==============================] - 3s 30ms/step - loss: 0.3785 - root_mean_squared_error: 0.2095 - val_loss: 0.3906 - val_root_mean_squared_error: 0.2163\n",
      "Epoch 31/75\n",
      "76/84 [==========================>...] - ETA: 0s - loss: 0.3793 - root_mean_squared_error: 0.2101\n",
      "spearman :\n",
      "train_spearman : 0.42571 | val_spearman : 0.35592 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00031: saving model to /content/saved models/weights.-031-0.38800\n",
      "84/84 [==============================] - 6s 76ms/step - loss: 0.3789 - root_mean_squared_error: 0.2099 - val_loss: 0.3880 - val_root_mean_squared_error: 0.2147\n",
      "Epoch 32/75\n",
      "78/84 [==========================>...] - ETA: 0s - loss: 0.3772 - root_mean_squared_error: 0.2085\n",
      "spearman :\n",
      "train_spearman : 0.41681 | val_spearman : 0.35133 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00032: saving model to /content/saved models/weights.-032-0.38744\n",
      "84/84 [==============================] - 2s 20ms/step - loss: 0.3773 - root_mean_squared_error: 0.2086 - val_loss: 0.3874 - val_root_mean_squared_error: 0.2143\n",
      "Epoch 33/75\n",
      "77/84 [==========================>...] - ETA: 0s - loss: 0.3768 - root_mean_squared_error: 0.2080\n",
      "spearman :\n",
      "train_spearman : 0.42046 | val_spearman : 0.35833 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00033: saving model to /content/saved models/weights.-033-0.38843\n",
      "84/84 [==============================] - 2s 20ms/step - loss: 0.3769 - root_mean_squared_error: 0.2083 - val_loss: 0.3884 - val_root_mean_squared_error: 0.2151\n",
      "Epoch 34/75\n",
      "77/84 [==========================>...] - ETA: 0s - loss: 0.3768 - root_mean_squared_error: 0.2082\n",
      "spearman :\n",
      "train_spearman : 0.42152 | val_spearman : 0.35548 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00034: saving model to /content/saved models/weights.-034-0.38852\n",
      "84/84 [==============================] - 2s 23ms/step - loss: 0.3764 - root_mean_squared_error: 0.2080 - val_loss: 0.3885 - val_root_mean_squared_error: 0.2151\n",
      "Epoch 35/75\n",
      "80/84 [===========================>..] - ETA: 0s - loss: 0.3756 - root_mean_squared_error: 0.2073\n",
      "spearman :\n",
      "train_spearman : 0.42662 | val_spearman : 0.36458 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00035: saving model to /content/saved models/weights.-035-0.38626\n",
      "84/84 [==============================] - 3s 31ms/step - loss: 0.3757 - root_mean_squared_error: 0.2073 - val_loss: 0.3863 - val_root_mean_squared_error: 0.2135\n",
      "Epoch 36/75\n",
      "74/84 [=========================>....] - ETA: 0s - loss: 0.3743 - root_mean_squared_error: 0.2059\n",
      "spearman :\n",
      "train_spearman : 0.43219 | val_spearman : 0.3653 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00036: saving model to /content/saved models/weights.-036-0.38721\n",
      "84/84 [==============================] - 2s 21ms/step - loss: 0.3749 - root_mean_squared_error: 0.2067 - val_loss: 0.3872 - val_root_mean_squared_error: 0.2140\n",
      "Epoch 37/75\n",
      "77/84 [==========================>...] - ETA: 0s - loss: 0.3742 - root_mean_squared_error: 0.2064\n",
      "spearman :\n",
      "train_spearman : 0.4326 | val_spearman : 0.3593 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00037: saving model to /content/saved models/weights.-037-0.38742\n",
      "84/84 [==============================] - 5s 64ms/step - loss: 0.3742 - root_mean_squared_error: 0.2062 - val_loss: 0.3874 - val_root_mean_squared_error: 0.2142\n",
      "Epoch 38/75\n",
      "76/84 [==========================>...] - ETA: 0s - loss: 0.3745 - root_mean_squared_error: 0.2063\n",
      "spearman :\n",
      "train_spearman : 0.43997 | val_spearman : 0.36175 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00038: saving model to /content/saved models/weights.-038-0.38761\n",
      "84/84 [==============================] - 2s 24ms/step - loss: 0.3740 - root_mean_squared_error: 0.2060 - val_loss: 0.3876 - val_root_mean_squared_error: 0.2144\n",
      "Epoch 39/75\n",
      "83/84 [============================>.] - ETA: 0s - loss: 0.3735 - root_mean_squared_error: 0.2055\n",
      "spearman :\n",
      "train_spearman : 0.43825 | val_spearman : 0.36684 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00039: saving model to /content/saved models/weights.-039-0.38762\n",
      "84/84 [==============================] - 2s 22ms/step - loss: 0.3734 - root_mean_squared_error: 0.2055 - val_loss: 0.3876 - val_root_mean_squared_error: 0.2146\n",
      "Epoch 40/75\n",
      "81/84 [===========================>..] - ETA: 0s - loss: 0.3727 - root_mean_squared_error: 0.2051\n",
      "spearman :\n",
      "train_spearman : 0.44113 | val_spearman : 0.36368 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00040: saving model to /content/saved models/weights.-040-0.38584\n",
      "84/84 [==============================] - 7s 80ms/step - loss: 0.3727 - root_mean_squared_error: 0.2051 - val_loss: 0.3858 - val_root_mean_squared_error: 0.2130\n",
      "Epoch 41/75\n",
      "81/84 [===========================>..] - ETA: 0s - loss: 0.3719 - root_mean_squared_error: 0.2040\n",
      "spearman :\n",
      "train_spearman : 0.44902 | val_spearman : 0.37296 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00041: saving model to /content/saved models/weights.-041-0.38605\n",
      "84/84 [==============================] - 2s 19ms/step - loss: 0.3719 - root_mean_squared_error: 0.2042 - val_loss: 0.3860 - val_root_mean_squared_error: 0.2131\n",
      "Epoch 42/75\n",
      "80/84 [===========================>..] - ETA: 0s - loss: 0.3709 - root_mean_squared_error: 0.2037\n",
      "spearman :\n",
      "train_spearman : 0.4524 | val_spearman : 0.36312 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00042: saving model to /content/saved models/weights.-042-0.38779\n",
      "84/84 [==============================] - 2s 20ms/step - loss: 0.3710 - root_mean_squared_error: 0.2035 - val_loss: 0.3878 - val_root_mean_squared_error: 0.2144\n",
      "Epoch 43/75\n",
      "79/84 [===========================>..] - ETA: 0s - loss: 0.3705 - root_mean_squared_error: 0.2038\n",
      "spearman :\n",
      "train_spearman : 0.45463 | val_spearman : 0.36737 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00043: saving model to /content/saved models/weights.-043-0.38688\n",
      "84/84 [==============================] - 2s 20ms/step - loss: 0.3708 - root_mean_squared_error: 0.2035 - val_loss: 0.3869 - val_root_mean_squared_error: 0.2137\n",
      "Epoch 44/75\n",
      "80/84 [===========================>..] - ETA: 0s - loss: 0.3701 - root_mean_squared_error: 0.2026\n",
      "spearman :\n",
      "train_spearman : 0.45495 | val_spearman : 0.36075 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00044: saving model to /content/saved models/weights.-044-0.38546\n",
      "84/84 [==============================] - 5s 57ms/step - loss: 0.3701 - root_mean_squared_error: 0.2029 - val_loss: 0.3855 - val_root_mean_squared_error: 0.2130\n",
      "Epoch 45/75\n",
      "78/84 [==========================>...] - ETA: 0s - loss: 0.3690 - root_mean_squared_error: 0.2018\n",
      "spearman :\n",
      "train_spearman : 0.46076 | val_spearman : 0.36229 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00045: saving model to /content/saved models/weights.-045-0.38617\n",
      "84/84 [==============================] - 2s 20ms/step - loss: 0.3692 - root_mean_squared_error: 0.2022 - val_loss: 0.3862 - val_root_mean_squared_error: 0.2135\n",
      "Epoch 46/75\n",
      "78/84 [==========================>...] - ETA: 0s - loss: 0.3688 - root_mean_squared_error: 0.2016\n",
      "spearman :\n",
      "train_spearman : 0.46657 | val_spearman : 0.37255 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00046: saving model to /content/saved models/weights.-046-0.38637\n",
      "84/84 [==============================] - 2s 20ms/step - loss: 0.3688 - root_mean_squared_error: 0.2017 - val_loss: 0.3864 - val_root_mean_squared_error: 0.2132\n",
      "Epoch 47/75\n",
      "78/84 [==========================>...] - ETA: 0s - loss: 0.3677 - root_mean_squared_error: 0.2009\n",
      "spearman :\n",
      "train_spearman : 0.4705 | val_spearman : 0.36031 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00047: saving model to /content/saved models/weights.-047-0.38624\n",
      "84/84 [==============================] - 2s 21ms/step - loss: 0.3677 - root_mean_squared_error: 0.2009 - val_loss: 0.3862 - val_root_mean_squared_error: 0.2130\n",
      "Epoch 48/75\n",
      "80/84 [===========================>..] - ETA: 0s - loss: 0.3674 - root_mean_squared_error: 0.2006\n",
      "spearman :\n",
      "train_spearman : 0.47228 | val_spearman : 0.36221 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00048: saving model to /content/saved models/weights.-048-0.38494\n",
      "84/84 [==============================] - 2s 29ms/step - loss: 0.3673 - root_mean_squared_error: 0.2006 - val_loss: 0.3849 - val_root_mean_squared_error: 0.2122\n",
      "Epoch 49/75\n",
      "78/84 [==========================>...] - ETA: 0s - loss: 0.3671 - root_mean_squared_error: 0.2002\n",
      "spearman :\n",
      "train_spearman : 0.47377 | val_spearman : 0.36472 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00049: saving model to /content/saved models/weights.-049-0.38560\n",
      "84/84 [==============================] - 6s 70ms/step - loss: 0.3668 - root_mean_squared_error: 0.2001 - val_loss: 0.3856 - val_root_mean_squared_error: 0.2131\n",
      "Epoch 50/75\n",
      "76/84 [==========================>...] - ETA: 0s - loss: 0.3656 - root_mean_squared_error: 0.1995\n",
      "spearman :\n",
      "train_spearman : 0.4823 | val_spearman : 0.37133 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00050: saving model to /content/saved models/weights.-050-0.38644\n",
      "84/84 [==============================] - 2s 21ms/step - loss: 0.3659 - root_mean_squared_error: 0.1996 - val_loss: 0.3864 - val_root_mean_squared_error: 0.2136\n",
      "Epoch 51/75\n",
      "84/84 [==============================] - ETA: 0s - loss: 0.3660 - root_mean_squared_error: 0.1995\n",
      "spearman :\n",
      "train_spearman : 0.48039 | val_spearman : 0.36533 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00051: saving model to /content/saved models/weights.-051-0.38553\n",
      "84/84 [==============================] - 2s 21ms/step - loss: 0.3660 - root_mean_squared_error: 0.1995 - val_loss: 0.3855 - val_root_mean_squared_error: 0.2130\n",
      "Epoch 52/75\n",
      "82/84 [============================>.] - ETA: 0s - loss: 0.3644 - root_mean_squared_error: 0.1983\n",
      "spearman :\n",
      "train_spearman : 0.48505 | val_spearman : 0.3661 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00052: saving model to /content/saved models/weights.-052-0.38489\n",
      "84/84 [==============================] - 2s 25ms/step - loss: 0.3648 - root_mean_squared_error: 0.1984 - val_loss: 0.3849 - val_root_mean_squared_error: 0.2125\n",
      "Epoch 53/75\n",
      "84/84 [==============================] - ETA: 0s - loss: 0.3644 - root_mean_squared_error: 0.1981\n",
      "spearman :\n",
      "train_spearman : 0.48662 | val_spearman : 0.37102 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00053: saving model to /content/saved models/weights.-053-0.38466\n",
      "84/84 [==============================] - 6s 67ms/step - loss: 0.3644 - root_mean_squared_error: 0.1981 - val_loss: 0.3847 - val_root_mean_squared_error: 0.2122\n",
      "Epoch 54/75\n",
      "77/84 [==========================>...] - ETA: 0s - loss: 0.3641 - root_mean_squared_error: 0.1978\n",
      "spearman :\n",
      "train_spearman : 0.48997 | val_spearman : 0.36897 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00054: saving model to /content/saved models/weights.-054-0.38668\n",
      "84/84 [==============================] - 2s 21ms/step - loss: 0.3641 - root_mean_squared_error: 0.1979 - val_loss: 0.3867 - val_root_mean_squared_error: 0.2133\n",
      "Epoch 55/75\n",
      "76/84 [==========================>...] - ETA: 0s - loss: 0.3635 - root_mean_squared_error: 0.1973\n",
      "spearman :\n",
      "train_spearman : 0.49473 | val_spearman : 0.3599 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00055: saving model to /content/saved models/weights.-055-0.38707\n",
      "84/84 [==============================] - 2s 20ms/step - loss: 0.3635 - root_mean_squared_error: 0.1975 - val_loss: 0.3871 - val_root_mean_squared_error: 0.2143\n",
      "Epoch 56/75\n",
      "77/84 [==========================>...] - ETA: 0s - loss: 0.3628 - root_mean_squared_error: 0.1966\n",
      "spearman :\n",
      "train_spearman : 0.49933 | val_spearman : 0.36018 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00056: saving model to /content/saved models/weights.-056-0.38534\n",
      "84/84 [==============================] - 2s 23ms/step - loss: 0.3625 - root_mean_squared_error: 0.1966 - val_loss: 0.3853 - val_root_mean_squared_error: 0.2126\n",
      "Epoch 57/75\n",
      "77/84 [==========================>...] - ETA: 0s - loss: 0.3620 - root_mean_squared_error: 0.1963\n",
      "spearman :\n",
      "train_spearman : 0.49884 | val_spearman : 0.37257 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00057: saving model to /content/saved models/weights.-057-0.38637\n",
      "84/84 [==============================] - 3s 34ms/step - loss: 0.3623 - root_mean_squared_error: 0.1965 - val_loss: 0.3864 - val_root_mean_squared_error: 0.2133\n",
      "Epoch 58/75\n",
      "78/84 [==========================>...] - ETA: 0s - loss: 0.3617 - root_mean_squared_error: 0.1956\n",
      "spearman :\n",
      "train_spearman : 0.50235 | val_spearman : 0.36129 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00058: saving model to /content/saved models/weights.-058-0.38609\n",
      "84/84 [==============================] - 2s 21ms/step - loss: 0.3615 - root_mean_squared_error: 0.1958 - val_loss: 0.3861 - val_root_mean_squared_error: 0.2127\n",
      "Epoch 59/75\n",
      "77/84 [==========================>...] - ETA: 0s - loss: 0.3611 - root_mean_squared_error: 0.1954\n",
      "spearman :\n",
      "train_spearman : 0.50296 | val_spearman : 0.36658 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00059: saving model to /content/saved models/weights.-059-0.38548\n",
      "84/84 [==============================] - 4s 51ms/step - loss: 0.3612 - root_mean_squared_error: 0.1955 - val_loss: 0.3855 - val_root_mean_squared_error: 0.2127\n",
      "Epoch 60/75\n",
      "76/84 [==========================>...] - ETA: 0s - loss: 0.3598 - root_mean_squared_error: 0.1943\n",
      "spearman :\n",
      "train_spearman : 0.50937 | val_spearman : 0.3669 | Learning_Rate : 9.999999747378752e-05\n",
      "\n",
      "Epoch 00060: ReduceLROnPlateau reducing learning rate to 3.1622775802825264e-05.\n",
      "\n",
      "Epoch 00060: saving model to /content/saved models/weights.-060-0.38633\n",
      "84/84 [==============================] - 2s 29ms/step - loss: 0.3602 - root_mean_squared_error: 0.1946 - val_loss: 0.3863 - val_root_mean_squared_error: 0.2133\n",
      "Epoch 61/75\n",
      "77/84 [==========================>...] - ETA: 0s - loss: 0.3587 - root_mean_squared_error: 0.1934\n",
      "spearman :\n",
      "train_spearman : 0.51597 | val_spearman : 0.36928 | Learning_Rate : 3.199999991920777e-05\n",
      "\n",
      "Epoch 00061: saving model to /content/saved models/weights.-061-0.38532\n",
      "84/84 [==============================] - 3s 33ms/step - loss: 0.3585 - root_mean_squared_error: 0.1932 - val_loss: 0.3853 - val_root_mean_squared_error: 0.2124\n",
      "Epoch 62/75\n",
      "77/84 [==========================>...] - ETA: 0s - loss: 0.3577 - root_mean_squared_error: 0.1923\n",
      "spearman :\n",
      "train_spearman : 0.51677 | val_spearman : 0.3724 | Learning_Rate : 3.199999991920777e-05\n",
      "\n",
      "Epoch 00062: saving model to /content/saved models/weights.-062-0.38561\n",
      "84/84 [==============================] - 3s 33ms/step - loss: 0.3579 - root_mean_squared_error: 0.1927 - val_loss: 0.3856 - val_root_mean_squared_error: 0.2125\n",
      "Epoch 63/75\n",
      "83/84 [============================>.] - ETA: 0s - loss: 0.3568 - root_mean_squared_error: 0.1914\n",
      "spearman :\n",
      "train_spearman : 0.51868 | val_spearman : 0.37176 | Learning_Rate : 3.199999991920777e-05\n",
      "\n",
      "Epoch 00063: saving model to /content/saved models/weights.-063-0.38420\n",
      "84/84 [==============================] - 3s 34ms/step - loss: 0.3568 - root_mean_squared_error: 0.1916 - val_loss: 0.3842 - val_root_mean_squared_error: 0.2117\n",
      "Epoch 64/75\n",
      "76/84 [==========================>...] - ETA: 0s - loss: 0.3569 - root_mean_squared_error: 0.1917\n",
      "spearman :\n",
      "train_spearman : 0.51997 | val_spearman : 0.37015 | Learning_Rate : 3.199999991920777e-05\n",
      "\n",
      "Epoch 00064: saving model to /content/saved models/weights.-064-0.38545\n",
      "84/84 [==============================] - 3s 32ms/step - loss: 0.3567 - root_mean_squared_error: 0.1916 - val_loss: 0.3855 - val_root_mean_squared_error: 0.2124\n",
      "Epoch 65/75\n",
      "77/84 [==========================>...] - ETA: 0s - loss: 0.3565 - root_mean_squared_error: 0.1921\n",
      "spearman :\n",
      "train_spearman : 0.52237 | val_spearman : 0.37525 | Learning_Rate : 3.199999991920777e-05\n",
      "\n",
      "Epoch 00065: saving model to /content/saved models/weights.-065-0.38498\n",
      "84/84 [==============================] - 3s 33ms/step - loss: 0.3569 - root_mean_squared_error: 0.1918 - val_loss: 0.3850 - val_root_mean_squared_error: 0.2120\n",
      "Epoch 66/75\n",
      "80/84 [===========================>..] - ETA: 0s - loss: 0.3563 - root_mean_squared_error: 0.1913\n",
      "spearman :\n",
      "train_spearman : 0.52306 | val_spearman : 0.3679 | Learning_Rate : 3.199999991920777e-05\n",
      "\n",
      "Epoch 00066: saving model to /content/saved models/weights.-066-0.38595\n",
      "84/84 [==============================] - 3s 33ms/step - loss: 0.3564 - root_mean_squared_error: 0.1915 - val_loss: 0.3860 - val_root_mean_squared_error: 0.2128\n",
      "Epoch 67/75\n",
      "75/84 [=========================>....] - ETA: 0s - loss: 0.3554 - root_mean_squared_error: 0.1908\n",
      "spearman :\n",
      "train_spearman : 0.52329 | val_spearman : 0.371 | Learning_Rate : 3.199999991920777e-05\n",
      "\n",
      "Epoch 00067: saving model to /content/saved models/weights.-067-0.38578\n",
      "84/84 [==============================] - 3s 34ms/step - loss: 0.3558 - root_mean_squared_error: 0.1909 - val_loss: 0.3858 - val_root_mean_squared_error: 0.2128\n",
      "Epoch 68/75\n",
      "74/84 [=========================>....] - ETA: 0s - loss: 0.3554 - root_mean_squared_error: 0.1908\n",
      "spearman :\n",
      "train_spearman : 0.52305 | val_spearman : 0.35092 | Learning_Rate : 3.199999991920777e-05\n",
      "\n",
      "Epoch 00068: saving model to /content/saved models/weights.-068-0.38635\n",
      "84/84 [==============================] - 2s 20ms/step - loss: 0.3555 - root_mean_squared_error: 0.1907 - val_loss: 0.3864 - val_root_mean_squared_error: 0.2131\n",
      "Epoch 69/75\n",
      "75/84 [=========================>....] - ETA: 0s - loss: 0.3554 - root_mean_squared_error: 0.1906\n",
      "spearman :\n",
      "train_spearman : 0.52436 | val_spearman : 0.35629 | Learning_Rate : 3.199999991920777e-05\n",
      "\n",
      "Epoch 00069: saving model to /content/saved models/weights.-069-0.38605\n",
      "84/84 [==============================] - 5s 57ms/step - loss: 0.3553 - root_mean_squared_error: 0.1905 - val_loss: 0.3861 - val_root_mean_squared_error: 0.2130\n",
      "Epoch 70/75\n",
      "75/84 [=========================>....] - ETA: 0s - loss: 0.3556 - root_mean_squared_error: 0.1909\n",
      "spearman :\n",
      "train_spearman : 0.52692 | val_spearman : 0.37379 | Learning_Rate : 3.199999991920777e-05\n",
      "\n",
      "Epoch 00070: ReduceLROnPlateau reducing learning rate to 9.999999259090306e-06.\n",
      "\n",
      "Epoch 00070: saving model to /content/saved models/weights.-070-0.38642\n",
      "84/84 [==============================] - 2s 21ms/step - loss: 0.3553 - root_mean_squared_error: 0.1905 - val_loss: 0.3864 - val_root_mean_squared_error: 0.2132\n",
      "Epoch 71/75\n",
      "76/84 [==========================>...] - ETA: 0s - loss: 0.3545 - root_mean_squared_error: 0.1898\n",
      "spearman :\n",
      "train_spearman : 0.52887 | val_spearman : 0.36145 | Learning_Rate : 9.999999747378752e-06\n",
      "\n",
      "Epoch 00071: saving model to /content/saved models/weights.-071-0.38603\n",
      "84/84 [==============================] - 2s 21ms/step - loss: 0.3545 - root_mean_squared_error: 0.1898 - val_loss: 0.3860 - val_root_mean_squared_error: 0.2128\n",
      "Epoch 72/75\n",
      "83/84 [============================>.] - ETA: 0s - loss: 0.3545 - root_mean_squared_error: 0.1901\n",
      "spearman :\n",
      "train_spearman : 0.53019 | val_spearman : 0.35788 | Learning_Rate : 9.999999747378752e-06\n",
      "\n",
      "Epoch 00072: saving model to /content/saved models/weights.-072-0.38619\n",
      "84/84 [==============================] - 2s 28ms/step - loss: 0.3547 - root_mean_squared_error: 0.1901 - val_loss: 0.3862 - val_root_mean_squared_error: 0.2129\n",
      "Epoch 73/75\n",
      "73/84 [=========================>....] - ETA: 0s - loss: 0.3534 - root_mean_squared_error: 0.1897\n",
      "spearman :\n",
      "train_spearman : 0.53101 | val_spearman : 0.35958 | Learning_Rate : 9.999999747378752e-06\n",
      "\n",
      "Epoch 00073: saving model to /content/saved models/weights.-073-0.38586\n",
      "84/84 [==============================] - 5s 61ms/step - loss: 0.3540 - root_mean_squared_error: 0.1895 - val_loss: 0.3859 - val_root_mean_squared_error: 0.2127\n",
      "Epoch 74/75\n",
      "76/84 [==========================>...] - ETA: 0s - loss: 0.3533 - root_mean_squared_error: 0.1888\n",
      "spearman :\n",
      "train_spearman : 0.53128 | val_spearman : 0.35925 | Learning_Rate : 9.999999747378752e-06\n",
      "\n",
      "Epoch 00074: saving model to /content/saved models/weights.-074-0.38604\n",
      "84/84 [==============================] - 2s 19ms/step - loss: 0.3542 - root_mean_squared_error: 0.1896 - val_loss: 0.3860 - val_root_mean_squared_error: 0.2128\n",
      "Epoch 75/75\n",
      "84/84 [==============================] - ETA: 0s - loss: 0.3545 - root_mean_squared_error: 0.1898\n",
      "spearman :\n",
      "train_spearman : 0.53174 | val_spearman : 0.36037 | Learning_Rate : 9.999999747378752e-06\n",
      "\n",
      "Epoch 00075: saving model to /content/saved models/weights.-075-0.38648\n",
      "84/84 [==============================] - 3s 31ms/step - loss: 0.3545 - root_mean_squared_error: 0.1898 - val_loss: 0.3865 - val_root_mean_squared_error: 0.2131\n"
     ]
    }
   ],
   "source": [
    "# LSTM : training a model\n",
    "tf.keras.backend.clear_session()\n",
    "opt = tf.keras.optimizers.Adam(learning_rate = 0.0001)\n",
    "rmse = tf.keras.metrics.RootMeanSquaredError()\n",
    " \n",
    "model.compile(loss = 'binary_crossentropy', optimizer = opt,  metrics = [rmse])\n",
    "history = model.fit(x = [tb_XLNet_train, ans_XLNet_train], y =  y_train.values,\n",
    "                    validation_data = ([tb_XLNet_cv, ans_XLNet_cv], y_cv.values),\n",
    "                    batch_size = 64, epochs = 75, callbacks = callbacks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 33
    },
    "colab_type": "code",
    "id": "dMdgZOgwbaF-",
    "outputId": "a7c2790a-ec1b-4713-cc53-84cb2ccf5f59"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.37525, 64)"
      ]
     },
     "execution_count": 85,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# best_results : (val_spearman, epoch_no)\n",
    "max(print_spearman_fn.spearman_dict['val_spearman']), np.argmax(print_spearman_fn.spearman_dict['val_spearman'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "R7A1Pbs-iYD8"
   },
   "source": [
    "### # Final Results :\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 163
    },
    "colab_type": "code",
    "id": "rn2bKsqSgEuK",
    "outputId": "268bdb85-60a8-4f2e-a6a3-98d86e5d54a8"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------------+----------------+--------------+\n",
      "|     Model     | train_spearman | val_spearman |\n",
      "+---------------+----------------+--------------+\n",
      "| baseline_LSTM |     0.4995     |   0.34899    |\n",
      "|      BERT     |    0.50081     |   0.39536    |\n",
      "|      USE      |    0.52245     |    0.4366    |\n",
      "|    RoBERTa    |    0.48059     |   0.41772    |\n",
      "|     XLNet     |    0.52237     |   0.37525    |\n",
      "+---------------+----------------+--------------+\n"
     ]
    }
   ],
   "source": [
    "# Final Results\n",
    "from prettytable import PrettyTable\n",
    "x = PrettyTable()\n",
    "\n",
    "x.field_names = [\"Model\", \"train_spearman\", \"val_spearman\"]\n",
    "x.add_row([\"baseline_LSTM\", 0.4995, 0.34899])\n",
    "x.add_row([\"BERT\", 0.50081, 0.39536])\n",
    "x.add_row([\"USE\",  0.52245, 0.4366])\n",
    "x.add_row([\"RoBERTa\", 0.48059, 0.41772])\n",
    "x.add_row([\"XLNet\", 0.52237, 0.37525])\n",
    "\n",
    "print(x)"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [
    "TLEozz70PP46",
    "onEsk2ueQk_s"
   ],
   "name": "3.1. individual modeling.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "0072d96c78a04a6c94a66d66fd322f9f": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_1545fb47849e467188b7dd6455df0584",
      "max": 23,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_24c6bace57ae41a88310b32174c7162e",
      "value": 23
     }
    },
    "018420f8e8b84a6e9fe879ad9b7905f8": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_10315fdabc2544eb8c7220a4c7707482",
      "max": 23,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_c612b892af2d47fca4cfafb42df21ab0",
      "value": 23
     }
    },
    "02c40c9d125e48da9255863bcbcf1f83": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "0618f78ac6c74b35a3fe2c1fe4ca03aa": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_1f5ba81b4e7d4d959a09d8ba149d5d59",
      "max": 23,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_8d78d026d9da4ae5b456e2bbfaa3169c",
      "value": 23
     }
    },
    "0648904a662a4886a0f079a6cddb4c70": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "07820b67d0d0475486c9644a85edced2": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_f043a98303e84490b01d2fd44157ea56",
       "IPY_MODEL_c894ae32b71843e4bcd376a8f1296659"
      ],
      "layout": "IPY_MODEL_ec97992060844172a5b9a04fdfaa44c0"
     }
    },
    "082e3889c7e24cbcaf7f8d8fd40e33e5": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "09c7528baf164d2ab097ffd86a293af1": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "0c09f1af502c4211855374ceb2d9f185": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_1b337bbed0d84d34940f37ef65bec3e3",
       "IPY_MODEL_1844897a82f848c1885097865c6bc33d"
      ],
      "layout": "IPY_MODEL_7caee6dc1d87471c81a7a7eac39b78de"
     }
    },
    "0dc6097d33b74b678790542a49ecb783": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "0f5270dfa6d94e2eb03335215181b182": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Downloading: 100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_b37515616973430cae58fe29b350660b",
      "max": 798011,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_b86df2b6d0554306bc570bd8d6ec7d7e",
      "value": 798011
     }
    },
    "10315fdabc2544eb8c7220a4c7707482": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "10549e4d4ffe448badf1768aa93142ec": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "13e45ca5a45148f9b913ca06a977bf49": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "147d5e4ea5994be9a4e6df38a0c78fae": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "1545fb47849e467188b7dd6455df0584": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "1659f99e09c5446bb3b980e3841886eb": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "168b77dc7a7e440a9e105f6faa1031f8": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_cdc75a0ff6e641189c13f84ec32f5bb7",
       "IPY_MODEL_34cca5b94cbd4ec39acd6d85bcdd644d"
      ],
      "layout": "IPY_MODEL_d97737c2459a47bb95128a60efbbf91a"
     }
    },
    "16baf2495d4e426e9e0ef51369764b86": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "16dde04360234e8b9e17471341047a08": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "16ff17f3143249638e26ccce45758c7e": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_0618f78ac6c74b35a3fe2c1fe4ca03aa",
       "IPY_MODEL_492a78cbbfcb4298b2208e9b657a9a05"
      ],
      "layout": "IPY_MODEL_0648904a662a4886a0f079a6cddb4c70"
     }
    },
    "17832dfd8ffd4584a81c26a43667b95b": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_de2ea4cf8b9b4a64948669b14624e3f1",
       "IPY_MODEL_8434e38f4f6c4e6084c74d2758c3b9db"
      ],
      "layout": "IPY_MODEL_799165714cfa45e4bcb44700e6750468"
     }
    },
    "180363b7b5c04a00bb8e95c1ae645eab": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_e6ea074710014546bd2dc007e53e0865",
       "IPY_MODEL_9160b4ee7e0f42daa271dea4cb1a82a3"
      ],
      "layout": "IPY_MODEL_93fc96a9b550409986c23a0ef8ff07d7"
     }
    },
    "1844897a82f848c1885097865c6bc33d": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_3bdfc1ceea0f4b85a2c373a28517e0c9",
      "placeholder": "​",
      "style": "IPY_MODEL_fadb842104cc433fb0b7010f79cfd631",
      "value": " 23/23 [08:30&lt;00:00, 22.20s/it]"
     }
    },
    "192cdbe6af654effb073e2172b653a49": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "1b337bbed0d84d34940f37ef65bec3e3": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_d265236a86d34e898315726538d389aa",
      "max": 23,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_fdbd355e29164575b4140c9da8b28705",
      "value": 23
     }
    },
    "1d2a0a1fe1634f16a13908ce185e49f4": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_a480a9eeca7c4dd889f9839c3195637e",
       "IPY_MODEL_5dc26995ef604ce4b74ef8c83a8c228c"
      ],
      "layout": "IPY_MODEL_84a276e30bf1499e91a888cc735f3bf1"
     }
    },
    "1d626fc98e2749a5a1c11a873beb10cf": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_95cfe85ab42b46d9bffd5665fb3febd5",
      "placeholder": "​",
      "style": "IPY_MODEL_f46d8820d092457ba9c958512e713c48",
      "value": " 232k/232k [00:00&lt;00:00, 306kB/s]"
     }
    },
    "1f5ba81b4e7d4d959a09d8ba149d5d59": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "20161806a2d149f389f3ec6e43ec4f99": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Downloading: 100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_3652fd96bd044fea92701a751da9e6c4",
      "max": 760,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_5dbc3bf1b2f3414bb9c82a27ad7b23f1",
      "value": 760
     }
    },
    "225c0b674051430c98c2bc111d5c352a": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "238746b3bbf04e2298c2a355345fa1d3": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "24c6bace57ae41a88310b32174c7162e": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "24ee8f7fbdfe45c49ec5452a2bd39e89": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_bd5c006a141a4e80922ee048ae04c9f7",
       "IPY_MODEL_4e6eec0b36b64445b40b9753d51f677f"
      ],
      "layout": "IPY_MODEL_2c7cf14ab33f40d9be631dd56c7b1b20"
     }
    },
    "28c6e2e040f5413d9a569a2dbf34f45c": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "29f5625daa904c2fbf068770bacd99de": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "2bb3a1b6466144728401886f51721bdc": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "2c7cf14ab33f40d9be631dd56c7b1b20": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "301af3078ebc4f6f868b22903757b273": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_d54a6f476a344f0b89c53a9b66a11e67",
      "max": 168,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_edae9bc6832b44d1a9b74262fd264440",
      "value": 168
     }
    },
    "30343d6c72a44bf8bb2640892c3c479d": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "342b574c8e67464f8baa8fc83cc5cf56": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "34419669d655486ca2f5e1d90bd188cb": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "34cca5b94cbd4ec39acd6d85bcdd644d": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_8e6ac950fc7b44f3993cf0f5682a487e",
      "placeholder": "​",
      "style": "IPY_MODEL_af0b6f34f2bb49c18a03921e7fe57b76",
      "value": " 23/23 [01:15&lt;00:00,  3.30s/it]"
     }
    },
    "3652fd96bd044fea92701a751da9e6c4": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "3b436761ce834477aef6ea24a471384f": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_709146c17a7241459099b6f852aeab21",
      "max": 168,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_937d3d3036e940c7bfb61d0b8844932c",
      "value": 168
     }
    },
    "3bdfc1ceea0f4b85a2c373a28517e0c9": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "40ce8221d7b6421aacd1e1719fe89953": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_34419669d655486ca2f5e1d90bd188cb",
      "placeholder": "​",
      "style": "IPY_MODEL_e8f84bb9b0ba416ca36433b2ab542551",
      "value": " 168/168 [25:52&lt;00:00,  9.24s/it]"
     }
    },
    "411cd0c4ddf8444f82b94598c32bc841": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_bc9d2dab2c2849adb9063c4ac0059ef4",
      "placeholder": "​",
      "style": "IPY_MODEL_42351d43cc264ba290edbe728308d54c",
      "value": " 168/168 [09:10&lt;00:00,  3.28s/it]"
     }
    },
    "42351d43cc264ba290edbe728308d54c": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "44546631613c4ef88aca301ade538c1d": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "48edaebcf82147319f24b3cc1a32c76c": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "492a78cbbfcb4298b2208e9b657a9a05": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_5c22116ba2dd4f30921082473d55feb2",
      "placeholder": "​",
      "style": "IPY_MODEL_8120d7023386492a8a2adc64b6e2ad9e",
      "value": " 23/23 [00:48&lt;00:00,  2.09s/it]"
     }
    },
    "4cacbd4222b84b64bf2655a472c169fd": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "4d58ab0e0305477188be681484d928aa": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_f13802026eaa4a5b8e7e3097e3c5b84a",
       "IPY_MODEL_56c07f7825d843598400abf0498aac40"
      ],
      "layout": "IPY_MODEL_7107da192fb942c3a59eb40c806476c8"
     }
    },
    "4e007ce6f520428c9282c1b4d2840b88": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_20161806a2d149f389f3ec6e43ec4f99",
       "IPY_MODEL_d7c0d20837d5450f8e6e57ceedf52784"
      ],
      "layout": "IPY_MODEL_09c7528baf164d2ab097ffd86a293af1"
     }
    },
    "4e6eec0b36b64445b40b9753d51f677f": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_ba5a8b57f4994ff09454253301a194c6",
      "placeholder": "​",
      "style": "IPY_MODEL_02c40c9d125e48da9255863bcbcf1f83",
      "value": " 168/168 [04:41&lt;00:00,  1.68s/it]"
     }
    },
    "56c07f7825d843598400abf0498aac40": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_48edaebcf82147319f24b3cc1a32c76c",
      "placeholder": "​",
      "style": "IPY_MODEL_2bb3a1b6466144728401886f51721bdc",
      "value": " 168/168 [02:38&lt;00:00,  1.06it/s]"
     }
    },
    "58a453298c43429e8292d3fa9f3c4b11": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "595bd7256a014f9abc143feb67ec7851": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "5adff9692eca4311a2157cac94a522d4": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "5bb660c119794b6abc5d5b0643a2cbfc": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_701a9da19a2b419bb54e6a746cd1037c",
       "IPY_MODEL_c758513fa19148eb92d089d69e745d20"
      ],
      "layout": "IPY_MODEL_db1b195f9e60412fa54a092851b8bf14"
     }
    },
    "5c22116ba2dd4f30921082473d55feb2": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "5dbc3bf1b2f3414bb9c82a27ad7b23f1": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "5dc26995ef604ce4b74ef8c83a8c228c": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_776ff4e1e1ae48a1a42640d72a3b904d",
      "placeholder": "​",
      "style": "IPY_MODEL_a1ded462845e49639d05f8c7c89568cd",
      "value": " 536M/536M [00:12&lt;00:00, 43.7MB/s]"
     }
    },
    "5eb8457f4c8a4164827f1e2309829d23": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "611f465a910c4865b1f70b0051c99c8b": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_5adff9692eca4311a2157cac94a522d4",
      "placeholder": "​",
      "style": "IPY_MODEL_87c171348f974f25a139767a23748e04",
      "value": " 23/23 [00:37&lt;00:00,  1.61s/it]"
     }
    },
    "628d0324f7034a36b7199ce0ea2b5537": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "62a3f36d22c34284966620513a0a4248": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "62cca8a6dc4741e1a48ed4ada9b3bb06": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_c8f8ea592c654181ac1655d7d9cf0f84",
      "max": 168,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_147d5e4ea5994be9a4e6df38a0c78fae",
      "value": 168
     }
    },
    "62e7d665f2df4e6a8a9a74c5e31fe2e3": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "630d5c3d71b84179a0b60fbc6c1a7b9c": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "635ee408dfe3404c99d9b08f7b7db947": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "6771fee3e62a4a2baff8a192189ebebe": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "68c7efc4afa949e583a4fc418965902b": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "6a7f283458e44839978e11960dbe592a": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_6d363da8804848df920c9863c0b58871",
      "placeholder": "​",
      "style": "IPY_MODEL_cbc0d08cea85421c947d1b3f9179e01f",
      "value": " 12/12 [00:33&lt;00:00,  2.82s/it]"
     }
    },
    "6c81df840b5048169571d2e189ff9a5d": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "6d363da8804848df920c9863c0b58871": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "701a9da19a2b419bb54e6a746cd1037c": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Downloading: 100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_30343d6c72a44bf8bb2640892c3c479d",
      "max": 565485600,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_5eb8457f4c8a4164827f1e2309829d23",
      "value": 565485600
     }
    },
    "709146c17a7241459099b6f852aeab21": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "7107da192fb942c3a59eb40c806476c8": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "73a4fa09665d4c73b3a0055d45f898d3": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "776ff4e1e1ae48a1a42640d72a3b904d": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "799165714cfa45e4bcb44700e6750468": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "7b296c3ac88147bca4b362b152d7323c": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "7caee6dc1d87471c81a7a7eac39b78de": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "7d30fdccfc8043d0ba305ab56a40c54e": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "8120d7023386492a8a2adc64b6e2ad9e": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "81dc6c322356403f8e25657c8264b8d2": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "82ce03604a4f41a698f77ff22a758707": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "8434e38f4f6c4e6084c74d2758c3b9db": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_1659f99e09c5446bb3b980e3841886eb",
      "placeholder": "​",
      "style": "IPY_MODEL_9da9b44a2e66438b8916d24c89937b46",
      "value": " 23/23 [04:51&lt;00:00, 12.66s/it]"
     }
    },
    "846aa2b5f5ea45149f8c924f8601f4c3": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_931619a17d8d4b1f9abb071fb0a17b7a",
       "IPY_MODEL_afc8cb74078d46399b8e4e5628942ae7"
      ],
      "layout": "IPY_MODEL_7b296c3ac88147bca4b362b152d7323c"
     }
    },
    "84a276e30bf1499e91a888cc735f3bf1": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "85f6a616ab024077b71db09bc1a99c94": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_df22315806544dca9f103f6bc25d93fb",
       "IPY_MODEL_1d626fc98e2749a5a1c11a873beb10cf"
      ],
      "layout": "IPY_MODEL_16baf2495d4e426e9e0ef51369764b86"
     }
    },
    "86200091be6a4c818ba2ca9273b834f2": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_018420f8e8b84a6e9fe879ad9b7905f8",
       "IPY_MODEL_cea7e3ad706945a6a6510a294c6c7d53"
      ],
      "layout": "IPY_MODEL_81dc6c322356403f8e25657c8264b8d2"
     }
    },
    "8793c64d528f49be821e254358c2d2e3": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "87c171348f974f25a139767a23748e04": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "88bb3253383f4ac2b6fc3102b147d1ca": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "8cbba98eeb584eec9704921f7e7442c8": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_62a3f36d22c34284966620513a0a4248",
      "max": 23,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_8793c64d528f49be821e254358c2d2e3",
      "value": 23
     }
    },
    "8d6e476417c2496588818f4ab41ec649": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_0f5270dfa6d94e2eb03335215181b182",
       "IPY_MODEL_9b6821e0177f4eac9bb0ff75d7981c27"
      ],
      "layout": "IPY_MODEL_9bf02940c29f4dd0bbc0c3b3e6228070"
     }
    },
    "8d78d026d9da4ae5b456e2bbfaa3169c": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "8e51481bf39342d0ae5d69aab423a06d": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_3b436761ce834477aef6ea24a471384f",
       "IPY_MODEL_411cd0c4ddf8444f82b94598c32bc841"
      ],
      "layout": "IPY_MODEL_e0f06efb9e0d4d8bb50c8db2567f18a6"
     }
    },
    "8e6ac950fc7b44f3993cf0f5682a487e": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "9160b4ee7e0f42daa271dea4cb1a82a3": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_cecec159b882408e861c4e009b0979e3",
      "placeholder": "​",
      "style": "IPY_MODEL_62e7d665f2df4e6a8a9a74c5e31fe2e3",
      "value": " 168/168 [04:18&lt;00:00,  1.54s/it]"
     }
    },
    "9175f5a49ec74397971d1fcfaefda03a": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "931619a17d8d4b1f9abb071fb0a17b7a": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_342b574c8e67464f8baa8fc83cc5cf56",
      "max": 168,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_88bb3253383f4ac2b6fc3102b147d1ca",
      "value": 168
     }
    },
    "937d3d3036e940c7bfb61d0b8844932c": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "93fc96a9b550409986c23a0ef8ff07d7": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "95cfe85ab42b46d9bffd5665fb3febd5": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "99762cc3d82e4237b7f8b43f69c04452": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "99ab339a4402497cbb80572366c49aae": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_62cca8a6dc4741e1a48ed4ada9b3bb06",
       "IPY_MODEL_bd630f0cc09a45cbba3910b75e4c823e"
      ],
      "layout": "IPY_MODEL_eefaad3cdef94003ba02408f4784f9e0"
     }
    },
    "99c3f363e10e48fc84b7dc8f86df3d53": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "9b1a02d275ed419ea593a1a646dd982b": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_8cbba98eeb584eec9704921f7e7442c8",
       "IPY_MODEL_611f465a910c4865b1f70b0051c99c8b"
      ],
      "layout": "IPY_MODEL_68c7efc4afa949e583a4fc418965902b"
     }
    },
    "9b6821e0177f4eac9bb0ff75d7981c27": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_238746b3bbf04e2298c2a355345fa1d3",
      "placeholder": "​",
      "style": "IPY_MODEL_bfc5f9b4f5874ba5b43180b03d7320fe",
      "value": " 798k/798k [00:10&lt;00:00, 76.1kB/s]"
     }
    },
    "9bf02940c29f4dd0bbc0c3b3e6228070": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "9cdabd8c25f045ce89627bde52ab9a21": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "9da9b44a2e66438b8916d24c89937b46": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "9fc2e5520fcd4e39be45e31096842c6c": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "a1cf58380f654e5b9aadcd13e49d0d33": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "a1ded462845e49639d05f8c7c89568cd": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "a480a9eeca7c4dd889f9839c3195637e": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Downloading: 100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_ad127bb1996f462cbf154f157ea4aed3",
      "max": 536063208,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_e05140f32795468584a139c9d33ba1fd",
      "value": 536063208
     }
    },
    "a6a69668740148558c5a8f61b3fc8fc8": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "abbb139816074ebeaecce3f12f994198": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "ac44a30beb2a45c29130c7f7cad7b8ae": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_0072d96c78a04a6c94a66d66fd322f9f",
       "IPY_MODEL_ea4b6964735144b88aa7bd5b1b8c7636"
      ],
      "layout": "IPY_MODEL_a1cf58380f654e5b9aadcd13e49d0d33"
     }
    },
    "ad127bb1996f462cbf154f157ea4aed3": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "af0b6f34f2bb49c18a03921e7fe57b76": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "afc8cb74078d46399b8e4e5628942ae7": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_d8079814c7794191b31a431d7a1e940b",
      "placeholder": "​",
      "style": "IPY_MODEL_225c0b674051430c98c2bc111d5c352a",
      "value": " 168/168 [16:23&lt;00:00,  5.86s/it]"
     }
    },
    "b37515616973430cae58fe29b350660b": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "b6fa98b87b32493aa7c57db485c0b29a": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_e652ecc7474249d7bc60f2687e0a5e40",
       "IPY_MODEL_f40413e300bb4099870dbca3b5495af2"
      ],
      "layout": "IPY_MODEL_44546631613c4ef88aca301ade538c1d"
     }
    },
    "b86df2b6d0554306bc570bd8d6ec7d7e": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "ba5a8b57f4994ff09454253301a194c6": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "bc9d2dab2c2849adb9063c4ac0059ef4": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "bd5c006a141a4e80922ee048ae04c9f7": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_29f5625daa904c2fbf068770bacd99de",
      "max": 168,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_630d5c3d71b84179a0b60fbc6c1a7b9c",
      "value": 168
     }
    },
    "bd630f0cc09a45cbba3910b75e4c823e": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_635ee408dfe3404c99d9b08f7b7db947",
      "placeholder": "​",
      "style": "IPY_MODEL_082e3889c7e24cbcaf7f8d8fd40e33e5",
      "value": " 168/168 [08:32&lt;00:00,  3.05s/it]"
     }
    },
    "bfc5f9b4f5874ba5b43180b03d7320fe": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "c612b892af2d47fca4cfafb42df21ab0": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "c758513fa19148eb92d089d69e745d20": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_ea62c2f49be941e28ca5813abdfe33da",
      "placeholder": "​",
      "style": "IPY_MODEL_99c3f363e10e48fc84b7dc8f86df3d53",
      "value": " 565M/565M [00:07&lt;00:00, 75.2MB/s]"
     }
    },
    "c761637a6a6b41d8ae2451ee3e9cb249": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_301af3078ebc4f6f868b22903757b273",
       "IPY_MODEL_40ce8221d7b6421aacd1e1719fe89953"
      ],
      "layout": "IPY_MODEL_9175f5a49ec74397971d1fcfaefda03a"
     }
    },
    "c894ae32b71843e4bcd376a8f1296659": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_cc8670a704fb4bf282388423b35c23d2",
      "placeholder": "​",
      "style": "IPY_MODEL_192cdbe6af654effb073e2172b653a49",
      "value": " 433/433 [00:00&lt;00:00, 12.2kB/s]"
     }
    },
    "c8f8ea592c654181ac1655d7d9cf0f84": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "cbc0d08cea85421c947d1b3f9179e01f": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "cc8670a704fb4bf282388423b35c23d2": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "cd61a31aa67048ccb483eec1793f9cf9": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_dcadc26b893d49ea98ac4e9bba54dec9",
      "max": 12,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_6c81df840b5048169571d2e189ff9a5d",
      "value": 12
     }
    },
    "cdc75a0ff6e641189c13f84ec32f5bb7": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_73a4fa09665d4c73b3a0055d45f898d3",
      "max": 23,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_628d0324f7034a36b7199ce0ea2b5537",
      "value": 23
     }
    },
    "cea7e3ad706945a6a6510a294c6c7d53": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_28c6e2e040f5413d9a569a2dbf34f45c",
      "placeholder": "​",
      "style": "IPY_MODEL_cedb401a3ce74a3da07b1987625a33cd",
      "value": " 23/23 [18:17&lt;00:00, 47.70s/it]"
     }
    },
    "cecec159b882408e861c4e009b0979e3": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "cedb401a3ce74a3da07b1987625a33cd": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "d265236a86d34e898315726538d389aa": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "d54a6f476a344f0b89c53a9b66a11e67": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "d7c0d20837d5450f8e6e57ceedf52784": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_9cdabd8c25f045ce89627bde52ab9a21",
      "placeholder": "​",
      "style": "IPY_MODEL_4cacbd4222b84b64bf2655a472c169fd",
      "value": " 760/760 [00:01&lt;00:00, 484B/s]"
     }
    },
    "d8079814c7794191b31a431d7a1e940b": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "d83706549c114e2381d0211f346f66f8": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_cd61a31aa67048ccb483eec1793f9cf9",
       "IPY_MODEL_6a7f283458e44839978e11960dbe592a"
      ],
      "layout": "IPY_MODEL_f28025790a204cdd877493c5e712dde0"
     }
    },
    "d97737c2459a47bb95128a60efbbf91a": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "db1b195f9e60412fa54a092851b8bf14": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "dcadc26b893d49ea98ac4e9bba54dec9": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "de2ea4cf8b9b4a64948669b14624e3f1": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_e0ae7eb7d3954636bbd95fa70697a534",
      "max": 23,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_f5301ead168c4027975a39dd89244805",
      "value": 23
     }
    },
    "df22315806544dca9f103f6bc25d93fb": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Downloading: 100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_58a453298c43429e8292d3fa9f3c4b11",
      "max": 231508,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_9fc2e5520fcd4e39be45e31096842c6c",
      "value": 231508
     }
    },
    "e05140f32795468584a139c9d33ba1fd": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "e0ae7eb7d3954636bbd95fa70697a534": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "e0f06efb9e0d4d8bb50c8db2567f18a6": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "e652ecc7474249d7bc60f2687e0a5e40": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_82ce03604a4f41a698f77ff22a758707",
      "max": 84,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_16dde04360234e8b9e17471341047a08",
      "value": 84
     }
    },
    "e6ea074710014546bd2dc007e53e0865": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_abbb139816074ebeaecce3f12f994198",
      "max": 168,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_7d30fdccfc8043d0ba305ab56a40c54e",
      "value": 168
     }
    },
    "e8f84bb9b0ba416ca36433b2ab542551": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "ea4b6964735144b88aa7bd5b1b8c7636": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_6771fee3e62a4a2baff8a192189ebebe",
      "placeholder": "​",
      "style": "IPY_MODEL_10549e4d4ffe448badf1768aa93142ec",
      "value": " 23/23 [00:31&lt;00:00,  1.39s/it]"
     }
    },
    "ea62c2f49be941e28ca5813abdfe33da": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "ec97992060844172a5b9a04fdfaa44c0": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "edae9bc6832b44d1a9b74262fd264440": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "eefaad3cdef94003ba02408f4784f9e0": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "f043a98303e84490b01d2fd44157ea56": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Downloading: 100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_0dc6097d33b74b678790542a49ecb783",
      "max": 433,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_f182fc80701b4df58ccd3592546880b6",
      "value": 433
     }
    },
    "f13802026eaa4a5b8e7e3097e3c5b84a": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_a6a69668740148558c5a8f61b3fc8fc8",
      "max": 168,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_13e45ca5a45148f9b913ca06a977bf49",
      "value": 168
     }
    },
    "f182fc80701b4df58ccd3592546880b6": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "f28025790a204cdd877493c5e712dde0": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "f40413e300bb4099870dbca3b5495af2": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_595bd7256a014f9abc143feb67ec7851",
      "placeholder": "​",
      "style": "IPY_MODEL_99762cc3d82e4237b7f8b43f69c04452",
      "value": " 84/84 [13:35&lt;00:00,  9.71s/it]"
     }
    },
    "f46d8820d092457ba9c958512e713c48": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "f5301ead168c4027975a39dd89244805": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "fadb842104cc433fb0b7010f79cfd631": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "fdbd355e29164575b4140c9da8b28705": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}