4_Final_pipeline_resub.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 50
    },
    "colab_type": "code",
    "id": "XWUJbBG6iqmZ",
    "outputId": "13add462-1e47-4710-a254-25a6b9653940"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/statsmodels/tools/_testing.py:19: FutureWarning: pandas.util.testing is deprecated. Use the functions in the public API at pandas.testing instead.\n",
      "  import pandas.util.testing as tm\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from tqdm.notebook import tqdm\n",
    "import re\n",
    "import warnings\n",
    "import html\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 98
    },
    "colab_type": "code",
    "id": "Rg4Z4VFdO9AW",
    "outputId": "1da20bc2-c7dc-42f1-acea-b4d95917a7be"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Archive:  /content/drive/My Drive/AAIC Course/Personal case study 2  - Google Quest/google-quest-challenge.zip\n",
      "  inflating: sample_submission.csv   \n",
      "  inflating: test.csv                \n",
      "  inflating: train.csv               \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "((6079, 41), (476, 11))"
      ]
     },
     "execution_count": 2,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "!unzip '/content/drive/My Drive/AAIC Course/Personal case study 2  - Google Quest/google-quest-challenge.zip'\n",
    "df_train = pd.read_csv('train.csv')\n",
    "df_test = pd.read_csv('test.csv')\n",
    "df_train.shape, df_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 33
    },
    "colab_type": "code",
    "id": "Bpw2970-zK26",
    "outputId": "f7b49c82-782f-4484-de32-5b7c99a88df3"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "30"
      ]
     },
     "execution_count": 5,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 1. setting up target features\n",
    "question_tar = ['question_asker_intent_understanding',\n",
    "       'question_body_critical', 'question_conversational',\n",
    "       'question_expect_short_answer', 'question_fact_seeking',\n",
    "       'question_has_commonly_accepted_answer',\n",
    "       'question_interestingness_others', 'question_interestingness_self',\n",
    "       'question_multi_intent', 'question_not_really_a_question',\n",
    "       'question_opinion_seeking', 'question_type_choice',\n",
    "       'question_type_compare', 'question_type_consequence',\n",
    "       'question_type_definition', 'question_type_entity',\n",
    "       'question_type_instructions', 'question_type_procedure',\n",
    "       'question_type_reason_explanation', 'question_type_spelling',\n",
    "       'question_well_written']\n",
    "       \n",
    "answer_tar = ['answer_helpful',\n",
    "       'answer_level_of_information', 'answer_plausible', 'answer_relevance',\n",
    "       'answer_satisfaction', 'answer_type_instructions',\n",
    "       'answer_type_procedure', 'answer_type_reason_explanation',\n",
    "       'answer_well_written']\n",
    " \n",
    "tar_features = question_tar + answer_tar\n",
    "len(tar_features)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Gw-x1fgmvJH5"
   },
   "source": [
    "# 4.1 BERT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 570
    },
    "colab_type": "code",
    "id": "ngp0RwgcyVoP",
    "outputId": "74c8da7e-ecee-422e-a0c4-3e845ff22c82"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting transformers\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/27/3c/91ed8f5c4e7ef3227b4119200fc0ed4b4fd965b1f0172021c25701087825/transformers-3.0.2-py3-none-any.whl (769kB)\n",
      "\r",
      "\u001b[K     |▍                               | 10kB 26.8MB/s eta 0:00:01\r",
      "\u001b[K     |▉                               | 20kB 5.8MB/s eta 0:00:01\r",
      "\u001b[K     |█▎                              | 30kB 7.3MB/s eta 0:00:01\r",
      "\u001b[K     |█▊                              | 40kB 8.2MB/s eta 0:00:01\r",
      "\u001b[K     |██▏                             | 51kB 6.9MB/s eta 0:00:01\r",
      "\u001b[K     |██▋                             | 61kB 7.8MB/s eta 0:00:01\r",
      "\u001b[K     |███                             | 71kB 7.8MB/s eta 0:00:01\r",
      "\u001b[K     |███▍                            | 81kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |███▉                            | 92kB 9.1MB/s eta 0:00:01\r",
      "\u001b[K     |████▎                           | 102kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |████▊                           | 112kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |█████▏                          | 122kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |█████▌                          | 133kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |██████                          | 143kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |██████▍                         | 153kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |██████▉                         | 163kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |███████▎                        | 174kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |███████▊                        | 184kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |████████                        | 194kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |████████▌                       | 204kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |█████████                       | 215kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |█████████▍                      | 225kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |█████████▉                      | 235kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |██████████▎                     | 245kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |██████████▋                     | 256kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |███████████                     | 266kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |███████████▌                    | 276kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |████████████                    | 286kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |████████████▍                   | 296kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |████████████▉                   | 307kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████▏                  | 317kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████▋                  | 327kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████                  | 337kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████▌                 | 348kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████                 | 358kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████▍                | 368kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████▊                | 378kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████▏               | 389kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████▋               | 399kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████               | 409kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████▌              | 419kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████              | 430kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████▎             | 440kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████▊             | 450kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████▏            | 460kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████▋            | 471kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████            | 481kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████▌           | 491kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████▉           | 501kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████▎          | 512kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████▊          | 522kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████▏         | 532kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████▋         | 542kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████         | 552kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████▍        | 563kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████▉        | 573kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████▎       | 583kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████▊       | 593kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████████▏      | 604kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████████▋      | 614kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████████      | 624kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████████▍     | 634kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████████▉     | 645kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████████▎    | 655kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████████▊    | 665kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████████▏   | 675kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████████▌   | 686kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████████████   | 696kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████████████▍  | 706kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████████████▉  | 716kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████████████▎ | 727kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████████████▊ | 737kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████████████ | 747kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████████████▌| 757kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████████████| 768kB 8.7MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████████████| 778kB 8.7MB/s \n",
      "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers) (3.0.12)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (2019.12.20)\n",
      "Collecting sacremoses\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n",
      "\r",
      "\u001b[K     |▍                               | 10kB 28.3MB/s eta 0:00:01\r",
      "\u001b[K     |▊                               | 20kB 36.9MB/s eta 0:00:01\r",
      "\u001b[K     |█▏                              | 30kB 44.5MB/s eta 0:00:01\r",
      "\u001b[K     |█▌                              | 40kB 48.1MB/s eta 0:00:01\r",
      "\u001b[K     |█▉                              | 51kB 49.1MB/s eta 0:00:01\r",
      "\u001b[K     |██▎                             | 61kB 51.7MB/s eta 0:00:01\r",
      "\u001b[K     |██▋                             | 71kB 51.9MB/s eta 0:00:01\r",
      "\u001b[K     |███                             | 81kB 53.3MB/s eta 0:00:01\r",
      "\u001b[K     |███▍                            | 92kB 54.8MB/s eta 0:00:01\r",
      "\u001b[K     |███▊                            | 102kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |████                            | 112kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |████▌                           | 122kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |████▉                           | 133kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |█████▏                          | 143kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |█████▋                          | 153kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |██████                          | 163kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |██████▎                         | 174kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |██████▊                         | 184kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |███████                         | 194kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |███████▍                        | 204kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |███████▉                        | 215kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |████████▏                       | 225kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |████████▌                       | 235kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |█████████                       | 245kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |█████████▎                      | 256kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |█████████▋                      | 266kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |██████████                      | 276kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |██████████▍                     | 286kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |██████████▊                     | 296kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |███████████▏                    | 307kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |███████████▌                    | 317kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |███████████▉                    | 327kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |████████████▎                   | 337kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |████████████▋                   | 348kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████                   | 358kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████▍                  | 368kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████▊                  | 378kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████                  | 389kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████▌                 | 399kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████▉                 | 409kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████▏                | 419kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████▋                | 430kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████                | 440kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████▎               | 450kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████▊               | 460kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████               | 471kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████▍              | 481kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████▉              | 491kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████▏             | 501kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████▌             | 512kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████             | 522kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████▎            | 532kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████▋            | 542kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████            | 552kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████▍           | 563kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████▊           | 573kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████▏          | 583kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████▌          | 593kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████▉          | 604kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████▎         | 614kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████▋         | 624kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████         | 634kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████▍        | 645kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████▊        | 655kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████        | 665kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████▌       | 675kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████▉       | 686kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████████▏      | 696kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████████▋      | 706kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████████      | 716kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████████▎     | 727kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████████▊     | 737kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████████     | 747kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████████▍    | 757kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████████▉    | 768kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████████▏   | 778kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████████▌   | 788kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████████████   | 798kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████████████▎  | 808kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████████████▋  | 819kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████████████  | 829kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████████████▍ | 839kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████████████▊ | 849kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████████████▏| 860kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████████████▌| 870kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████████████▉| 880kB 49.3MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████████████| 890kB 49.3MB/s \n",
      "\u001b[?25hRequirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers) (0.7)\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers) (1.18.5)\n",
      "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers) (4.41.1)\n",
      "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers) (2.23.0)\n",
      "Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from transformers) (20.4)\n",
      "Collecting tokenizers==0.8.1.rc1\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/40/d0/30d5f8d221a0ed981a186c8eb986ce1c94e3a6e87f994eae9f4aa5250217/tokenizers-0.8.1rc1-cp36-cp36m-manylinux1_x86_64.whl (3.0MB)\n",
      "\u001b[K     |████████████████████████████████| 3.0MB 50.2MB/s \n",
      "\u001b[?25hCollecting sentencepiece!=0.1.92\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d4/a4/d0a884c4300004a78cca907a6ff9a5e9fe4f090f5d95ab341c53d28cbc58/sentencepiece-0.1.91-cp36-cp36m-manylinux1_x86_64.whl (1.1MB)\n",
      "\u001b[K     |████████████████████████████████| 1.1MB 55.6MB/s \n",
      "\u001b[?25hRequirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (1.15.0)\n",
      "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (7.1.2)\n",
      "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (0.16.0)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2020.6.20)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (1.24.3)\n",
      "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2.10)\n",
      "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (3.0.4)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->transformers) (2.4.7)\n",
      "Building wheels for collected packages: sacremoses\n",
      "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for sacremoses: filename=sacremoses-0.0.43-cp36-none-any.whl size=893260 sha256=88ee00994acbababe20e39c3c2e63e6656373bee44b7b58510bff6f3e7308284\n",
      "  Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n",
      "Successfully built sacremoses\n",
      "Installing collected packages: sacremoses, tokenizers, sentencepiece, transformers\n",
      "Successfully installed sacremoses-0.0.43 sentencepiece-0.1.91 tokenizers-0.8.1rc1 transformers-3.0.2\n"
     ]
    }
   ],
   "source": [
    "!pip install transformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 262,
     "referenced_widgets": [
      "f7623078e00d4b3faeadbd70a873deb8",
      "0bdd00e54a1b49ef8566f8aba098b318",
      "1ccd161c379343eeb71a2dcb6f677255",
      "59ffe7f9c15b46a4a7a614abf225f132",
      "b9a064c9aa2a46f1861153799921f4b6",
      "d6f97127143c47169a4666964eef86da",
      "4cd55bb65ee94b6680349b9ffbc11aea",
      "3866d379246644b9927ecb425a88366a",
      "f32bf4db71c04005845441e04bc6f571",
      "bbd2a1699be94416b249a4ca99b07c67",
      "d46b71aa651d4cce8a5c380ba97c3763",
      "a4be9ead558e4c2db6e08a7adc591e72",
      "14cb55958b3b4d3c8cfccd070251b077",
      "6b2b9b02b14a44f0ab526c72976e68dd",
      "bab76f5d068c4e35999b74bcbe0200db",
      "44fc91ba6a98493d864b7b85fb569ac5",
      "6044c3933b514544bc8275968abfa763",
      "56cbda827835433b8ffae3a80afc1d81",
      "d7dbe389d90a4143beb453df5d14841e",
      "03c50f80c04b4167b1ad96b9380ffc0d",
      "7b3d671d13f344769522e3f83fbe2714",
      "88bef24e4b1d45acb843a0f9f6dfd1b1",
      "9f1685bb55614aa597dca385a8e69842",
      "20a2c5f7ca794559837b78e1af80d8f0"
     ]
    },
    "colab_type": "code",
    "id": "Hku0m_5FvEvf",
    "outputId": "b920e3e1-17d5-4862-e41f-6c3e42aabbae"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f7623078e00d4b3faeadbd70a873deb8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=433.0, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f32bf4db71c04005845441e04bc6f571",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=231508.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6044c3933b514544bc8275968abfa763",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=536063208.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing TFBertModel: ['mlm___cls', 'nsp___cls']\n",
      "- This IS expected if you are initializing TFBertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPretraining model).\n",
      "- This IS NOT expected if you are initializing TFBertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "All the weights of TFBertModel were initialized from the model checkpoint at bert-base-uncased.\n",
      "If your task is similar to the task the model of the ckeckpoint was trained on, you can already use TFBertModel for predictions without further training.\n"
     ]
    }
   ],
   "source": [
    "# 1. defining model and tokenizer\n",
    "import tensorflow as tf\n",
    "from transformers import BertTokenizer, TFBertModel, BertConfig\n",
    " \n",
    "# Load pretrained model/tokenizer\n",
    "config = BertConfig.from_pretrained('bert-base-uncased', output_hidden_states=True)\n",
    "bert_tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\n",
    "bert_model = TFBertModel.from_pretrained('bert-base-uncased', config = config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "640pwAQ7vPDn"
   },
   "outputs": [],
   "source": [
    "# 2. tokenizer\n",
    "title_body_train_tokens = bert_tokenizer.batch_encode_plus(title_body_train, max_length = 512, truncation = True, pad_to_max_length = True, return_tensors=\"tf\")\n",
    "ans_train_tokens = bert_tokenizer.batch_encode_plus(answer_train, max_length = 512, truncation = True, pad_to_max_length = True, return_tensors=\"tf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "0tDq-Ss5vXy9"
   },
   "outputs": [],
   "source": [
    "# 3. bert model prediction\n",
    "x = bert_model([ans_train_input_ids[i*batch_size:(i+1)*batch_size], ans_train_attn_mask[i*batch_size:(i+1)*batch_size], ans_train_token_typ_ids[i*batch_size:(i+1)*batch_size]])[2]\n",
    "lst = [i[:, 0, :] for i in x[-4:]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "zgSO3OsO8J0Q"
   },
   "source": [
    "# 4.2. USE Embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "yhfrVy3JBjH1"
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import tensorflow_hub as hub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 82
    },
    "colab_type": "code",
    "id": "n-aQpp3S8MAZ",
    "outputId": "9aed9fcd-0bc5-4597-a8bd-147049440eeb"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:absl:Using /tmp/tfhub_modules to cache modules.\n",
      "INFO:absl:Downloading TF-Hub Module 'https://tfhub.dev/google/universal-sentence-encoder-large/5'.\n",
      "INFO:absl:Downloaded https://tfhub.dev/google/universal-sentence-encoder-large/5, Total size: 577.10MB\n",
      "INFO:absl:Downloaded TF-Hub Module 'https://tfhub.dev/google/universal-sentence-encoder-large/5'.\n"
     ]
    }
   ],
   "source": [
    "# loading USE model\n",
    "use_model = hub.load(\"https://tfhub.dev/google/universal-sentence-encoder-large/5\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "nC7qxGeu9BLT"
   },
   "source": [
    "# 4.3. Sentence Embedding : RoBERTa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 311,
     "referenced_widgets": [
      "2d2ce31cf3c44e0fbe0dcacb589e3912",
      "7b3de90cce9142308bc357f0f052929e",
      "07e7b019663e487287644406bc7a1612",
      "f16ca228750142c19710657f938c4cfe",
      "b088f7eb2c384213944b38c48659c89f",
      "8542f2bf25a640a287c9f76f66e2ac60",
      "6357a43b81544e6c9299fc2b30b3ed33",
      "3551a411cc0b4466a9760a9de4764813",
      "7fe7f7454c204fb5bb3d20b8208b652c",
      "40de2f8b36b04a5481a8eb43d13e921a",
      "ed1fa678699845e68b937709287a2009",
      "ea3489f39bc14d4ba8574d187133fd8e",
      "782a4d1e297445b6a1a5b005fd1d2250",
      "c1f0e1271ec54914bfacb3d61047bc5e",
      "ae529881693746beb77ee846797c75b9",
      "1d9e3d03386a47d8b65b5bb8d35b6447",
      "d479f4414b6247fa9c19d4a3807f9440",
      "ec28300e36c24a07991f6a94c1bbd7d1",
      "1dc104cf8ed845a1840b051583c920fb",
      "0609d8ecd7f44bddbc18120bacabe9c4",
      "ff608fc9cf5f43d7bfd371fc270e7810",
      "e91d5f782791409aa47e35fe63e43d08",
      "223be068a8b34fe08cdee9e78eaf8896",
      "996b984802d94a76850ffaa7a10e0faa",
      "051c94c779a64e5593a491e0899966fc",
      "0fc05e4422bf4a1a8f381c25c5023ba9",
      "dffeef9372084b3288fcc53c7efa4325",
      "ebe34cb546554d498b8b70d5453ce9fe",
      "a4a7789bb5b84d1e92e123a3249bdc1c",
      "9b604969ed95490aaf078123c72a3a86",
      "1acf9d0df80f450081e9f08a6f118717",
      "6ad534f7296740e7a03b888ef8991190"
     ]
    },
    "colab_type": "code",
    "id": "iQ9PYY1A9BL4",
    "outputId": "13008095-2b9b-433a-df2a-22c5ebf790d0"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2d2ce31cf3c44e0fbe0dcacb589e3912",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=481.0, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7fe7f7454c204fb5bb3d20b8208b652c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=898823.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d479f4414b6247fa9c19d4a3807f9440",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=456318.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "051c94c779a64e5593a491e0899966fc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=657434796.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:transformers.modeling_tf_utils:Some weights of the model checkpoint at roberta-base were not used when initializing TFRobertaModel: ['lm_head']\n",
      "- This IS expected if you are initializing TFRobertaModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPretraining model).\n",
      "- This IS NOT expected if you are initializing TFRobertaModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "WARNING:transformers.modeling_tf_utils:All the weights of TFRobertaModel were initialized from the model checkpoint at roberta-base.\n",
      "If your task is similar to the task the model of the ckeckpoint was trained on, you can already use TFRobertaModel for predictions without further training.\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "from transformers import RobertaConfig, RobertaTokenizer, TFRobertaModel\n",
    " \n",
    "# Load pretrained model/tokenizer\n",
    "config = RobertaConfig.from_pretrained('roberta-base', output_hidden_states=True)\n",
    "roberta_tokenizer = RobertaTokenizer.from_pretrained('roberta-base')\n",
    "roberta_model = TFRobertaModel.from_pretrained('roberta-base', config = config)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "-G6cTx9aNzR1"
   },
   "source": [
    "# 4.4. Sentence Embedding : XLNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 262,
     "referenced_widgets": [
      "5294efcd04824c0698933bec33acbfd8",
      "4c8abcd74b7d409fa876d902e5ab15a0",
      "03edebaef113444f844ee57fed52a920",
      "4004f8c0687941c9a994af47e2c804de",
      "ee134f48faec4047917667634c6336c4",
      "448c0da207c343509f49bfb43d9f10ed",
      "67cfe01966a747e29a2eab02546d97f7",
      "aa12f08ae2ef4f519a312554014689f8",
      "f24e7848bf554382866eb823398bcd9c",
      "8a139a6f91274578be7958a853e0d398",
      "4ac1341d43f043c1a8886b91d1b848a1",
      "50773aa185de4beaa026617fa15894f2",
      "ff46e71a4f154d6db85235fe23f8d264",
      "8bf63b85a8294aae94218cddecb9843e",
      "14c30e31004f4cd98d4d748f903a122f",
      "a9bfb4eff2684374acba4a3d8f5f15da",
      "c70b86ec72e74665adb01e3f093aa3a6",
      "7016449cb8f84983b2aa0e370d667afc",
      "f3a2574ad79b41c288ffd3cbcddf16fe",
      "88ffe4c3fcf14a62bd48d34dd4494845",
      "7a0651fa60cf43249a36a55919679d76",
      "635c1242fc3e49a68783d67484ceac42",
      "2faa97bdfdb84f3cb5175058f4a20df4",
      "7543537519e949e79ef5f264a4851c39"
     ]
    },
    "colab_type": "code",
    "id": "csjpdrBoNzSI",
    "outputId": "0ef62a3c-3193-403e-f8dc-d5a1ccd85b9c"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5294efcd04824c0698933bec33acbfd8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=760.0, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f24e7848bf554382866eb823398bcd9c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=798011.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c70b86ec72e74665adb01e3f093aa3a6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=565485600.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:transformers.modeling_tf_utils:Some weights of the model checkpoint at xlnet-base-cased were not used when initializing TFXLNetModel: ['lm_loss']\n",
      "- This IS expected if you are initializing TFXLNetModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPretraining model).\n",
      "- This IS NOT expected if you are initializing TFXLNetModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "WARNING:transformers.modeling_tf_utils:All the weights of TFXLNetModel were initialized from the model checkpoint at xlnet-base-cased.\n",
      "If your task is similar to the task the model of the ckeckpoint was trained on, you can already use TFXLNetModel for predictions without further training.\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "from transformers import XLNetConfig, XLNetTokenizer, TFXLNetModel\n",
    " \n",
    "# Load pretrained model/tokenizer\n",
    "config = XLNetConfig.from_pretrained('xlnet-base-cased', output_hidden_states=True)\n",
    "xlnet_tokenizer = XLNetTokenizer.from_pretrained('xlnet-base-cased')\n",
    "xlnet_model = TFXLNetModel.from_pretrained('xlnet-base-cased', config = config)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "o4__knI8ltqP"
   },
   "source": [
    "# 5. Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "86mCipknlxDb"
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "tf.compat.v1.enable_eager_execution()\n",
    "from tensorflow.keras.layers import Input, Dense, Dropout, Concatenate, BatchNormalization\n",
    "from tensorflow.keras.models import Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "tgRsF52ymu9h"
   },
   "outputs": [],
   "source": [
    "class block(tf.keras.layers.Layer):\n",
    "  def __init__(self, d1_rate = 0.2, dense_1_units = 1024, d2_rate = 0.2, dense_2_units = 512, d3_rate = 0.1, seed = 42):\n",
    "    super().__init__()\n",
    "    self.d1_rate = d1_rate\n",
    "    self.dense_1_units = dense_1_units\n",
    "    self.d2_rate = d2_rate\n",
    "    self.dense_2_units = dense_2_units\n",
    "    self.d3_rate = d3_rate\n",
    "    self.seed = seed\n",
    "\n",
    "  def build(self, input_shape):\n",
    "    # self.inp = Input(name = 'ans_bert_out', shape = (768*4,), dtype = 'float32')\n",
    "    self.dropout_1 =  Dropout(rate =self.d1_rate, seed = self.seed)\n",
    "\n",
    "    self.dense_1 = Dense(units = self.dense_1_units, activation = 'relu', kernel_initializer = tf.keras.initializers.he_normal(seed = self.seed))\n",
    "    self.dropout_2 = Dropout(rate = self.d2_rate, seed = self.seed)\n",
    "\n",
    "    self.dense_2 = Dense(units = self.dense_2_units, activation = 'relu', kernel_initializer = tf.keras.initializers.he_normal(seed = self.seed))\n",
    "    self.dropout_3 = Dropout(rate = self.d3_rate, seed = self.seed)\n",
    "\n",
    "  def call(self, X):\n",
    "    dropout_1 = self.dropout_1(X)\n",
    "\n",
    "    dense_1 = self.dense_1(dropout_1)\n",
    "    dropout_2 = self.dropout_2(dense_1)\n",
    "\n",
    "    dense_2 = self.dense_2(dropout_2)\n",
    "    dropout_3 = self.dropout_3(dense_2)\n",
    "    return dropout_3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "5WTGGroGtKgh",
    "outputId": "82bab982-b933-4350-e78b-cc801774a415"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"functional_1\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "tb_use_out (InputLayer)         [(None, 512)]        0                                            \n",
      "__________________________________________________________________________________________________\n",
      "tb_roberta_out (InputLayer)     [(None, 3072)]       0                                            \n",
      "__________________________________________________________________________________________________\n",
      "block_1 (block)                 (None, 512)          1050112     tb_use_out[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "block_2 (block)                 (None, 512)          3671552     tb_roberta_out[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 21)           10773       block_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 21)           10773       block_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "ans_use_out (InputLayer)        [(None, 512)]        0                                            \n",
      "__________________________________________________________________________________________________\n",
      "ans_roberta_out (InputLayer)    [(None, 3072)]       0                                            \n",
      "__________________________________________________________________________________________________\n",
      "tb_bert_out (InputLayer)        [(None, 3072)]       0                                            \n",
      "__________________________________________________________________________________________________\n",
      "tb_xlnet_out (InputLayer)       [(None, 3072)]       0                                            \n",
      "__________________________________________________________________________________________________\n",
      "ans_bert_out (InputLayer)       [(None, 3072)]       0                                            \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_concat (TensorFlowO [(None, 533)]        0           dense_1[0][0]                    \n",
      "                                                                 ans_use_out[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_concat_1 (TensorFlo [(None, 3093)]       0           dense_2[0][0]                    \n",
      "                                                                 ans_roberta_out[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "ans_xlnet_out (InputLayer)      [(None, 3072)]       0                                            \n",
      "__________________________________________________________________________________________________\n",
      "block (block)                   (None, 512)          3671552     tb_bert_out[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "block_3 (block)                 (None, 512)          3671552     tb_xlnet_out[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "block_4 (block)                 (None, 512)          7342592     ans_bert_out[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "block_5 (block)                 (None, 512)          1071616     tf_op_layer_concat[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_6 (block)                 (None, 512)          3693056     tf_op_layer_concat_1[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_7 (block)                 (None, 512)          3671552     ans_xlnet_out[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dense (Dense)                   (None, 21)           10773       block[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "dense_3 (Dense)                 (None, 21)           10773       block_3[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_4 (Dense)                 (None, 9)            4617        block_4[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_5 (Dense)                 (None, 9)            4617        block_5[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_6 (Dense)                 (None, 9)            4617        block_6[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_7 (Dense)                 (None, 9)            4617        block_7[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Mean/input (TensorF [(4, None, 21)]      0           dense[0][0]                      \n",
      "                                                                 dense_1[0][0]                    \n",
      "                                                                 dense_2[0][0]                    \n",
      "                                                                 dense_3[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Mean_1/input (Tenso [(4, None, 9)]       0           dense_4[0][0]                    \n",
      "                                                                 dense_5[0][0]                    \n",
      "                                                                 dense_6[0][0]                    \n",
      "                                                                 dense_7[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Mean (TensorFlowOpL [(None, 21)]         0           tf_op_layer_Mean/input[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Mean_1 (TensorFlowO [(None, 9)]          0           tf_op_layer_Mean_1/input[0][0]   \n",
      "__________________________________________________________________________________________________\n",
      "concatenate (Concatenate)       (None, 30)           0           tf_op_layer_Mean[0][0]           \n",
      "                                                                 tf_op_layer_Mean_1[0][0]         \n",
      "==================================================================================================\n",
      "Total params: 27,905,144\n",
      "Trainable params: 27,905,144\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "tf.keras.backend.clear_session()\n",
    "seed = 42\n",
    "\n",
    "# *-----------------title_body-----------------*\n",
    "tb_bert_input = Input(name = 'tb_bert_out', shape = (768*4,), dtype = 'float32')\n",
    "tb_bert_block = block(d1_rate = 0.2, dense_1_units = 1024, d2_rate = 0.2, dense_2_units = 512, d3_rate = 0.1, seed = 42)(tb_bert_input)\n",
    "tb_bert_out = Dense(units = 21, activation = 'sigmoid', kernel_initializer = tf.keras.initializers.he_normal(seed = seed))(tb_bert_block)\n",
    "\n",
    "tb_use_input = Input(name = 'tb_use_out', shape = (512,), dtype = 'float32')\n",
    "tb_use_block = block(d1_rate = 0.2, dense_1_units = 1024, d2_rate = 0.2, dense_2_units = 512, d3_rate = 0.2, seed = 42)(tb_use_input)\n",
    "tb_use_out = Dense(units = 21, activation = 'sigmoid', kernel_initializer = tf.keras.initializers.he_normal(seed = seed))(tb_use_block)\n",
    "\n",
    "tb_roberta_input = Input(name = 'tb_roberta_out', shape = (768*4,), dtype = 'float32')\n",
    "tb_roberta_block = block(d1_rate = 0.2, dense_1_units = 1024, d2_rate = 0.2, dense_2_units = 512, d3_rate = 0.1, seed = 42)(tb_roberta_input)\n",
    "tb_roberta_out = Dense(units = 21, activation = 'sigmoid', kernel_initializer = tf.keras.initializers.he_normal(seed = seed))(tb_roberta_block)\n",
    "\n",
    "tb_xlnet_input = Input(name = 'tb_xlnet_out', shape = (768*4,), dtype = 'float32')\n",
    "tb_xlnet_block = block(d1_rate = 0.2, dense_1_units = 1024, d2_rate = 0.2, dense_2_units = 512, d3_rate = 0.1, seed = 42)(tb_xlnet_input)\n",
    "tb_xlnet_out = Dense(units = 21, activation = 'sigmoid', kernel_initializer = tf.keras.initializers.he_normal(seed = seed))(tb_xlnet_block)\n",
    "\n",
    "tb_out = tf.reduce_mean([tb_bert_out ,tb_use_out ,tb_roberta_out ,tb_xlnet_out], axis = 0)\n",
    "\n",
    "# *----------------- answer -----------------*\n",
    "ans_bert_input = Input(name = 'ans_bert_out', shape = (768*4,), dtype = 'float32')\n",
    "ans_bert_block = block(d1_rate = 0.2, dense_1_units = 2048, d2_rate = 0.2, dense_2_units = 512, d3_rate = 0.1, seed = 42)(ans_bert_input)\n",
    "ans_bert_out = Dense(units = 9, activation = 'sigmoid', kernel_initializer = tf.keras.initializers.he_normal(seed = seed))(ans_bert_block)\n",
    "\n",
    "ans_use_input = Input(name = 'ans_use_out', shape = (512,), dtype = 'float32')\n",
    "ans_use_concat = tf.concat([tb_use_out, ans_use_input], axis = -1) \n",
    "ans_use_block = block(d1_rate = 0.2, dense_1_units = 1024, d2_rate = 0.2, dense_2_units = 512, d3_rate = 0.2, seed = 42)(ans_use_concat)\n",
    "ans_use_out = Dense(units = 9, activation = 'sigmoid', kernel_initializer = tf.keras.initializers.he_normal(seed = seed))(ans_use_block)\n",
    "\n",
    "ans_roberta_input = Input(name = 'ans_roberta_out', shape = (768*4,), dtype = 'float32')\n",
    "ans_roberta_concat = tf.concat([tb_roberta_out, ans_roberta_input], axis = -1) \n",
    "ans_roberta_block = block(d1_rate = 0.2, dense_1_units = 1024, d2_rate = 0.2, dense_2_units = 512, d3_rate = 0.1, seed = 42)(ans_roberta_concat)\n",
    "ans_roberta_out = Dense(units = 9, activation = 'sigmoid', kernel_initializer = tf.keras.initializers.he_normal(seed = seed))(ans_roberta_block)\n",
    "\n",
    "ans_xlnet_input = Input(name = 'ans_xlnet_out', shape = (768*4,), dtype = 'float32')\n",
    "ans_xlnet_block = block(d1_rate = 0.2, dense_1_units = 1024, d2_rate = 0.2, dense_2_units = 512, d3_rate = 0.1,  seed = 42)(ans_xlnet_input)\n",
    "ans_xlnet_out = Dense(units = 9, activation = 'sigmoid', kernel_initializer = tf.keras.initializers.he_normal(seed = seed))(ans_xlnet_block)\n",
    "\n",
    "ans_out = tf.reduce_mean([ans_bert_out ,ans_use_out ,ans_roberta_out ,ans_xlnet_out], axis = 0)\n",
    "\n",
    "# *----------------- concat -----------------*\n",
    "out =  Concatenate(axis = -1)([tb_out, ans_out])\n",
    "\n",
    "model = Model(inputs = [tb_bert_input, tb_use_input, tb_roberta_input, tb_xlnet_input, ans_bert_input, ans_use_input, ans_roberta_input, ans_xlnet_input], outputs = out)\n",
    "model.load_weights('/content/drive/My Drive/AAIC Course/Personal case study 2  - Google Quest/weights.-029-0.37055')\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "n2x0Chu0w9pG"
   },
   "outputs": [],
   "source": [
    "# post processing function : binning\n",
    "def return_bins(arr):\n",
    "  val = np.unique(arr)\n",
    "  bins = []\n",
    "  for i in range(len(val)):\n",
    "    if i > 0:\n",
    "      bins.append((val[i-1] + val[i])/2)\n",
    "  return bins\n",
    "\n",
    "unique_val_30 = [np.unique(df_train[tar_features].values[:, i]) for i in range(30)]\n",
    "bins_30 = [return_bins(df_train[tar_features].values[:, i]) for i in range(30)]\n",
    "\n",
    "def binned_out(y_pred):\n",
    "  col = y_pred.shape[1]\n",
    "  final_pred = np.zeros(y_pred.shape)\n",
    "  for i in range(col):\n",
    "    idx = np.digitize(y_pred[:, i], bins_30[i])\n",
    "    final_pred[:, i] = unique_val_30[i][idx]\n",
    "  return final_pred"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "p0AddxjezYe6"
   },
   "source": [
    "# 6. Final_pipeline:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 50
    },
    "colab_type": "code",
    "id": "gtdHWtaOGBPH",
    "outputId": "b6f5329b-4df7-442e-9045-f43259db3b9b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "# defining a function to remove stop_words\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "nltk.download('stopwords')\n",
    "stop_words = set(stopwords.words('english'))\n",
    "stop_words.remove('no'); stop_words.remove('not'); stop_words.remove('nor')\n",
    " \n",
    "def stopwrd_removal(sent):\n",
    "  lst = []\n",
    "  for wrd in sent.split():\n",
    "    if wrd not in stop_words:\n",
    "      lst.append(wrd)\n",
    "  return \" \".join(lst)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "nLLiIWXu2iwo"
   },
   "outputs": [],
   "source": [
    "# 1. input text preprocessor\n",
    "def text_preprocessor(corpus, stop_word = False, remove_specialchar = False):\n",
    "  clean_corpus = []\n",
    "  for doc in corpus:\n",
    "    # 1. remove html tags, html urls, replace html comparison operators\n",
    "    clean_str = re.sub('<.*?>', '', doc)\n",
    "    clean_str = clean_str.replace('&lt;', '<')\\\n",
    "                .replace('&gt;', '>')\\\n",
    "                .replace('&le;', '<=' )\\\n",
    "                .replace('&ge;', '>=')\n",
    "\n",
    "    # 2. remove latex i,e., mostly formulas since it's mathematics based dataset\n",
    "    clean_str = re.sub('\\$.*?\\$', '', clean_str)\n",
    "\n",
    "    # 3. all lowercase \n",
    "    clean_str = clean_str.lower()\n",
    "\n",
    "    # 4. decontractions\n",
    "    clean_str = clean_str.replace(\"won't\", \"will not\").replace(\"can\\'t\", \"can not\").replace(\"n\\'t\", \" not\").replace(\"\\'re\", \" are\").\\\n",
    "                                                  replace(\"\\'s\", \" is\").replace(\"\\'d\", \" would\").replace(\"\\'ll\", \" will\").\\\n",
    "                                                  replace(\"\\'t\", \" not\").replace(\"\\'ve\", \" have\").replace(\"\\'m\", \" am\")\n",
    "\n",
    "    # 5. removing non-english or hebrew characters\n",
    "    clean_str = clean_str.encode(\"ascii\", \"ignore\").decode()\n",
    "\n",
    "    # 6. remove all special-characters other than alpha-numericals\n",
    "    if remove_specialchar == True:\n",
    "      clean_str = re.sub('[^A-Za-z0-9]+', ' ', clean_str)\n",
    "\n",
    "    # 7. Stop_word removal\n",
    "    if stop_word == True:\n",
    "      clean_str = stopwrd_removal(clean_str)\n",
    "\n",
    "    # 8. remove all white-space i.e., \\n, \\t, and extra_spaces\n",
    "    clean_str = re.sub('  +', ' ', clean_str)\n",
    "    clean_str = clean_str.replace(\"\\n\", \" \").replace(\"\\t\", \" \").strip()\n",
    "\n",
    "    clean_corpus.append(clean_str)\n",
    "  return clean_corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "CxOd9Tilzbbd"
   },
   "outputs": [],
   "source": [
    "# 2. get all sentence embeddings \n",
    "def get_embeddings(title_body_pr, answer_pr):\n",
    "  # 2.1 BERT_embeddings\n",
    "  title_body_BERT_tokens = bert_tokenizer.batch_encode_plus(title_body_pr, max_length = 512, truncation = True, pad_to_max_length = True, return_tensors=\"tf\") # Tokenizing\n",
    "  ans_BERT_tokens = bert_tokenizer.batch_encode_plus(answer_pr, max_length = 512, truncation = True, pad_to_max_length = True, return_tensors=\"tf\")\n",
    "  x_BERT = bert_model([title_body_BERT_tokens['input_ids'], title_body_BERT_tokens['attention_mask'], title_body_BERT_tokens['token_type_ids']])[2] # Prediction\n",
    "  tb_BERT = tf.concat([tf.reduce_mean(i, axis = 1) for i in x_BERT[-4:]], axis = 1)\n",
    "  x_BERT = bert_model([ans_BERT_tokens['input_ids'], ans_BERT_tokens['attention_mask'], ans_BERT_tokens['token_type_ids']])[2]\n",
    "  ans_BERT = tf.concat([i[:, 0, :] for i in x_BERT[-4:]], axis = 1)\n",
    "\n",
    "  # 2.2 USE_embeddings\n",
    "  tb_USE = use_model(title_body_pr)\n",
    "  ans_USE = use_model(answer_pr)\n",
    "\n",
    "  # 2.3 RoBERTa_embeddings\n",
    "  title_body_RoBERTa_tokens = roberta_tokenizer.batch_encode_plus(title_body_pr, max_length = 512, truncation = True, pad_to_max_length = True, return_tensors=\"tf\") # Tokenizing\n",
    "  ans_RoBERTa_tokens = roberta_tokenizer.batch_encode_plus(answer_pr, max_length = 512, truncation = True, pad_to_max_length = True, return_tensors=\"tf\")\n",
    "  x_RoBERTa = roberta_model([title_body_RoBERTa_tokens['input_ids'], title_body_RoBERTa_tokens['attention_mask']])[2] # Prediction\n",
    "  tb_RoBERTa = tf.concat([i[:, 0, :] for i in x_RoBERTa[-4:]], axis = 1)\n",
    "  x_RoBERTa = roberta_model([ans_RoBERTa_tokens['input_ids'], ans_RoBERTa_tokens['attention_mask']])[2]\n",
    "  ans_RoBERTa = tf.concat([i[:, 0, :] for i in x_RoBERTa[-4:]], axis = 1)\n",
    "\n",
    "  # 2.4 XLNet_embeddings\n",
    "  title_body_XLNet_tokens = xlnet_tokenizer.batch_encode_plus(title_body_pr, max_length = 512, truncation = True, pad_to_max_length = True, return_tensors=\"tf\") # Tokenizing\n",
    "  ans_XLNet_tokens = xlnet_tokenizer.batch_encode_plus(answer_pr, max_length = 512, truncation = True, pad_to_max_length = True, return_tensors=\"tf\")\n",
    "  x_XLNet = xlnet_model([title_body_XLNet_tokens['input_ids'], title_body_XLNet_tokens['attention_mask']])[1] # Prediction\n",
    "  tb_XLNet = tf.concat([tf.reduce_mean(i, axis = 1) for i in x_XLNet[-4:]], axis = 1)\n",
    "  x_XLNet = xlnet_model([ans_XLNet_tokens['input_ids'], ans_XLNet_tokens['attention_mask']])[1]\n",
    "  ans_XLNet = tf.concat([tf.reduce_mean(i, axis = 1) for i in x_XLNet[-4:]], axis = 1)\n",
    "\n",
    "  return [tb_BERT, tb_USE, tb_RoBERTa, tb_XLNet, ans_BERT, ans_USE, ans_RoBERTa, ans_XLNet]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Smp9qsdIEj6a"
   },
   "outputs": [],
   "source": [
    "# 3. final_function\n",
    "def final_pipeline(ques_title, ques_body, answer):\n",
    "  # 1. text_preprocessing\n",
    "  title_body_pr = text_preprocessor([ques_title + ' ' + ques_body])\n",
    "  answer_pr = text_preprocessor([answer])\n",
    "\n",
    "  # 2. get all sentence embeddings   \n",
    "  ins = get_embeddings(title_body_pr, answer_pr)\n",
    "\n",
    "  # 3. model_prediction\n",
    "  pred = model.predict(x = ins)\n",
    "\n",
    "  # 4. binned_output\n",
    "  final_out = binned_out(pred)\n",
    "  return final_out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "9wGQjcL2GbIK"
   },
   "source": [
    "### Final_prediction :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 131
    },
    "colab_type": "code",
    "id": "sVXqtUcCzrf8",
    "outputId": "3b65c5d3-b568-422e-99fb-0c0a5f728545"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.88888889 0.55555556 0.         0.66666667 0.66666667 0.5\n",
      "  0.66666667 0.5        0.5        0.         0.66666667 0.33333333\n",
      "  0.         0.         0.         0.         0.5        0.\n",
      "  0.5        0.         0.88888889 1.         0.66666667 1.\n",
      "  1.         0.86666667 0.33333333 0.         0.66666667 0.88888889]]\n",
      "CPU times: user 741 ms, sys: 26.4 ms, total: 767 ms\n",
      "Wall time: 779 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "dp = 0\n",
    "ques_title = df_train.question_title.values[dp]\n",
    "ques_body = df_train.question_body.values[dp]\n",
    "answer = df_train.answer.values[dp]\n",
    "\n",
    "print(final_pipeline(ques_title, ques_body, answer))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Summary :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Please summarize your total work and the below topics.\n",
    "\n",
    "1. The business problem you are solving. \n",
    "2. The loss function used and why\n",
    "3. Metric used.\n",
    "4. Final summary with findings and results."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Business Problem :\n",
    "\n",
    "  * 'To build a predictive algorithm for different subjective aspects of question-answering'.\n",
    "  * For any question-aswering platform, it is important to know that - how are the questions and its corresponding answers interacting with users with different contexts.\n",
    "  * Provided title, body and answer pair we want to predict some 30 continuous target features, all are ranging from 0 to 1. \n",
    "    \n",
    "### 2. Loss function :\n",
    "\n",
    "The appropriate loss function for this particular problem is 'binary_cross_entropy'.\n",
    "  \n",
    "* $binary\\_cross\\_entropy(yi, pi) = -\\sum_{i=1}^n{(y_i\\log(p_i) + (1 - y_i)\\log(1 - p_i))}$\n",
    "    \n",
    "Q. Why BCE? since its regression problem then why not MSE (mean_squared_error)?\n",
    "   \n",
    "- Answer is simple, all features are ranging from 0 to 1. if we take MSE of 0.51 and 0.65, $MSE =  (0.51-0.65)^2 = 0.0196$.\n",
    "- MSE results are very smaller and hence gradients will also be very small.\n",
    "- Ultimately the model will not able make any progress on training data.\n",
    "- Inversely, BCE is made for values ranging between 0 to 1, and hence BCE is the appropriate loss function even if it's regression problem.\n",
    "\n",
    "### 3. Business Metric :\n",
    "\n",
    "- Appropriate metric for the current problem is **spearman__rank_correlation_coefficient (rho)**\n",
    "\n",
    "* $\\rho(X, Y) = \\frac{\\text{cov}(X_r,Y_r)}{\\sigma_{xr} \\sigma_{yr}}$\n",
    "\n",
    "- Given two random variable (y_true) and (y_pred), computing how these arrays are correlated to each other. \n",
    "- **spearman__rank_correlation** is the robust metric which can capture - complex relationships between arrays upto moderate extent.\n",
    "- Because, 'ranking' or 'the order of values' is the basis of computing covariance between variables, this enables 'spearman__rank_correlation' to capture relationships robustly.\n",
    "\n",
    "### 4. Findings and summary :\n",
    "\n",
    "1. RESULTS :\n",
    "\n",
    "|         Model          | train_spearman | val_spearman |\n",
    "|------------------------|----------------|--------------|\n",
    "|     baseline_LSTM      |     0.4995     |   0.34899    |\n",
    "|          BERT          |    0.50081     |   0.39536    |\n",
    "|          USE           |    0.52245     |    0.4366    |\n",
    "|        RoBERTa         |    0.48059     |   0.41772    |\n",
    "|         XLNet          |    0.52237     |   0.37525    |\n",
    "| BERT_USE_RoBERTa_XLNet |     0.5068     |   0.43924    |\n",
    "\n",
    "  - Combination of 'BERT, USE, RoBERTa, XLNet' has given winning results but not with significant margin. Hence 'USE' is preferable.\n",
    "  - So far, 'USE' embeddings are far more preferable at production environment for our task, Reasons are as follow :\n",
    "    1. Comparitively, forward pass through 'USE' is not very expensive task in terms of time and computation.\n",
    "    2. USE was actually built to capture semantic similarity.\n",
    "    \n",
    "2. No. of datapoints that actually matters in this kind of NLP problems. We dealt with very less no.of datapoints hence model failed to show further improvements.\n",
    "\n",
    "3. Pretrained sentence embeddings are more powerful when :\n",
    "    1. Manually trained on the similar data or the task as current problem has...\n",
    "    2. Fine tuned on current dataset.\n",
    "    \n",
    "(NOTE : Both task are computationally very expensive, hence not able to introduced in the current work.)\n",
    "\n",
    "4. Imbalanced data : in this experiment very basic and key learning is no model can deal with imbalanced data. Complex models like BERT can handle data imabalancing upto some extent if not completely. Hence better we get balanced and enough data.\n",
    "\n",
    "5. 'Post processing : target feature binning', since the metric is 'spearman_rank_corr' which takes order into consideration. It became important to get similar values as (y_true) has in prediction results (y_pred). And that has shown smaller improvements if not very significant improvement."
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "4.Final_pipeline.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "03c50f80c04b4167b1ad96b9380ffc0d": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_20a2c5f7ca794559837b78e1af80d8f0",
      "placeholder": "​",
      "style": "IPY_MODEL_9f1685bb55614aa597dca385a8e69842",
      "value": " 536M/536M [00:12&lt;00:00, 43.3MB/s]"
     }
    },
    "03edebaef113444f844ee57fed52a920": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Downloading: 100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_448c0da207c343509f49bfb43d9f10ed",
      "max": 760,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_ee134f48faec4047917667634c6336c4",
      "value": 760
     }
    },
    "051c94c779a64e5593a491e0899966fc": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_dffeef9372084b3288fcc53c7efa4325",
       "IPY_MODEL_ebe34cb546554d498b8b70d5453ce9fe"
      ],
      "layout": "IPY_MODEL_0fc05e4422bf4a1a8f381c25c5023ba9"
     }
    },
    "0609d8ecd7f44bddbc18120bacabe9c4": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_996b984802d94a76850ffaa7a10e0faa",
      "placeholder": "​",
      "style": "IPY_MODEL_223be068a8b34fe08cdee9e78eaf8896",
      "value": " 456k/456k [00:00&lt;00:00, 973kB/s]"
     }
    },
    "07e7b019663e487287644406bc7a1612": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Downloading: 100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_8542f2bf25a640a287c9f76f66e2ac60",
      "max": 481,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_b088f7eb2c384213944b38c48659c89f",
      "value": 481
     }
    },
    "0bdd00e54a1b49ef8566f8aba098b318": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "0fc05e4422bf4a1a8f381c25c5023ba9": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "14c30e31004f4cd98d4d748f903a122f": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "14cb55958b3b4d3c8cfccd070251b077": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "1acf9d0df80f450081e9f08a6f118717": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "1ccd161c379343eeb71a2dcb6f677255": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Downloading: 100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_d6f97127143c47169a4666964eef86da",
      "max": 433,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_b9a064c9aa2a46f1861153799921f4b6",
      "value": 433
     }
    },
    "1d9e3d03386a47d8b65b5bb8d35b6447": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "1dc104cf8ed845a1840b051583c920fb": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Downloading: 100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_e91d5f782791409aa47e35fe63e43d08",
      "max": 456318,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_ff608fc9cf5f43d7bfd371fc270e7810",
      "value": 456318
     }
    },
    "20a2c5f7ca794559837b78e1af80d8f0": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "223be068a8b34fe08cdee9e78eaf8896": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "2d2ce31cf3c44e0fbe0dcacb589e3912": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_07e7b019663e487287644406bc7a1612",
       "IPY_MODEL_f16ca228750142c19710657f938c4cfe"
      ],
      "layout": "IPY_MODEL_7b3de90cce9142308bc357f0f052929e"
     }
    },
    "2faa97bdfdb84f3cb5175058f4a20df4": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "3551a411cc0b4466a9760a9de4764813": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "3866d379246644b9927ecb425a88366a": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "4004f8c0687941c9a994af47e2c804de": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_aa12f08ae2ef4f519a312554014689f8",
      "placeholder": "​",
      "style": "IPY_MODEL_67cfe01966a747e29a2eab02546d97f7",
      "value": " 760/760 [00:02&lt;00:00, 348B/s]"
     }
    },
    "40de2f8b36b04a5481a8eb43d13e921a": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "448c0da207c343509f49bfb43d9f10ed": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "44fc91ba6a98493d864b7b85fb569ac5": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "4ac1341d43f043c1a8886b91d1b848a1": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Downloading: 100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_8bf63b85a8294aae94218cddecb9843e",
      "max": 798011,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_ff46e71a4f154d6db85235fe23f8d264",
      "value": 798011
     }
    },
    "4c8abcd74b7d409fa876d902e5ab15a0": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "4cd55bb65ee94b6680349b9ffbc11aea": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "50773aa185de4beaa026617fa15894f2": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_a9bfb4eff2684374acba4a3d8f5f15da",
      "placeholder": "​",
      "style": "IPY_MODEL_14c30e31004f4cd98d4d748f903a122f",
      "value": " 798k/798k [00:01&lt;00:00, 568kB/s]"
     }
    },
    "5294efcd04824c0698933bec33acbfd8": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_03edebaef113444f844ee57fed52a920",
       "IPY_MODEL_4004f8c0687941c9a994af47e2c804de"
      ],
      "layout": "IPY_MODEL_4c8abcd74b7d409fa876d902e5ab15a0"
     }
    },
    "56cbda827835433b8ffae3a80afc1d81": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "59ffe7f9c15b46a4a7a614abf225f132": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_3866d379246644b9927ecb425a88366a",
      "placeholder": "​",
      "style": "IPY_MODEL_4cd55bb65ee94b6680349b9ffbc11aea",
      "value": " 433/433 [00:00&lt;00:00, 567B/s]"
     }
    },
    "6044c3933b514544bc8275968abfa763": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_d7dbe389d90a4143beb453df5d14841e",
       "IPY_MODEL_03c50f80c04b4167b1ad96b9380ffc0d"
      ],
      "layout": "IPY_MODEL_56cbda827835433b8ffae3a80afc1d81"
     }
    },
    "6357a43b81544e6c9299fc2b30b3ed33": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "635c1242fc3e49a68783d67484ceac42": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "67cfe01966a747e29a2eab02546d97f7": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "6ad534f7296740e7a03b888ef8991190": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "6b2b9b02b14a44f0ab526c72976e68dd": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "7016449cb8f84983b2aa0e370d667afc": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "7543537519e949e79ef5f264a4851c39": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "782a4d1e297445b6a1a5b005fd1d2250": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "7a0651fa60cf43249a36a55919679d76": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "7b3d671d13f344769522e3f83fbe2714": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "7b3de90cce9142308bc357f0f052929e": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "7fe7f7454c204fb5bb3d20b8208b652c": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_ed1fa678699845e68b937709287a2009",
       "IPY_MODEL_ea3489f39bc14d4ba8574d187133fd8e"
      ],
      "layout": "IPY_MODEL_40de2f8b36b04a5481a8eb43d13e921a"
     }
    },
    "8542f2bf25a640a287c9f76f66e2ac60": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "88bef24e4b1d45acb843a0f9f6dfd1b1": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "88ffe4c3fcf14a62bd48d34dd4494845": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_7543537519e949e79ef5f264a4851c39",
      "placeholder": "​",
      "style": "IPY_MODEL_2faa97bdfdb84f3cb5175058f4a20df4",
      "value": " 565M/565M [00:15&lt;00:00, 36.4MB/s]"
     }
    },
    "8a139a6f91274578be7958a853e0d398": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "8bf63b85a8294aae94218cddecb9843e": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "996b984802d94a76850ffaa7a10e0faa": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "9b604969ed95490aaf078123c72a3a86": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "9f1685bb55614aa597dca385a8e69842": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "a4a7789bb5b84d1e92e123a3249bdc1c": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "a4be9ead558e4c2db6e08a7adc591e72": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_44fc91ba6a98493d864b7b85fb569ac5",
      "placeholder": "​",
      "style": "IPY_MODEL_bab76f5d068c4e35999b74bcbe0200db",
      "value": " 232k/232k [00:00&lt;00:00, 623kB/s]"
     }
    },
    "a9bfb4eff2684374acba4a3d8f5f15da": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "aa12f08ae2ef4f519a312554014689f8": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "ae529881693746beb77ee846797c75b9": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "b088f7eb2c384213944b38c48659c89f": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "b9a064c9aa2a46f1861153799921f4b6": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "bab76f5d068c4e35999b74bcbe0200db": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "bbd2a1699be94416b249a4ca99b07c67": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "c1f0e1271ec54914bfacb3d61047bc5e": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "c70b86ec72e74665adb01e3f093aa3a6": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_f3a2574ad79b41c288ffd3cbcddf16fe",
       "IPY_MODEL_88ffe4c3fcf14a62bd48d34dd4494845"
      ],
      "layout": "IPY_MODEL_7016449cb8f84983b2aa0e370d667afc"
     }
    },
    "d46b71aa651d4cce8a5c380ba97c3763": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Downloading: 100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_6b2b9b02b14a44f0ab526c72976e68dd",
      "max": 231508,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_14cb55958b3b4d3c8cfccd070251b077",
      "value": 231508
     }
    },
    "d479f4414b6247fa9c19d4a3807f9440": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_1dc104cf8ed845a1840b051583c920fb",
       "IPY_MODEL_0609d8ecd7f44bddbc18120bacabe9c4"
      ],
      "layout": "IPY_MODEL_ec28300e36c24a07991f6a94c1bbd7d1"
     }
    },
    "d6f97127143c47169a4666964eef86da": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "d7dbe389d90a4143beb453df5d14841e": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Downloading: 100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_88bef24e4b1d45acb843a0f9f6dfd1b1",
      "max": 536063208,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_7b3d671d13f344769522e3f83fbe2714",
      "value": 536063208
     }
    },
    "dffeef9372084b3288fcc53c7efa4325": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Downloading: 100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_9b604969ed95490aaf078123c72a3a86",
      "max": 657434796,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_a4a7789bb5b84d1e92e123a3249bdc1c",
      "value": 657434796
     }
    },
    "e91d5f782791409aa47e35fe63e43d08": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "ea3489f39bc14d4ba8574d187133fd8e": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_1d9e3d03386a47d8b65b5bb8d35b6447",
      "placeholder": "​",
      "style": "IPY_MODEL_ae529881693746beb77ee846797c75b9",
      "value": " 899k/899k [00:01&lt;00:00, 515kB/s]"
     }
    },
    "ebe34cb546554d498b8b70d5453ce9fe": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_6ad534f7296740e7a03b888ef8991190",
      "placeholder": "​",
      "style": "IPY_MODEL_1acf9d0df80f450081e9f08a6f118717",
      "value": " 657M/657M [00:14&lt;00:00, 45.3MB/s]"
     }
    },
    "ec28300e36c24a07991f6a94c1bbd7d1": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "ed1fa678699845e68b937709287a2009": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Downloading: 100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_c1f0e1271ec54914bfacb3d61047bc5e",
      "max": 898823,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_782a4d1e297445b6a1a5b005fd1d2250",
      "value": 898823
     }
    },
    "ee134f48faec4047917667634c6336c4": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "f16ca228750142c19710657f938c4cfe": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_3551a411cc0b4466a9760a9de4764813",
      "placeholder": "​",
      "style": "IPY_MODEL_6357a43b81544e6c9299fc2b30b3ed33",
      "value": " 481/481 [00:02&lt;00:00, 193B/s]"
     }
    },
    "f24e7848bf554382866eb823398bcd9c": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_4ac1341d43f043c1a8886b91d1b848a1",
       "IPY_MODEL_50773aa185de4beaa026617fa15894f2"
      ],
      "layout": "IPY_MODEL_8a139a6f91274578be7958a853e0d398"
     }
    },
    "f32bf4db71c04005845441e04bc6f571": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_d46b71aa651d4cce8a5c380ba97c3763",
       "IPY_MODEL_a4be9ead558e4c2db6e08a7adc591e72"
      ],
      "layout": "IPY_MODEL_bbd2a1699be94416b249a4ca99b07c67"
     }
    },
    "f3a2574ad79b41c288ffd3cbcddf16fe": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Downloading: 100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_635c1242fc3e49a68783d67484ceac42",
      "max": 565485600,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_7a0651fa60cf43249a36a55919679d76",
      "value": 565485600
     }
    },
    "f7623078e00d4b3faeadbd70a873deb8": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_1ccd161c379343eeb71a2dcb6f677255",
       "IPY_MODEL_59ffe7f9c15b46a4a7a614abf225f132"
      ],
      "layout": "IPY_MODEL_0bdd00e54a1b49ef8566f8aba098b318"
     }
    },
    "ff46e71a4f154d6db85235fe23f8d264": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "ff608fc9cf5f43d7bfd371fc270e7810": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}