troll_contro_nlp.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/pauldudko/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/home/pauldudko/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/home/pauldudko/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/home/pauldudko/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/home/pauldudko/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/home/pauldudko/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "/home/pauldudko/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/home/pauldudko/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/home/pauldudko/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/home/pauldudko/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/home/pauldudko/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/home/pauldudko/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import torch\n",
    "import numpy as np\n",
    "import os\n",
    "import matplotlib.pyplot as plt\n",
    "import sklearn.model_selection as model_selection\n",
    "import tensorflow_hub as hub\n",
    "import tensorflow as tf\n",
    "import re\n",
    "import random\n",
    "import html\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from tqdm import tqdm\n",
    "from urllib.parse import urlparse, urlsplit\n",
    "from math import floor, ceil\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.metrics import roc_curve, auc, classification_report, balanced_accuracy_score\n",
    "from sklearn.utils import shuffle\n",
    "from scipy import stats\n",
    "from scipy.stats import spearmanr, rankdata, mode\n",
    "from transformers.modeling_tf_distilbert import TFDistilBertModel\n",
    "from transformers.modeling_tf_roberta import TFRobertaModel\n",
    "from transformers import (\n",
    "    WEIGHTS_NAME,\n",
    "    AdamW,\n",
    "    BertConfig,\n",
    "    BertForTokenClassification,\n",
    "    BertTokenizer,\n",
    "    BertForPreTraining,\n",
    "    BertModel,\n",
    "    CamembertConfig,\n",
    "    CamembertForTokenClassification,\n",
    "    CamembertTokenizer,\n",
    "    DistilBertModel,\n",
    "    DistilBertConfig,\n",
    "    DistilBertForTokenClassification,\n",
    "    DistilBertTokenizer,\n",
    "    RobertaConfig,\n",
    "    RobertaModel,\n",
    "    RobertaForMaskedLM,\n",
    "    RobertaForTokenClassification,\n",
    "    RobertaTokenizer,\n",
    "    XLMRobertaConfig,\n",
    "    XLMRobertaForTokenClassification,\n",
    "    XLMRobertaTokenizer,\n",
    "    get_linear_schedule_with_warmup,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data  tdr-ppcs.ipynb  tdr.ipynb\r\n"
     ]
    }
   ],
   "source": [
    "!ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "b'Skipping line 4581: expected 20 fields, saw 22\\nSkipping line 9684: expected 20 fields, saw 21\\nSkipping line 11215: expected 20 fields, saw 27\\n'\n",
      "b'Skipping line 33342: expected 20 fields, saw 21\\nSkipping line 38620: expected 20 fields, saw 22\\nSkipping line 40411: expected 20 fields, saw 21\\n'\n",
      "/usr/lib/python3/dist-packages/IPython/core/interactiveshell.py:2718: DtypeWarning: Columns (1,2,3,6) have mixed types.Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    }
   ],
   "source": [
    "troll_df = pd.read_csv(\"data/comments.csv\", error_bad_lines=False)\n",
    "normie_df = pd.read_csv(\"data/csv-zusammenfuehren.de_3mjg6fs7.csv\", error_bad_lines=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 6711 entries, 0 to 6710\n",
      "Data columns (total 28 columns):\n",
      " #   Column                           Non-Null Count  Dtype  \n",
      "---  ------                           --------------  -----  \n",
      " 0   fullname                         6711 non-null   object \n",
      " 1   archived                         6711 non-null   bool   \n",
      " 2   author.name                      6711 non-null   object \n",
      " 3   author_flair_text                807 non-null    object \n",
      " 4   body                             6711 non-null   object \n",
      " 5   controversiality                 6711 non-null   int64  \n",
      " 6   created_utc                      6711 non-null   float64\n",
      " 7   distinguished                    0 non-null      float64\n",
      " 8   downs                            6711 non-null   int64  \n",
      " 9   edited                           6711 non-null   object \n",
      " 10  gilded                           6711 non-null   int64  \n",
      " 11  id                               6711 non-null   object \n",
      " 12  is_submitter                     6711 non-null   bool   \n",
      " 13  likes                            0 non-null      float64\n",
      " 14  link_author                      6711 non-null   object \n",
      " 15  link_id                          6711 non-null   object \n",
      " 16  link_permalink                   6711 non-null   object \n",
      " 17  link_title                       6711 non-null   object \n",
      " 18  link_url                         6711 non-null   object \n",
      " 19  name                             6711 non-null   object \n",
      " 20  num_comments                     6711 non-null   int64  \n",
      " 21  parent_id                        6711 non-null   object \n",
      " 22  permalink                        6711 non-null   object \n",
      " 23  score                            6711 non-null   int64  \n",
      " 24  stickied                         6711 non-null   bool   \n",
      " 25  subreddit.display_name_prefixed  6711 non-null   object \n",
      " 26  subreddit_type                   6711 non-null   object \n",
      " 27  ups                              6711 non-null   int64  \n",
      "dtypes: bool(3), float64(3), int64(6), object(16)\n",
      "memory usage: 1.3+ MB\n"
     ]
    }
   ],
   "source": [
    "troll_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    6711.000000\n",
       "mean        0.036507\n",
       "std         0.187563\n",
       "min         0.000000\n",
       "25%         0.000000\n",
       "50%         0.000000\n",
       "75%         0.000000\n",
       "max         1.000000\n",
       "Name: controversiality, dtype: float64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "troll_df['controversiality'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 45143 entries, 0 to 45142\n",
      "Data columns (total 20 columns):\n",
      " #   Column                  Non-Null Count  Dtype  \n",
      "---  ------                  --------------  -----  \n",
      " 0   body                    45143 non-null  object \n",
      " 1   score_hidden            11985 non-null  object \n",
      " 2   archived                11982 non-null  object \n",
      " 3   name                    11981 non-null  object \n",
      " 4   author                  45131 non-null  object \n",
      " 5   author_flair_text       11461 non-null  object \n",
      " 6   downs                   11979 non-null  object \n",
      " 7   created_utc             45129 non-null  float64\n",
      " 8   subreddit_id            45129 non-null  object \n",
      " 9   link_id                 45129 non-null  object \n",
      " 10  parent_id               45129 non-null  object \n",
      " 11  score                   45129 non-null  float64\n",
      " 12  retrieved_on            44990 non-null  float64\n",
      " 13  controversiality        45129 non-null  float64\n",
      " 14  gilded                  45129 non-null  float64\n",
      " 15  id                      45129 non-null  object \n",
      " 16  subreddit               45129 non-null  object \n",
      " 17  ups                     31545 non-null  float64\n",
      " 18  distinguished           497 non-null    object \n",
      " 19  author_flair_css_class  12635 non-null  object \n",
      "dtypes: float64(6), object(14)\n",
      "memory usage: 6.9+ MB\n"
     ]
    }
   ],
   "source": [
    "normie_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    45129.000000\n",
       "mean         0.020585\n",
       "std          0.141993\n",
       "min          0.000000\n",
       "25%          0.000000\n",
       "50%          0.000000\n",
       "75%          0.000000\n",
       "max          1.000000\n",
       "Name: controversiality, dtype: float64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "normie_df['controversiality'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>fullname</th>\n",
       "      <th>archived</th>\n",
       "      <th>author.name</th>\n",
       "      <th>author_flair_text</th>\n",
       "      <th>body</th>\n",
       "      <th>controversiality</th>\n",
       "      <th>created_utc</th>\n",
       "      <th>distinguished</th>\n",
       "      <th>downs</th>\n",
       "      <th>edited</th>\n",
       "      <th>...</th>\n",
       "      <th>link_url</th>\n",
       "      <th>name</th>\n",
       "      <th>num_comments</th>\n",
       "      <th>parent_id</th>\n",
       "      <th>permalink</th>\n",
       "      <th>score</th>\n",
       "      <th>stickied</th>\n",
       "      <th>subreddit.display_name_prefixed</th>\n",
       "      <th>subreddit_type</th>\n",
       "      <th>ups</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>t1_d687zh5</td>\n",
       "      <td>True</td>\n",
       "      <td>BlackToLive</td>\n",
       "      <td>NaN</td>\n",
       "      <td>A hard look at training and tactics\" = They wi...</td>\n",
       "      <td>0</td>\n",
       "      <td>1.470604e+09</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>http://www.reuters.com/article/us-usa-police-c...</td>\n",
       "      <td>t1_d687zh5</td>\n",
       "      <td>119</td>\n",
       "      <td>t3_4wkn7m</td>\n",
       "      <td>/r/Bad_Cop_No_Donut/comments/4wkn7m/chicago_po...</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "      <td>r/Bad_Cop_No_Donut</td>\n",
       "      <td>public</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>t1_d5wqzhx</td>\n",
       "      <td>True</td>\n",
       "      <td>BlackToLive</td>\n",
       "      <td>NaN</td>\n",
       "      <td>They deserve all of the hate</td>\n",
       "      <td>0</td>\n",
       "      <td>1.469847e+09</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>http://reason.com/blog/2016/07/28/pine-bluff-c...</td>\n",
       "      <td>t1_d5wqzhx</td>\n",
       "      <td>96</td>\n",
       "      <td>t3_4v5xpc</td>\n",
       "      <td>/r/Bad_Cop_No_Donut/comments/4v5xpc/arkansas_p...</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "      <td>r/Bad_Cop_No_Donut</td>\n",
       "      <td>public</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>t1_d5qvqfw</td>\n",
       "      <td>True</td>\n",
       "      <td>BlackToLive</td>\n",
       "      <td>NaN</td>\n",
       "      <td>I guess that's what they mean when say \"I don'...</td>\n",
       "      <td>0</td>\n",
       "      <td>1.469498e+09</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>http://countercurrentnews.com/2016/07/no-charg...</td>\n",
       "      <td>t1_d5qvqfw</td>\n",
       "      <td>210</td>\n",
       "      <td>t1_d5qeyrw</td>\n",
       "      <td>/r/Bad_Cop_No_Donut/comments/4uiezg/no_charges...</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "      <td>r/Bad_Cop_No_Donut</td>\n",
       "      <td>public</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>t1_d5quz9y</td>\n",
       "      <td>True</td>\n",
       "      <td>BlackToLive</td>\n",
       "      <td>NaN</td>\n",
       "      <td>It's never too late for them, It's never too c...</td>\n",
       "      <td>0</td>\n",
       "      <td>1.469497e+09</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>http://www.opposingviews.com/i/society/police-...</td>\n",
       "      <td>t1_d5quz9y</td>\n",
       "      <td>18</td>\n",
       "      <td>t3_4uicjv</td>\n",
       "      <td>/r/Bad_Cop_No_Donut/comments/4uicjv/police_off...</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "      <td>r/Bad_Cop_No_Donut</td>\n",
       "      <td>public</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>t1_d565ls1</td>\n",
       "      <td>True</td>\n",
       "      <td>BlackToLive</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://petitions.whitehouse.gov//petition/pet...</td>\n",
       "      <td>0</td>\n",
       "      <td>1.468114e+09</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>http://www.thelibertyconservative.com/favor-po...</td>\n",
       "      <td>t1_d565ls1</td>\n",
       "      <td>12</td>\n",
       "      <td>t1_d55o1gr</td>\n",
       "      <td>/r/Good_Cop_Free_Donut/comments/4s0s3j/you_can...</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "      <td>r/Good_Cop_Free_Donut</td>\n",
       "      <td>public</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 28 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     fullname  archived  author.name author_flair_text  \\\n",
       "0  t1_d687zh5      True  BlackToLive               NaN   \n",
       "1  t1_d5wqzhx      True  BlackToLive               NaN   \n",
       "2  t1_d5qvqfw      True  BlackToLive               NaN   \n",
       "3  t1_d5quz9y      True  BlackToLive               NaN   \n",
       "4  t1_d565ls1      True  BlackToLive               NaN   \n",
       "\n",
       "                                                body  controversiality  \\\n",
       "0  A hard look at training and tactics\" = They wi...                 0   \n",
       "1                       They deserve all of the hate                 0   \n",
       "2  I guess that's what they mean when say \"I don'...                 0   \n",
       "3  It's never too late for them, It's never too c...                 0   \n",
       "4  https://petitions.whitehouse.gov//petition/pet...                 0   \n",
       "\n",
       "    created_utc  distinguished  downs edited  ...  \\\n",
       "0  1.470604e+09            NaN      0  False  ...   \n",
       "1  1.469847e+09            NaN      0  False  ...   \n",
       "2  1.469498e+09            NaN      0  False  ...   \n",
       "3  1.469497e+09            NaN      0  False  ...   \n",
       "4  1.468114e+09            NaN      0  False  ...   \n",
       "\n",
       "                                            link_url        name  \\\n",
       "0  http://www.reuters.com/article/us-usa-police-c...  t1_d687zh5   \n",
       "1  http://reason.com/blog/2016/07/28/pine-bluff-c...  t1_d5wqzhx   \n",
       "2  http://countercurrentnews.com/2016/07/no-charg...  t1_d5qvqfw   \n",
       "3  http://www.opposingviews.com/i/society/police-...  t1_d5quz9y   \n",
       "4  http://www.thelibertyconservative.com/favor-po...  t1_d565ls1   \n",
       "\n",
       "   num_comments   parent_id  \\\n",
       "0           119   t3_4wkn7m   \n",
       "1            96   t3_4v5xpc   \n",
       "2           210  t1_d5qeyrw   \n",
       "3            18   t3_4uicjv   \n",
       "4            12  t1_d55o1gr   \n",
       "\n",
       "                                           permalink score stickied  \\\n",
       "0  /r/Bad_Cop_No_Donut/comments/4wkn7m/chicago_po...     1    False   \n",
       "1  /r/Bad_Cop_No_Donut/comments/4v5xpc/arkansas_p...     1    False   \n",
       "2  /r/Bad_Cop_No_Donut/comments/4uiezg/no_charges...     1    False   \n",
       "3  /r/Bad_Cop_No_Donut/comments/4uicjv/police_off...     1    False   \n",
       "4  /r/Good_Cop_Free_Donut/comments/4s0s3j/you_can...     1    False   \n",
       "\n",
       "  subreddit.display_name_prefixed subreddit_type ups  \n",
       "0              r/Bad_Cop_No_Donut         public   1  \n",
       "1              r/Bad_Cop_No_Donut         public   1  \n",
       "2              r/Bad_Cop_No_Donut         public   1  \n",
       "3              r/Bad_Cop_No_Donut         public   1  \n",
       "4           r/Good_Cop_Free_Donut         public   1  \n",
       "\n",
       "[5 rows x 28 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "troll_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "troll_df['subreddit'] = troll_df[\"subreddit.display_name_prefixed\"].apply(lambda x: x.split(\"/\")[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>fullname</th>\n",
       "      <td>t1_d687zh5</td>\n",
       "      <td>t1_d5wqzhx</td>\n",
       "      <td>t1_d5qvqfw</td>\n",
       "      <td>t1_d5quz9y</td>\n",
       "      <td>t1_d565ls1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>archived</th>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>author.name</th>\n",
       "      <td>BlackToLive</td>\n",
       "      <td>BlackToLive</td>\n",
       "      <td>BlackToLive</td>\n",
       "      <td>BlackToLive</td>\n",
       "      <td>BlackToLive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>author_flair_text</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>body</th>\n",
       "      <td>A hard look at training and tactics\" = They wi...</td>\n",
       "      <td>They deserve all of the hate</td>\n",
       "      <td>I guess that's what they mean when say \"I don'...</td>\n",
       "      <td>It's never too late for them, It's never too c...</td>\n",
       "      <td>https://petitions.whitehouse.gov//petition/pet...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>controversiality</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>created_utc</th>\n",
       "      <td>1.4706e+09</td>\n",
       "      <td>1.46985e+09</td>\n",
       "      <td>1.4695e+09</td>\n",
       "      <td>1.4695e+09</td>\n",
       "      <td>1.46811e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>distinguished</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>downs</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>edited</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gilded</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <td>d687zh5</td>\n",
       "      <td>d5wqzhx</td>\n",
       "      <td>d5qvqfw</td>\n",
       "      <td>d5quz9y</td>\n",
       "      <td>d565ls1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>is_submitter</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>likes</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>link_author</th>\n",
       "      <td>pheonix200</td>\n",
       "      <td>2centzworth</td>\n",
       "      <td>King_Andersons</td>\n",
       "      <td>[deleted]</td>\n",
       "      <td>[deleted]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>link_id</th>\n",
       "      <td>t3_4wkn7m</td>\n",
       "      <td>t3_4v5xpc</td>\n",
       "      <td>t3_4uiezg</td>\n",
       "      <td>t3_4uicjv</td>\n",
       "      <td>t3_4s0s3j</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>link_permalink</th>\n",
       "      <td>https://www.reddit.com/r/Bad_Cop_No_Donut/comm...</td>\n",
       "      <td>https://www.reddit.com/r/Bad_Cop_No_Donut/comm...</td>\n",
       "      <td>https://www.reddit.com/r/Bad_Cop_No_Donut/comm...</td>\n",
       "      <td>https://www.reddit.com/r/Bad_Cop_No_Donut/comm...</td>\n",
       "      <td>https://www.reddit.com/r/Good_Cop_Free_Donut/c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>link_title</th>\n",
       "      <td>Chicago police may have violated policy in fat...</td>\n",
       "      <td>Arkansas police refuse to return medals secret...</td>\n",
       "      <td>No Charges for Cops Who ‘Accidentally’ Fired 1...</td>\n",
       "      <td>Police Officer Shatters Woman's Window During ...</td>\n",
       "      <td>You Can Be In Favor Of Policing Reform Without...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>link_url</th>\n",
       "      <td>http://www.reuters.com/article/us-usa-police-c...</td>\n",
       "      <td>http://reason.com/blog/2016/07/28/pine-bluff-c...</td>\n",
       "      <td>http://countercurrentnews.com/2016/07/no-charg...</td>\n",
       "      <td>http://www.opposingviews.com/i/society/police-...</td>\n",
       "      <td>http://www.thelibertyconservative.com/favor-po...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>name</th>\n",
       "      <td>t1_d687zh5</td>\n",
       "      <td>t1_d5wqzhx</td>\n",
       "      <td>t1_d5qvqfw</td>\n",
       "      <td>t1_d5quz9y</td>\n",
       "      <td>t1_d565ls1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>num_comments</th>\n",
       "      <td>119</td>\n",
       "      <td>96</td>\n",
       "      <td>210</td>\n",
       "      <td>18</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>parent_id</th>\n",
       "      <td>t3_4wkn7m</td>\n",
       "      <td>t3_4v5xpc</td>\n",
       "      <td>t1_d5qeyrw</td>\n",
       "      <td>t3_4uicjv</td>\n",
       "      <td>t1_d55o1gr</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>permalink</th>\n",
       "      <td>/r/Bad_Cop_No_Donut/comments/4wkn7m/chicago_po...</td>\n",
       "      <td>/r/Bad_Cop_No_Donut/comments/4v5xpc/arkansas_p...</td>\n",
       "      <td>/r/Bad_Cop_No_Donut/comments/4uiezg/no_charges...</td>\n",
       "      <td>/r/Bad_Cop_No_Donut/comments/4uicjv/police_off...</td>\n",
       "      <td>/r/Good_Cop_Free_Donut/comments/4s0s3j/you_can...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>score</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>stickied</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>subreddit.display_name_prefixed</th>\n",
       "      <td>r/Bad_Cop_No_Donut</td>\n",
       "      <td>r/Bad_Cop_No_Donut</td>\n",
       "      <td>r/Bad_Cop_No_Donut</td>\n",
       "      <td>r/Bad_Cop_No_Donut</td>\n",
       "      <td>r/Good_Cop_Free_Donut</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>subreddit_type</th>\n",
       "      <td>public</td>\n",
       "      <td>public</td>\n",
       "      <td>public</td>\n",
       "      <td>public</td>\n",
       "      <td>public</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ups</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>subreddit</th>\n",
       "      <td>Bad_Cop_No_Donut</td>\n",
       "      <td>Bad_Cop_No_Donut</td>\n",
       "      <td>Bad_Cop_No_Donut</td>\n",
       "      <td>Bad_Cop_No_Donut</td>\n",
       "      <td>Good_Cop_Free_Donut</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                 0  \\\n",
       "fullname                                                                t1_d687zh5   \n",
       "archived                                                                      True   \n",
       "author.name                                                            BlackToLive   \n",
       "author_flair_text                                                              NaN   \n",
       "body                             A hard look at training and tactics\" = They wi...   \n",
       "controversiality                                                                 0   \n",
       "created_utc                                                             1.4706e+09   \n",
       "distinguished                                                                  NaN   \n",
       "downs                                                                            0   \n",
       "edited                                                                       False   \n",
       "gilded                                                                           0   \n",
       "id                                                                         d687zh5   \n",
       "is_submitter                                                                 False   \n",
       "likes                                                                          NaN   \n",
       "link_author                                                             pheonix200   \n",
       "link_id                                                                  t3_4wkn7m   \n",
       "link_permalink                   https://www.reddit.com/r/Bad_Cop_No_Donut/comm...   \n",
       "link_title                       Chicago police may have violated policy in fat...   \n",
       "link_url                         http://www.reuters.com/article/us-usa-police-c...   \n",
       "name                                                                    t1_d687zh5   \n",
       "num_comments                                                                   119   \n",
       "parent_id                                                                t3_4wkn7m   \n",
       "permalink                        /r/Bad_Cop_No_Donut/comments/4wkn7m/chicago_po...   \n",
       "score                                                                            1   \n",
       "stickied                                                                     False   \n",
       "subreddit.display_name_prefixed                                 r/Bad_Cop_No_Donut   \n",
       "subreddit_type                                                              public   \n",
       "ups                                                                              1   \n",
       "subreddit                                                         Bad_Cop_No_Donut   \n",
       "\n",
       "                                                                                 1  \\\n",
       "fullname                                                                t1_d5wqzhx   \n",
       "archived                                                                      True   \n",
       "author.name                                                            BlackToLive   \n",
       "author_flair_text                                                              NaN   \n",
       "body                                                  They deserve all of the hate   \n",
       "controversiality                                                                 0   \n",
       "created_utc                                                            1.46985e+09   \n",
       "distinguished                                                                  NaN   \n",
       "downs                                                                            0   \n",
       "edited                                                                       False   \n",
       "gilded                                                                           0   \n",
       "id                                                                         d5wqzhx   \n",
       "is_submitter                                                                 False   \n",
       "likes                                                                          NaN   \n",
       "link_author                                                            2centzworth   \n",
       "link_id                                                                  t3_4v5xpc   \n",
       "link_permalink                   https://www.reddit.com/r/Bad_Cop_No_Donut/comm...   \n",
       "link_title                       Arkansas police refuse to return medals secret...   \n",
       "link_url                         http://reason.com/blog/2016/07/28/pine-bluff-c...   \n",
       "name                                                                    t1_d5wqzhx   \n",
       "num_comments                                                                    96   \n",
       "parent_id                                                                t3_4v5xpc   \n",
       "permalink                        /r/Bad_Cop_No_Donut/comments/4v5xpc/arkansas_p...   \n",
       "score                                                                            1   \n",
       "stickied                                                                     False   \n",
       "subreddit.display_name_prefixed                                 r/Bad_Cop_No_Donut   \n",
       "subreddit_type                                                              public   \n",
       "ups                                                                              1   \n",
       "subreddit                                                         Bad_Cop_No_Donut   \n",
       "\n",
       "                                                                                 2  \\\n",
       "fullname                                                                t1_d5qvqfw   \n",
       "archived                                                                      True   \n",
       "author.name                                                            BlackToLive   \n",
       "author_flair_text                                                              NaN   \n",
       "body                             I guess that's what they mean when say \"I don'...   \n",
       "controversiality                                                                 0   \n",
       "created_utc                                                             1.4695e+09   \n",
       "distinguished                                                                  NaN   \n",
       "downs                                                                            0   \n",
       "edited                                                                       False   \n",
       "gilded                                                                           0   \n",
       "id                                                                         d5qvqfw   \n",
       "is_submitter                                                                 False   \n",
       "likes                                                                          NaN   \n",
       "link_author                                                         King_Andersons   \n",
       "link_id                                                                  t3_4uiezg   \n",
       "link_permalink                   https://www.reddit.com/r/Bad_Cop_No_Donut/comm...   \n",
       "link_title                       No Charges for Cops Who ‘Accidentally’ Fired 1...   \n",
       "link_url                         http://countercurrentnews.com/2016/07/no-charg...   \n",
       "name                                                                    t1_d5qvqfw   \n",
       "num_comments                                                                   210   \n",
       "parent_id                                                               t1_d5qeyrw   \n",
       "permalink                        /r/Bad_Cop_No_Donut/comments/4uiezg/no_charges...   \n",
       "score                                                                            1   \n",
       "stickied                                                                     False   \n",
       "subreddit.display_name_prefixed                                 r/Bad_Cop_No_Donut   \n",
       "subreddit_type                                                              public   \n",
       "ups                                                                              1   \n",
       "subreddit                                                         Bad_Cop_No_Donut   \n",
       "\n",
       "                                                                                 3  \\\n",
       "fullname                                                                t1_d5quz9y   \n",
       "archived                                                                      True   \n",
       "author.name                                                            BlackToLive   \n",
       "author_flair_text                                                              NaN   \n",
       "body                             It's never too late for them, It's never too c...   \n",
       "controversiality                                                                 0   \n",
       "created_utc                                                             1.4695e+09   \n",
       "distinguished                                                                  NaN   \n",
       "downs                                                                            0   \n",
       "edited                                                                       False   \n",
       "gilded                                                                           0   \n",
       "id                                                                         d5quz9y   \n",
       "is_submitter                                                                 False   \n",
       "likes                                                                          NaN   \n",
       "link_author                                                              [deleted]   \n",
       "link_id                                                                  t3_4uicjv   \n",
       "link_permalink                   https://www.reddit.com/r/Bad_Cop_No_Donut/comm...   \n",
       "link_title                       Police Officer Shatters Woman's Window During ...   \n",
       "link_url                         http://www.opposingviews.com/i/society/police-...   \n",
       "name                                                                    t1_d5quz9y   \n",
       "num_comments                                                                    18   \n",
       "parent_id                                                                t3_4uicjv   \n",
       "permalink                        /r/Bad_Cop_No_Donut/comments/4uicjv/police_off...   \n",
       "score                                                                            1   \n",
       "stickied                                                                     False   \n",
       "subreddit.display_name_prefixed                                 r/Bad_Cop_No_Donut   \n",
       "subreddit_type                                                              public   \n",
       "ups                                                                              1   \n",
       "subreddit                                                         Bad_Cop_No_Donut   \n",
       "\n",
       "                                                                                 4  \n",
       "fullname                                                                t1_d565ls1  \n",
       "archived                                                                      True  \n",
       "author.name                                                            BlackToLive  \n",
       "author_flair_text                                                              NaN  \n",
       "body                             https://petitions.whitehouse.gov//petition/pet...  \n",
       "controversiality                                                                 0  \n",
       "created_utc                                                            1.46811e+09  \n",
       "distinguished                                                                  NaN  \n",
       "downs                                                                            0  \n",
       "edited                                                                       False  \n",
       "gilded                                                                           0  \n",
       "id                                                                         d565ls1  \n",
       "is_submitter                                                                 False  \n",
       "likes                                                                          NaN  \n",
       "link_author                                                              [deleted]  \n",
       "link_id                                                                  t3_4s0s3j  \n",
       "link_permalink                   https://www.reddit.com/r/Good_Cop_Free_Donut/c...  \n",
       "link_title                       You Can Be In Favor Of Policing Reform Without...  \n",
       "link_url                         http://www.thelibertyconservative.com/favor-po...  \n",
       "name                                                                    t1_d565ls1  \n",
       "num_comments                                                                    12  \n",
       "parent_id                                                               t1_d55o1gr  \n",
       "permalink                        /r/Good_Cop_Free_Donut/comments/4s0s3j/you_can...  \n",
       "score                                                                            1  \n",
       "stickied                                                                     False  \n",
       "subreddit.display_name_prefixed                              r/Good_Cop_Free_Donut  \n",
       "subreddit_type                                                              public  \n",
       "ups                                                                              1  \n",
       "subreddit                                                      Good_Cop_Free_Donut  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "troll_df.head().T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>body</th>\n",
       "      <td>I guess there was some awkwardness, but it was...</td>\n",
       "      <td>Food schmooze isn't bad, I've gotten a few int...</td>\n",
       "      <td>Is there a fatigue animation before sleep that...</td>\n",
       "      <td>I recently saw some Shohei Inamura films, spec...</td>\n",
       "      <td>How long did that take?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>score_hidden</th>\n",
       "      <td>false</td>\n",
       "      <td>false</td>\n",
       "      <td>false</td>\n",
       "      <td>false</td>\n",
       "      <td>false</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>archived</th>\n",
       "      <td>false</td>\n",
       "      <td>false</td>\n",
       "      <td>false</td>\n",
       "      <td>false</td>\n",
       "      <td>false</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>name</th>\n",
       "      <td>t1_cob0s2g</td>\n",
       "      <td>t1_cov0mx1</td>\n",
       "      <td>t1_colfupi</td>\n",
       "      <td>t1_coyhhvn</td>\n",
       "      <td>t1_cos5df5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>author</th>\n",
       "      <td>elevenghosts</td>\n",
       "      <td>BeerBaldBeard</td>\n",
       "      <td>tmrxwoot</td>\n",
       "      <td>Zassolluto711</td>\n",
       "      <td>Ditto_B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>author_flair_text</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>downs</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>created_utc</th>\n",
       "      <td>1.42307e+09</td>\n",
       "      <td>1.42474e+09</td>\n",
       "      <td>1.42394e+09</td>\n",
       "      <td>1.425e+09</td>\n",
       "      <td>1.42449e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>subreddit_id</th>\n",
       "      <td>t5_2qh1i</td>\n",
       "      <td>t5_2qji0</td>\n",
       "      <td>t5_2rron</td>\n",
       "      <td>t5_2qh3s</td>\n",
       "      <td>t5_2qtol</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>link_id</th>\n",
       "      <td>t3_2urgi1</td>\n",
       "      <td>t3_2wj4w8</td>\n",
       "      <td>t3_2vvy1m</td>\n",
       "      <td>t3_2x9q0x</td>\n",
       "      <td>t3_2w566b</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>parent_id</th>\n",
       "      <td>t1_cob0b2n</td>\n",
       "      <td>t3_2wj4w8</td>\n",
       "      <td>t1_colddwr</td>\n",
       "      <td>t3_2x9q0x</td>\n",
       "      <td>t1_contf71</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>score</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>retrieved_on</th>\n",
       "      <td>1.4242e+09</td>\n",
       "      <td>1.42476e+09</td>\n",
       "      <td>1.42459e+09</td>\n",
       "      <td>1.42748e+09</td>\n",
       "      <td>1.42471e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>controversiality</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gilded</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <td>cob0s2g</td>\n",
       "      <td>cov0mx1</td>\n",
       "      <td>colfupi</td>\n",
       "      <td>coyhhvn</td>\n",
       "      <td>cos5df5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>subreddit</th>\n",
       "      <td>AskReddit</td>\n",
       "      <td>Connecticut</td>\n",
       "      <td>MonsterHunter</td>\n",
       "      <td>movies</td>\n",
       "      <td>iastate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ups</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>distinguished</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>author_flair_css_class</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                        0  \\\n",
       "body                    I guess there was some awkwardness, but it was...   \n",
       "score_hidden                                                        false   \n",
       "archived                                                            false   \n",
       "name                                                           t1_cob0s2g   \n",
       "author                                                       elevenghosts   \n",
       "author_flair_text                                                     NaN   \n",
       "downs                                                                   0   \n",
       "created_utc                                                   1.42307e+09   \n",
       "subreddit_id                                                     t5_2qh1i   \n",
       "link_id                                                         t3_2urgi1   \n",
       "parent_id                                                      t1_cob0b2n   \n",
       "score                                                                   1   \n",
       "retrieved_on                                                   1.4242e+09   \n",
       "controversiality                                                        0   \n",
       "gilded                                                                  0   \n",
       "id                                                                cob0s2g   \n",
       "subreddit                                                       AskReddit   \n",
       "ups                                                                     1   \n",
       "distinguished                                                         NaN   \n",
       "author_flair_css_class                                                NaN   \n",
       "\n",
       "                                                                        1  \\\n",
       "body                    Food schmooze isn't bad, I've gotten a few int...   \n",
       "score_hidden                                                        false   \n",
       "archived                                                            false   \n",
       "name                                                           t1_cov0mx1   \n",
       "author                                                      BeerBaldBeard   \n",
       "author_flair_text                                                     NaN   \n",
       "downs                                                                   0   \n",
       "created_utc                                                   1.42474e+09   \n",
       "subreddit_id                                                     t5_2qji0   \n",
       "link_id                                                         t3_2wj4w8   \n",
       "parent_id                                                       t3_2wj4w8   \n",
       "score                                                                   1   \n",
       "retrieved_on                                                  1.42476e+09   \n",
       "controversiality                                                        0   \n",
       "gilded                                                                  0   \n",
       "id                                                                cov0mx1   \n",
       "subreddit                                                     Connecticut   \n",
       "ups                                                                     1   \n",
       "distinguished                                                         NaN   \n",
       "author_flair_css_class                                                NaN   \n",
       "\n",
       "                                                                        2  \\\n",
       "body                    Is there a fatigue animation before sleep that...   \n",
       "score_hidden                                                        false   \n",
       "archived                                                            false   \n",
       "name                                                           t1_colfupi   \n",
       "author                                                           tmrxwoot   \n",
       "author_flair_text                                                     NaN   \n",
       "downs                                                                   0   \n",
       "created_utc                                                   1.42394e+09   \n",
       "subreddit_id                                                     t5_2rron   \n",
       "link_id                                                         t3_2vvy1m   \n",
       "parent_id                                                      t1_colddwr   \n",
       "score                                                                   1   \n",
       "retrieved_on                                                  1.42459e+09   \n",
       "controversiality                                                        0   \n",
       "gilded                                                                  0   \n",
       "id                                                                colfupi   \n",
       "subreddit                                                   MonsterHunter   \n",
       "ups                                                                     1   \n",
       "distinguished                                                         NaN   \n",
       "author_flair_css_class                                                NaN   \n",
       "\n",
       "                                                                        3  \\\n",
       "body                    I recently saw some Shohei Inamura films, spec...   \n",
       "score_hidden                                                        false   \n",
       "archived                                                            false   \n",
       "name                                                           t1_coyhhvn   \n",
       "author                                                      Zassolluto711   \n",
       "author_flair_text                                                     NaN   \n",
       "downs                                                                   0   \n",
       "created_utc                                                     1.425e+09   \n",
       "subreddit_id                                                     t5_2qh3s   \n",
       "link_id                                                         t3_2x9q0x   \n",
       "parent_id                                                       t3_2x9q0x   \n",
       "score                                                                   2   \n",
       "retrieved_on                                                  1.42748e+09   \n",
       "controversiality                                                        0   \n",
       "gilded                                                                  0   \n",
       "id                                                                coyhhvn   \n",
       "subreddit                                                          movies   \n",
       "ups                                                                     2   \n",
       "distinguished                                                         NaN   \n",
       "author_flair_css_class                                                NaN   \n",
       "\n",
       "                                              4  \n",
       "body                    How long did that take?  \n",
       "score_hidden                              false  \n",
       "archived                                  false  \n",
       "name                                 t1_cos5df5  \n",
       "author                                  Ditto_B  \n",
       "author_flair_text                           NaN  \n",
       "downs                                         0  \n",
       "created_utc                         1.42449e+09  \n",
       "subreddit_id                           t5_2qtol  \n",
       "link_id                               t3_2w566b  \n",
       "parent_id                            t1_contf71  \n",
       "score                                         2  \n",
       "retrieved_on                        1.42471e+09  \n",
       "controversiality                              0  \n",
       "gilded                                        0  \n",
       "id                                      cos5df5  \n",
       "subreddit                               iastate  \n",
       "ups                                           2  \n",
       "distinguished                               NaN  \n",
       "author_flair_css_class                      NaN  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "normie_df.head().T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def url_tracker(text):\n",
    "    find = re.compile('www\\.[\\S]+|https?://[\\S]+[\\w|/]')\n",
    "    urls = re.findall(find, html.unescape(text))\n",
    "    uri = \"\"\n",
    "    if len(urls)>0:\n",
    "\n",
    "        for url in urls:\n",
    "            if url != \"\":\n",
    "                uri = uri+\" \"+netloc_parser(url)+\" \"+url_parser(url)+\" \"\n",
    "    return uri\n",
    "\n",
    "def url_parser(text):\n",
    "    path = urlsplit(text).path\n",
    "    words = re.findall(r\"[\\w']+\", path)\n",
    "    uri = \"\"\n",
    "    for word in words: \n",
    "        uri = uri + word + \" \"\n",
    "    uri.strip()\n",
    "    return uri\n",
    "\n",
    "def netloc_parser(text):\n",
    "    ntlc = urlsplit(text).netloc.split(\".\")\n",
    "    uri = ntlc[0]\n",
    "    \n",
    "    if uri == \"www\":\n",
    "        uri = ntlc[1]\n",
    "        \n",
    "    if uri == \"reddit\":\n",
    "        uri = \"\"\n",
    "    elif uri == \"i\":\n",
    "        uri = \"imgur\"\n",
    "    elif uri == \"youtu\":\n",
    "        uri = \"youtube\"\n",
    "    \n",
    "    return uri\n",
    "\n",
    "normie_df['link_data'] = normie_df['body'].apply(lambda x : url_tracker(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "SYMBOLS_TO_ISOLATE = '.,?!-;*\"…:—()%#$&_/@＼・ω+=”“[]^–>\\\\°<~•≠™ʊɒ∞§{}·τα❤☺ɡ|¢→̶`❥━┣┫┗Ｏ►★©―ɪ✔®\\x96\\x92●£♥➤´¹☕≈÷♡◐║▬′ɔː€۩۞†μ✒➥═☆ˌ◄½ʻπδηλσερνʃ✬ＳＵＰＥＲＩＴ☻±♍µº¾✓◾؟．⬅℅»Вав❣⋅¿¬♫ＣＭβ█▓▒░⇒⭐›¡₂₃❧▰▔◞▀▂▃▄▅▆▇↙γ̄″☹➡«φ⅓„✋：¥̲̅́∙‛◇✏▷❓❗¶˚˙）сиʿ✨。ɑ\\x80◕！％¯−ﬂﬁ₁²ʌ¼⁴⁄₄⌠♭✘╪▶☭✭♪☔☠♂☃☎✈✌✰❆☙○‣⚓年∎ℒ▪▙☏⅛ｃａｓǀ℮¸ｗ‚∼‖ℳ❄←☼⋆ʒ⊂、⅔¨͡๏⚾⚽Φ×θ￦？（℃⏩☮⚠月✊❌⭕▸■⇌☐☑⚡☄ǫ╭∩╮，例＞ʕɐ̣Δ₀✞┈╱╲▏▕┃╰▊▋╯┳┊≥☒↑☝ɹ✅☛♩☞ＡＪＢ◔◡↓♀⬆̱ℏ\\x91⠀ˤ╚↺⇤∏✾◦♬³の｜／∵∴√Ω¤☜▲↳▫‿⬇✧ｏｖｍ－２０８＇‰≤∕ˆ⚜☁'\n",
    "SYMBOLS_TO_REMOVE = '\\n\\r\\xa0\\ue014\\t\\uf818\\uf04a\\xad\\uf0e0\\u200b\\u200eعدويهصقأناخلىبمغرЕ\\u202a\\u202c🏻ᴍʏʀᴇɴᴅᴏᴀᴋʜᴜʟᴛᴄᴘʙғᴊᴡɢשלוםבי‼\\x81エンジ故障\\u2009ᴵ͞\\u200fאעכחஜᴠ‐\\x7fἐὶήιὲκἀίῃἴξＨ\\ufeff\\u2028\\u3000تحكسة👮💙فزط\\u2008🏾\\x08‑地獄谷улкнПоАН歌舞伎הмυтѕ⤵\\u200aэпрд\\x95\\u2002\\u2004ˢᵒʳʸᴼᴷᴺʷᵗʰᵉᵘ\\x13\\ue602άοόςέὸתמדףנרךצט\\uf0b7\\uf04c\\x9f\\x10成都\\u202d💤🍇\\ue613小土豆🏡❔⁉\\u202f👠》कर्मा🇹🇼🌸蔡英文🌞🎲レクサス外国人关系Сб💋💀🎄💜ьыгя不是\\x9c\\x9d🗑\\u2005💃📣༼つ༽ḷЗз▱ц￼卖温哥华议会下降你失去所有的钱加拿大坏税骗子🐝ツ🎅\\x85🍺آإشء🎵🌎͟ἔ油别克🤡й\\u2003🚀🤴ʲшчИОРФДЯМюж🖑ὐύύ特殊作戦群щ💨圆明园קℐ\\u200d𝓒𝓲𝓿𝓵안영하세요ЖљКћ🍀😫🤤ῦ我出生在了可以说普通话汉语好极🎼🕺🍸🥂🗽🎇🎊🆘🤠👩🖒🚪天一家⚲\\u2006⚭⚆⬭⬯⏖新✀╌🇫🇷🇩🇪🇮🇬🇧😷🇨🇦ХШ🌐\\x1f杀鸡给猴看ʁ𝗪𝗵𝗲𝗻𝘆𝗼𝘂𝗿𝗮𝗹𝗶𝘇𝗯𝘁𝗰𝘀𝘅𝗽𝘄𝗱📺ϖ\\u2000үսᴦᎥһͺ\\u2007հ\\u2001ɩｙｅ൦ｌƽｈ𝐓𝐡𝐞𝐫𝐮𝐝𝐚𝐃𝐜𝐩𝐭𝐢𝐨𝐧Ƅᴨןᑯ໐ΤᏧ௦Іᴑ܁𝐬𝐰𝐲𝐛𝐦𝐯𝐑𝐙𝐣𝐇𝐂𝐘𝟎ԜТᗞ౦〔Ꭻ𝐳𝐔𝐱𝟔𝟓𝐅🐋ﬃ💘💓ё𝘥𝘯𝘶💐🌋🌄🌅𝙬𝙖𝙨𝙤𝙣𝙡𝙮𝙘𝙠𝙚𝙙𝙜𝙧𝙥𝙩𝙪𝙗𝙞𝙝𝙛👺🐷ℋ𝐀𝐥𝐪🚶𝙢Ἱ🤘ͦ💸ج패티Ｗ𝙇ᵻ👂👃ɜ🎫\\uf0a7БУі🚢🚂ગુજરાતીῆ🏃𝓬𝓻𝓴𝓮𝓽𝓼☘﴾̯﴿₽\\ue807𝑻𝒆𝒍𝒕𝒉𝒓𝒖𝒂𝒏𝒅𝒔𝒎𝒗𝒊👽😙\\u200cЛ‒🎾👹⎌🏒⛸公寓养宠物吗🏄🐀🚑🤷操美𝒑𝒚𝒐𝑴🤙🐒欢迎来到阿拉斯ספ𝙫🐈𝒌𝙊𝙭𝙆𝙋𝙍𝘼𝙅ﷻ🦄巨收赢得白鬼愤怒要买额ẽ🚗🐳𝟏𝐟𝟖𝟑𝟕𝒄𝟗𝐠𝙄𝙃👇锟斤拷𝗢𝟳𝟱𝟬⦁マルハニチロ株式社⛷한국어ㄸㅓ니͜ʖ𝘿𝙔₵𝒩ℯ𝒾𝓁𝒶𝓉𝓇𝓊𝓃𝓈𝓅ℴ𝒻𝒽𝓀𝓌𝒸𝓎𝙏ζ𝙟𝘃𝗺𝟮𝟭𝟯𝟲👋🦊多伦🐽🎻🎹⛓🏹🍷🦆为和中友谊祝贺与其想象对法如直接问用自己猜本传教士没积唯认识基督徒曾经让相信耶稣复活死怪他但当们聊些政治题时候战胜因圣把全堂结婚孩恐惧且栗谓这样还♾🎸🤕🤒⛑🎁批判检讨🏝🦁🙋😶쥐스탱트뤼도석유가격인상이경제황을렵게만들지않록잘관리해야합다캐나에서대마초와화약금의품런성분갈때는반드시허된사용🔫👁凸ὰ💲🗯𝙈Ἄ𝒇𝒈𝒘𝒃𝑬𝑶𝕾𝖙𝖗𝖆𝖎𝖌𝖍𝖕𝖊𝖔𝖑𝖉𝖓𝖐𝖜𝖞𝖚𝖇𝕿𝖘𝖄𝖛𝖒𝖋𝖂𝕴𝖟𝖈𝕸👑🚿💡知彼百\\uf005𝙀𝒛𝑲𝑳𝑾𝒋𝟒😦𝙒𝘾𝘽🏐𝘩𝘨ὼṑ𝑱𝑹𝑫𝑵𝑪🇰🇵👾ᓇᒧᔭᐃᐧᐦᑳᐨᓃᓂᑲᐸᑭᑎᓀᐣ🐄🎈🔨🐎🤞🐸💟🎰🌝🛳点击查版🍭𝑥𝑦𝑧ＮＧ👣\\uf020っ🏉ф💭🎥Ξ🐴👨🤳🦍\\x0b🍩𝑯𝒒𝟐🏂👳🍗🕉🐲چی𝑮𝗕𝗴🍒ꜥⲣⲏ🐑⏰鉄リ事件ї💊「」\\uf203\\uf09a\\uf222\\ue608\\uf202\\uf099\\uf469\\ue607\\uf410\\ue600燻製シ虚偽屁理屈Г𝑩𝑰𝒀𝑺🌤𝗳𝗜𝗙𝗦𝗧🍊ὺἈἡχῖΛ⤏🇳𝒙ψՁմեռայինրւդձ冬至ὀ𝒁🔹🤚🍎𝑷🐂💅𝘬𝘱𝘸𝘷𝘐𝘭𝘓𝘖𝘹𝘲𝘫کΒώ💢ΜΟΝΑΕ🇱♲𝝈↴💒⊘Ȼ🚴🖕🖤🥘📍👈➕🚫🎨🌑🐻𝐎𝐍𝐊𝑭🤖🎎😼🕷ｇｒｎｔｉｄｕｆｂｋ𝟰🇴🇭🇻🇲𝗞𝗭𝗘𝗤👼📉🍟🍦🌈🔭《🐊🐍\\uf10aლڡ🐦\\U0001f92f\\U0001f92a🐡💳ἱ🙇𝗸𝗟𝗠𝗷🥜さようなら🔼'\n",
    "POSITIVE_EMOJI = '😜😎😁💖😀😂😄😋👏😊👍😃😘👌🙂😉😏🎉😅👻🙃😆🤗🤓😌🤑😛🤣😝💪😗🥰😇🤠🤡🥳🥴🤩😺😸😹😻😽✌️🤟🤘'\n",
    "NEGATIVE_EMOJI = '😢👎😱😳😧🙀😐😕😮😖😟😡😠😤😞😭😥😔😓😪😨😩🙁😵😒͝😣😲😯🤢َِ😰👿👿🤥😬😷🤒🤕🤯🤬🥺🙀😿😾🖕🏻🖕🏼'\n",
    "NEUTRAL_EMOJI = '🐶️🍕🐵💵🔥💥🚌🌟💩💯⛽🚄🏼🚲😈🙏🎯🌹💔👊🙄⛺🍾🏽🎆🍻⏺🌏💞🚓🔔📚🏀👐🍽🎶🌺🤔🐰🐇🏈😺🌍🍔🐮🍁🍆🍑🌮🌯🤦🙈😴🆕👅👥👄🔄🔤👉👤🤧👶👲🔛🎓🏿🇺🇸🌠🐟💫💰🚬💎🐱🙆💕𝘊𝘦𝘳𝘢𝘵𝘰𝘤𝘺𝘴𝘪𝘧𝘮𝘣💗💚🐾🐕🔗🚽🏆🎃🖐🙅⛲🍰🤐👆🙌💛👀🙊🙉'\n",
    "ISOLATE_DICT = {ord(c):' special symbol '.format(ord(c)) for c in SYMBOLS_TO_ISOLATE}\n",
    "REMOVE_DICT = {ord(c):'' for c in SYMBOLS_TO_REMOVE}\n",
    "NEUTRAL_EMOJI_DICT = {ord(c):' neutral emoji ' for c in NEUTRAL_EMOJI}\n",
    "POSITIVE_EMOJI_DICT = {ord(c):' positive emoji ' for c in POSITIVE_EMOJI}\n",
    "NEGATIVE_EMOJI_DICT = {ord(c):' negative emoji ' for c in NEGATIVE_EMOJI}\n",
    "CONTRACTION_MAPPING = {\"ain't\": \"is not\", \"aren't\": \"are not\",\"can't\": \"cannot\", \"'cause\": \"because\", \"could've\": \"could have\", \"couldn't\": \"could not\", \"didn't\": \"did not\",  \"doesn't\": \"does not\", \"don't\": \"do not\", \"hadn't\": \"had not\", \"hasn't\": \"has not\", \"haven't\": \"have not\", \"he'd\": \"he would\",\"he'll\": \"he will\", \"he's\": \"he is\", \"how'd\": \"how did\", \"how'd'y\": \"how do you\", \"how'll\": \"how will\", \"how's\": \"how is\",  \"I'd\": \"I would\", \"I'd've\": \"I would have\", \"I'll\": \"I will\", \"I'll've\": \"I will have\",\"I'm\": \"I am\", \"I've\": \"I have\", \"i'd\": \"i would\", \"i'd've\": \"i would have\", \"i'll\": \"i will\",  \"i'll've\": \"i will have\",\"i'm\": \"i am\", \"i've\": \"i have\", \"isn't\": \"is not\", \"it'd\": \"it would\", \"it'd've\": \"it would have\", \"it'll\": \"it will\", \"it'll've\": \"it will have\",\"it's\": \"it is\", \"let's\": \"let us\", \"ma'am\": \"madam\", \"mayn't\": \"may not\", \"might've\": \"might have\",\"mightn't\": \"might not\",\"mightn't've\": \"might not have\", \"must've\": \"must have\", \"mustn't\": \"must not\", \"mustn't've\": \"must not have\", \"needn't\": \"need not\", \"needn't've\": \"need not have\",\"o'clock\": \"of the clock\", \"oughtn't\": \"ought not\", \"oughtn't've\": \"ought not have\", \"shan't\": \"shall not\", \"sha'n't\": \"shall not\", \"shan't've\": \"shall not have\", \"she'd\": \"she would\", \"she'd've\": \"she would have\", \"she'll\": \"she will\", \"she'll've\": \"she will have\", \"she's\": \"she is\", \"should've\": \"should have\", \"shouldn't\": \"should not\", \"shouldn't've\": \"should not have\", \"so've\": \"so have\",\"so's\": \"so as\", \"this's\": \"this is\",\"that'd\": \"that would\", \"that'd've\": \"that would have\", \"that's\": \"that is\", \"there'd\": \"there would\", \"there'd've\": \"there would have\", \"there's\": \"there is\", \"here's\": \"here is\",\"they'd\": \"they would\", \"they'd've\": \"they would have\", \"they'll\": \"they will\", \"they'll've\": \"they will have\", \"they're\": \"they are\", \"they've\": \"they have\", \"to've\": \"to have\", \"wasn't\": \"was not\", \"we'd\": \"we would\", \"we'd've\": \"we would have\", \"we'll\": \"we will\", \"we'll've\": \"we will have\", \"we're\": \"we are\", \"we've\": \"we have\", \"weren't\": \"were not\", \"what'll\": \"what will\", \"what'll've\": \"what will have\", \"what're\": \"what are\",  \"what's\": \"what is\", \"what've\": \"what have\", \"when's\": \"when is\", \"when've\": \"when have\", \"where'd\": \"where did\", \"where's\": \"where is\", \"where've\": \"where have\", \"who'll\": \"who will\", \"who'll've\": \"who will have\", \"who's\": \"who is\", \"who've\": \"who have\", \"why's\": \"why is\", \"why've\": \"why have\", \"will've\": \"will have\", \"won't\": \"will not\", \"won't've\": \"will not have\", \"would've\": \"would have\", \"wouldn't\": \"would not\", \"wouldn't've\": \"would not have\", \"y'all\": \"you all\", \"y'all'd\": \"you all would\",\"y'all'd've\": \"you all would have\",\"y'all're\": \"you all are\",\"y'all've\": \"you all have\",\"you'd\": \"you would\", \"you'd've\": \"you would have\", \"you'll\": \"you will\", \"you'll've\": \"you will have\", \"you're\": \"you are\", \"you've\": \"you have\" }\n",
    "\n",
    "\n",
    "def handle_punctuation(text):\n",
    "    text = text.translate(REMOVE_DICT)\n",
    "    text = text.translate(NEUTRAL_EMOJI_DICT)\n",
    "    text = text.translate(POSITIVE_EMOJI_DICT)\n",
    "    text = text.translate(NEGATIVE_EMOJI_DICT)\n",
    "    text = text.translate(ISOLATE_DICT)\n",
    "    return text\n",
    "\n",
    "def clean_contractions(text, mapping=CONTRACTION_MAPPING):\n",
    "    specials = [\"’\", \"‘\", \"´\", \"`\"]\n",
    "    for s in specials:\n",
    "        text = text.replace(s, \"'\")\n",
    "    text = ' '.join([mapping[t] if t in mapping else t for t in text.split(\" \")])\n",
    "    return text\n",
    "\n",
    "\n",
    "def text_general_reworking(text):\n",
    "    spaces = ['\\u200b', '\\u200e', '\\u202a', '\\u202c', '\\ufeff', '\\uf0d8', '\\u2061', '\\x10', '\\x7f', '\\x9d', '\\xad',\n",
    "              '\\xa0']\n",
    "    for space in spaces:\n",
    "        text = text.replace(space, ' ')\n",
    "    text = re.sub('\\s+', ' ', text)\n",
    "    text = re.sub(r'\\.{2,}', ' ', text)\n",
    "    text = re.sub(r'(:\\s?\\)|:-\\)|\\(\\s?:|\\(-:|:\\'\\))', ' positive emoji ', text)\n",
    "    text = re.sub(r'(:\\s?D|:-D|x-?D|X-?D)', ' positive emoji ', text)\n",
    "    text = re.sub(r'(<3|:\\*)', ' positive emoji ', text)\n",
    "    text = re.sub(r'(;-?\\)|;-?D|\\(-?;)', ' positive emoji ', text)\n",
    "    text = re.sub(r'(:\\s?\\(|:-\\(|\\)\\s?:|\\)-:)', ' negative emoji ', text)\n",
    "    text = re.sub(r'(:,\\(|:\\'\\(|:\"\\()', ' negative emoji ', text)\n",
    "    text = re.sub(r'((www\\.[\\S]+)|(https?://[\\S]+))', ' link ', text)\n",
    "    text = re.sub(r'\\brt\\b', '', text)\n",
    "    text = re.sub(r'\\[math\\]', ' LaTex math ', text)\n",
    "    text = re.sub(r'\\[\\/math\\]', ' LaTex math ', text)\n",
    "    text = re.sub(r'\\\\', ' LaTex ', text)\n",
    "    text = re.sub(r'\\brt\\b', '', text)\n",
    "    text = re.sub(\"([^\\\"^\\'].\\s)(\\\")([A-Z,a-z?])\", r\"\\1\\3\", text)\n",
    "    text = re.sub(\"(\\')(.\\\")\", r\"\\2\", text)\n",
    "    text = text.lower()\n",
    "    if '-' in text:\n",
    "        text = text.replace('-', ' - ')\n",
    "    text.strip()\n",
    "    return text\n",
    "\n",
    "def text_preprocess(x):\n",
    "    x = text_general_reworking(x)\n",
    "    x = handle_punctuation(x)\n",
    "    x = clean_contractions(x)\n",
    "    return x\n",
    "\n",
    "\n",
    "troll_df['netloc'] = troll_df['link_url'].apply(lambda x : url_parser(x))\n",
    "troll_df['link_data'] = troll_df['link_title'].astype('str') + \" \" + troll_df['netloc'].astype('str')\n",
    "#normie_df['body'] = normie_df['link_data'].astype('str') + \" \" + normie_df['body'].astype('str')\n",
    "\n",
    "troll_df['body'] = troll_df['body'].apply(html.unescape)\n",
    "normie_df['body'] = normie_df['body'].apply(html.unescape)\n",
    "troll_df['body'] = troll_df['body'].apply(lambda x: text_preprocess(x))\n",
    "troll_df['link_data'] = troll_df['link_data'].apply(lambda x: text_preprocess(x))\n",
    "normie_df['body'] = normie_df['body'].apply(lambda x: text_preprocess(x))\n",
    "normie_df['link_data'] = normie_df['link_data'].apply(lambda x: text_preprocess(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "troll_df.dropna(subset = ['body','subreddit'], inplace = True)\n",
    "normie_df.dropna(subset = ['body','subreddit'], inplace = True)\n",
    "normie_df = normie_df[(normie_df['body'].astype('str')!='[deleted]')  &  (normie_df['body'].astype('str')!='[removed]')]\n",
    "normie_df.drop_duplicates(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "troll_df['troll'] = True\n",
    "normie_df['troll'] = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols = [\"body\",'created_utc','link_data',\"subreddit\",\"score\",'controversiality', \"troll\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "normie_link_df = normie_df[(normie_df['link_data'].astype('str')!=\"\") & (normie_df['controversiality']==0)][cols].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 13422 entries, 6174 to 8174\n",
      "Data columns (total 7 columns):\n",
      " #   Column            Non-Null Count  Dtype  \n",
      "---  ------            --------------  -----  \n",
      " 0   body              13422 non-null  object \n",
      " 1   created_utc       13422 non-null  float64\n",
      " 2   link_data         13422 non-null  object \n",
      " 3   subreddit         13422 non-null  object \n",
      " 4   score             13422 non-null  float64\n",
      " 5   controversiality  13422 non-null  float64\n",
      " 6   troll             13422 non-null  bool   \n",
      "dtypes: bool(1), float64(3), object(3)\n",
      "memory usage: 747.1+ KB\n"
     ]
    }
   ],
   "source": [
    "data_df = troll_df[cols]\n",
    "normie_contro_df = normie_df[normie_df['controversiality']==1][cols].copy()\n",
    "data_df = data_df.append(normie_contro_df)\n",
    "n = troll_df.shape[0] - normie_contro_df.shape[0] \n",
    "data_df = data_df.append(normie_df[normie_df['controversiality']==0][cols].sample(n))\n",
    "data_df = shuffle(data_df)\n",
    "data_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_df['body'] = data_df['body'].astype('str')\n",
    "data_df['link_data'] = data_df['link_data'].astype('str')\n",
    "data_df['subreddit'] = data_df['subreddit'].astype('str')\n",
    "data_df['troll'] = data_df['troll'].astype('int')\n",
    "data_df['controversiality'] = data_df['controversiality'].astype('int')\n",
    "\n",
    "def subreddit_decoder(text):\n",
    "    if '_' in text:\n",
    "        text = text.replace('_', ' ')\n",
    "    text = re.sub(\"([a-z])([A-Z])\", r\"\\1 \\2\", text)\n",
    "    return text\n",
    "\n",
    "data_df['subreddit'] = data_df['subreddit'].apply(lambda x: subreddit_decoder(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>body</th>\n",
       "      <th>created_utc</th>\n",
       "      <th>link_data</th>\n",
       "      <th>subreddit</th>\n",
       "      <th>score</th>\n",
       "      <th>controversiality</th>\n",
       "      <th>troll</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6174</th>\n",
       "      <td>special symbol  france threatens haha</td>\n",
       "      <td>1.443650e+09</td>\n",
       "      <td>france threatens to walk away from ttip negoti...</td>\n",
       "      <td>worldnews</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2610</th>\n",
       "      <td>future is here special symbol</td>\n",
       "      <td>1.515363e+09</td>\n",
       "      <td>ties special symbol network is the platform fo...</td>\n",
       "      <td>Database</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38188</th>\n",
       "      <td>special symbol removed special symbol</td>\n",
       "      <td>1.493285e+09</td>\n",
       "      <td></td>\n",
       "      <td>Official DP</td>\n",
       "      <td>-3.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5908</th>\n",
       "      <td>we do special symbol  i did not know that spec...</td>\n",
       "      <td>1.429122e+09</td>\n",
       "      <td></td>\n",
       "      <td>pics</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1711</th>\n",
       "      <td>run special symbol  forrest special symbol  ru...</td>\n",
       "      <td>1.433680e+09</td>\n",
       "      <td>my sweet cat zagcmdl jpg</td>\n",
       "      <td>pics</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29117</th>\n",
       "      <td>eh special symbol  there are new 0days all the...</td>\n",
       "      <td>1.478784e+09</td>\n",
       "      <td></td>\n",
       "      <td>techsupport</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3336</th>\n",
       "      <td>haha special symbol  seems that mr special sym...</td>\n",
       "      <td>1.446211e+09</td>\n",
       "      <td>cops across the nation join nypd in tarantino ...</td>\n",
       "      <td>Bad Cop No Donut</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44552</th>\n",
       "      <td>special symbol  you are assuming on faith tha...</td>\n",
       "      <td>1.484082e+09</td>\n",
       "      <td></td>\n",
       "      <td>Capitalism VSocialism</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2437</th>\n",
       "      <td>have we actually seen anything from ibm regard...</td>\n",
       "      <td>1.515451e+09</td>\n",
       "      <td>ibm executive brief special symbol  three ways...</td>\n",
       "      <td>Stellar</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28397</th>\n",
       "      <td>good luck special symbol  lol special symbol  ...</td>\n",
       "      <td>1.475424e+09</td>\n",
       "      <td></td>\n",
       "      <td>Silverbugs</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    body   created_utc  \\\n",
       "6174              special symbol  france threatens haha   1.443650e+09   \n",
       "2610                      future is here special symbol   1.515363e+09   \n",
       "38188             special symbol removed special symbol   1.493285e+09   \n",
       "5908   we do special symbol  i did not know that spec...  1.429122e+09   \n",
       "1711   run special symbol  forrest special symbol  ru...  1.433680e+09   \n",
       "29117  eh special symbol  there are new 0days all the...  1.478784e+09   \n",
       "3336   haha special symbol  seems that mr special sym...  1.446211e+09   \n",
       "44552   special symbol  you are assuming on faith tha...  1.484082e+09   \n",
       "2437   have we actually seen anything from ibm regard...  1.515451e+09   \n",
       "28397  good luck special symbol  lol special symbol  ...  1.475424e+09   \n",
       "\n",
       "                                               link_data  \\\n",
       "6174   france threatens to walk away from ttip negoti...   \n",
       "2610   ties special symbol network is the platform fo...   \n",
       "38188                                                      \n",
       "5908                                                       \n",
       "1711                           my sweet cat zagcmdl jpg    \n",
       "29117                                                      \n",
       "3336   cops across the nation join nypd in tarantino ...   \n",
       "44552                                                      \n",
       "2437   ibm executive brief special symbol  three ways...   \n",
       "28397                                                      \n",
       "\n",
       "                   subreddit  score  controversiality  troll  \n",
       "6174               worldnews    1.0                 0      1  \n",
       "2610                Database    1.0                 0      1  \n",
       "38188            Official DP   -3.0                 0      0  \n",
       "5908                    pics    3.0                 1      0  \n",
       "1711                    pics    2.0                 0      1  \n",
       "29117            techsupport    1.0                 0      0  \n",
       "3336        Bad Cop No Donut    1.0                 0      1  \n",
       "44552  Capitalism VSocialism    1.0                 0      0  \n",
       "2437                 Stellar    1.0                 0      1  \n",
       "28397             Silverbugs    1.0                 0      0  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>troll</th>\n",
       "      <th>False</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>controversiality</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0.0</th>\n",
       "      <td>44200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1.0</th>\n",
       "      <td>929</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "troll             False\n",
       "controversiality       \n",
       "0.0               44200\n",
       "1.0                 929"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.crosstab(normie_df['controversiality'], normie_df['troll'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>troll</th>\n",
       "      <th>True</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>controversiality</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6466</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>245</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "troll             True\n",
       "controversiality      \n",
       "0                 6466\n",
       "1                  245"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.crosstab(troll_df['controversiality'], troll_df['troll'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>troll</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>All</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>controversiality</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5782</td>\n",
       "      <td>6466</td>\n",
       "      <td>12248</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>929</td>\n",
       "      <td>245</td>\n",
       "      <td>1174</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>All</th>\n",
       "      <td>6711</td>\n",
       "      <td>6711</td>\n",
       "      <td>13422</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "troll                0     1    All\n",
       "controversiality                   \n",
       "0                 5782  6466  12248\n",
       "1                  929   245   1174\n",
       "All               6711  6711  13422"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.crosstab(data_df['controversiality'], data_df['troll'], margins = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "count    13422.000000\n",
      "mean         9.748473\n",
      "std          4.417478\n",
      "min          2.000000\n",
      "25%          6.000000\n",
      "50%         10.000000\n",
      "75%         13.000000\n",
      "max         25.000000\n",
      "Name: subreddit, dtype: float64 count    13422.000000\n",
      "mean       192.142453\n",
      "std        377.289469\n",
      "min          1.000000\n",
      "25%         44.000000\n",
      "50%         98.000000\n",
      "75%        211.000000\n",
      "max      14164.000000\n",
      "Name: body, dtype: float64 count    13422.000000\n",
      "mean        88.395545\n",
      "std        116.963048\n",
      "min          0.000000\n",
      "25%          0.000000\n",
      "50%         33.000000\n",
      "75%        157.000000\n",
      "max       4216.000000\n",
      "Name: link_data, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "subr_len = data_df['subreddit'].map(str).apply(len).describe()\n",
    "body_len = data_df['body'].map(str).apply(len).describe()\n",
    "link_len = data_df['link_data'].map(str).apply(len).describe()\n",
    "print(subr_len, body_len, link_len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_SEQUENCE_LENGTH = 512\n",
    "QUESTION_MAX_LENGTH = 483\n",
    "ANSWER_MAX_LENGTH = 25"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Code below is partly based on:\n",
    "\n",
    "https://www.kaggle.com/shuheigoda/23th-place-solusion\n",
    "https://github.com/oleg-yaroshevskiy/quest_qa_labeling/blob/master/model.py\n",
    "https://www.kaggle.com/abhishek/distilbert-use-features-oof"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def trim_input(question_tokens, answer_tokens, max_seq_len, q_max_len, a_max_len):\n",
    "    q_len = len(question_tokens)\n",
    "    a_len = len(answer_tokens)\n",
    "    if q_len + a_len + 4 > max_seq_len:\n",
    "        if a_max_len <= a_len and q_max_len <= q_len:\n",
    "            q_new_len_head = floor((q_max_len - q_max_len/2))\n",
    "            question_tokens = question_tokens[:q_new_len_head] + question_tokens[q_new_len_head - q_max_len:]\n",
    "            a_new_len_head = floor((a_max_len - a_max_len/2))\n",
    "            answer_tokens = answer_tokens[:a_new_len_head] + answer_tokens[a_new_len_head - a_max_len:]\n",
    "        #elif q_len <= a_len and q_len < q_max_len:\n",
    "        #   a_max_len = a_max_len + (q_max_len - q_len - 1)\n",
    "        #   a_new_len_head = floor((a_max_len - a_max_len/2))\n",
    "        #   answer_tokens = answer_tokens[:a_new_len_head] + answer_tokens[a_new_len_head - a_max_len:]\n",
    "        elif a_len < q_len:\n",
    "            assert a_len <= a_max_len\n",
    "            q_max_len = q_max_len + (a_max_len - a_len - 1)\n",
    "            q_new_len_head = floor((q_max_len - q_max_len/2))\n",
    "            question_tokens = question_tokens[:q_new_len_head] + question_tokens[q_new_len_head - q_max_len:]\n",
    "        else:\n",
    "            raise ValueError(\"unreachable: q_len: {}, a_len: {}, q_max_len: {}, a_max_len: {}\".format(q_len, a_len, q_max_len, a_max_len))\n",
    "    return question_tokens, answer_tokens\n",
    "\n",
    "\n",
    "def get_transformer_inputs(question, question_title, answer, tokenizer, question_only):\n",
    "    question = \"{} [SEP] {}\".format(question_title, question)\n",
    "    question_tokens = tokenizer.tokenize(question)\n",
    "    if question_only:\n",
    "        answer_tokens = []\n",
    "    else:\n",
    "        answer_tokens = tokenizer.tokenize(answer)\n",
    "    question_tokens, answer_tokens = trim_input(question_tokens, answer_tokens, MAX_SEQUENCE_LENGTH, \n",
    "                                                QUESTION_MAX_LENGTH , ANSWER_MAX_LENGTH)\n",
    "    ids = tokenizer.convert_tokens_to_ids([\"[CLS]\"] + question_tokens + [\"[SEP]\"] + answer_tokens + [\"[SEP]\"])\n",
    "    padded_ids = ids + [tokenizer.pad_token_id] * (MAX_SEQUENCE_LENGTH - len(ids))\n",
    "    token_type_ids = [0] * (1 + len(question_tokens) + 1) + [1] * (len(answer_tokens) + 1) \\\n",
    "                                                        + [0] * (MAX_SEQUENCE_LENGTH - len(ids))\n",
    "    attention_mask = [1] * len(ids) + [0] * (MAX_SEQUENCE_LENGTH - len(ids))\n",
    "    return padded_ids, token_type_ids, attention_mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "([101, 102, 2065, 2017, 2064, 8054, 3087, 2008, 1996, 2280, 25467, 4005, 2085, 1999, 3094, 1997, 3607, 2003, 8699, 2017, 2323, 2131, 2019, 4469, 2833, 6463, 2078, 1998, 2028, 1997, 2216, 11281, 2845, 6665, 2569, 6454, 102, 3198, 8398, 6793, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "['[CLS]', '[SEP]', 'if', 'you', 'can', 'convince', 'anyone', 'that', 'the', 'former', 'kgb', 'agent', 'now', 'in', 'command', 'of', 'russia', 'is', 'neutral', 'you', 'should', 'get', 'an', 'extra', 'food', 'ratio', '##n', 'and', 'one', 'of', 'those', 'fancy', 'russian', 'medals', 'special', 'symbol', '[SEP]', 'ask', 'trump', 'supporters', '[SEP]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n"
     ]
    }
   ],
   "source": [
    "bert_tokenizer_path = '../models/bert/bert-base/bert-base-uncased-vocab.txt'\n",
    "bert_tokenizer = BertTokenizer(vocab_file = bert_tokenizer_path)\n",
    "sample_args = data_df[\"body\"].values[15],data_df[\"link_data\"].values[15], data_df[\"subreddit\"].values[15]\n",
    "sample_ids = get_transformer_inputs(*sample_args, bert_tokenizer, False)\n",
    "print(sample_ids)\n",
    "print(bert_tokenizer.convert_ids_to_tokens(sample_ids[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "output_categories = ['troll','controversiality']\n",
    "\n",
    "def compute_input_arrays(df, tokenizer, question_only=False):\n",
    "    input_ids, input_token_type_ids, input_attention_masks = [], [], []\n",
    "    for question, question_title, answer in zip(df[\"body\"].values, df[\"link_data\"].values, df[\"subreddit\"].values):\n",
    "        ids, type_ids, mask = get_transformer_inputs(question, question_title, answer, tokenizer, question_only=question_only)\n",
    "        input_ids.append(ids)\n",
    "        input_token_type_ids.append(type_ids)\n",
    "        input_attention_masks.append(mask)\n",
    "    return (\n",
    "        np.asarray(input_ids, dtype=np.int32),\n",
    "        np.asarray(input_token_type_ids, dtype=np.int32),\n",
    "        np.asarray(input_attention_masks, dtype=np.int32)\n",
    "    )\n",
    "\n",
    "def compute_output_arrays(df):\n",
    "    return np.asarray(df[output_categories], dtype=np.int32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BertStyleModel(torch.nn.Module):\n",
    "    \n",
    "    def __init__(self, model_type):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.model_type = model_type\n",
    "        if (model_type == 'roberta'):\n",
    "            config_path = '../models/bert/roberta-transformers-pytorch/roberta-base/roberta-base-config.json'\n",
    "            model_path = '../models/bert/roberta-transformers-pytorch/roberta-base/roberta-base-pytorch_model.bin'\n",
    "            config = RobertaConfig.from_json_file(config_path)\n",
    "            config.output_hidden_states = True\n",
    "            self.bert = RobertaModel.from_pretrained(model_path, config=config)\n",
    "        elif (model_type == 'distilbert'):\n",
    "            config_path = '../models/bert/distilbert-transformers-pytorch/distilbert-base-uncased-config.json'\n",
    "            config = DistilBertConfig.from_json_file(config_path)\n",
    "            model_path = '../models/bert/distilbert-transformers-pytorch/distilbert-base-uncased-pytorch_model.bin'\n",
    "            config.output_hidden_states = True\n",
    "            self.bert = DistilBertModel.from_pretrained(model_path, config=config)\n",
    "        elif (model_type == 'bert'):\n",
    "            config_path = '../models/bert/bert-base/bert-base-uncased-config.json'\n",
    "            config = BertConfig.from_json_file(config_path)\n",
    "            config.output_hidden_states = True\n",
    "            model_path = '../models/bert/bert-base/bert-base-uncased-pytorch_model.bin'\n",
    "            self.bert = BertModel.from_pretrained(model_path, config=config)\n",
    "            \n",
    "        self.dropout = nn.Dropout(p=0.2)\n",
    "        self.high_dropout = nn.Dropout(p=0.5)   \n",
    "        self.cls_token_head = nn.Sequential(\n",
    "            nn.Dropout(0.2),\n",
    "            nn.Linear(768 * 4, 768),\n",
    "            nn.ReLU(inplace=True),\n",
    "        )\n",
    "        self.classifier = nn.Linear(768, 2)\n",
    "        \n",
    "    def forward(self, input_ids=None, attention_mask=None, token_type_ids=None):\n",
    "        \n",
    "        if (self.model_type == 'roberta'):\n",
    "            outputs = self.bert(input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids)\n",
    "            hidden_layers = outputs[2]\n",
    "        elif (self.model_type == 'distilbert'):\n",
    "            outputs = self.bert(input_ids, attention_mask=attention_mask)\n",
    "            hidden_layers = outputs[1]\n",
    "        elif (self.model_type == 'bert'):     \n",
    "            outputs = self.bert(input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids)\n",
    "            hidden_layers = outputs[2]\n",
    "        \n",
    "        hidden_states_cls_embeddings = [x[:, 0] for x in hidden_layers[-4:]]\n",
    "        x = torch.cat(hidden_states_cls_embeddings, dim=-1)\n",
    "        cls_output = self.cls_token_head(x)\n",
    "        logits = torch.mean(torch.stack([\n",
    "            #Multi Sample Dropout takes place here\n",
    "            self.classifier(self.high_dropout(cls_output))\n",
    "            for _ in range(5)\n",
    "        ], dim=0), dim=0)\n",
    "        outputs = logits\n",
    "        return outputs        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_spearmanr(trues, preds):\n",
    "    rhos = []\n",
    "    for col_trues, col_pred in zip(trues.T, preds.T):\n",
    "        if len(np.unique(col_pred)) == 1:\n",
    "            col_pred[np.random.randint(0, len(col_pred) - 1)] = col_pred.max() + 1\n",
    "        rhos.append(spearmanr(col_trues, col_pred).correlation)\n",
    "    return np.mean(rhos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_and_predict(train_data, valid_data, test_data, epochs, batch_size, model_type, device, fold):\n",
    "    \n",
    "    dataloader = torch.utils.data.DataLoader(train_data, shuffle=True, batch_size=batch_size)\n",
    "    valid_dataloader = torch.utils.data.DataLoader(valid_data, shuffle=False, batch_size=batch_size)\n",
    "    test_dataloader = torch.utils.data.DataLoader(test_data, shuffle=False, batch_size=batch_size)\n",
    "\n",
    "    model = BertStyleModel(model_type).to(device)\n",
    "    \n",
    "    for n, _ in model.named_parameters():\n",
    "        print(n)\n",
    " \n",
    "    test_predictions = []\n",
    "    valid_predictions = []\n",
    "\n",
    "    no_decay = [\"bias\", \"LayerNorm.weight\"]\n",
    "    optimizer_grouped_parameters = [\n",
    "        {\n",
    "            \"params\": [p for n, p in model.named_parameters() if p.requires_grad and not any(nd in n for nd in no_decay) and \"bert\" in n],\n",
    "            \"weight_decay\": 1e-2,\n",
    "            \"lr\": 5e-5\n",
    "        },\n",
    "        {\n",
    "            \"params\": [p for n, p in model.named_parameters() if  p.requires_grad and any(nd in n for nd in no_decay) and \"bert\" in n], \n",
    "            \"weight_decay\": 0.0,\n",
    "            \"lr\": 5e-5\n",
    "        },\n",
    "        {\n",
    "            \"params\": [p for n, p in model.named_parameters() if p.requires_grad and \"bert\" not in n],\n",
    "            \"weight_decay\": 1e-2,\n",
    "            \"lr\": 5e-4\n",
    "            \n",
    "        }\n",
    "    ]\n",
    "    optimizer = AdamW(optimizer_grouped_parameters)\n",
    "    scheduler = get_linear_schedule_with_warmup(\n",
    "        optimizer, num_warmup_steps=int(len(dataloader) * (epochs) * 0.05),\n",
    "        num_training_steps=len(dataloader) * (epochs)\n",
    "    )\n",
    "\n",
    "    for epoch in range(epochs): \n",
    "        import time\n",
    "        start = time.time()\n",
    "        model.train()\n",
    "        train_losses = []\n",
    "        train_preds = []\n",
    "        train_targets = []\n",
    "        for input_ids, token_type_ids, attention_mask, targets in tqdm(dataloader, total=len(dataloader)):\n",
    "            input_ids = input_ids.to(device)\n",
    "            token_type_ids = token_type_ids.to(device)\n",
    "            attention_mask = attention_mask.to(device)\n",
    "            targets = targets.to(device)\n",
    "            \n",
    "            outputs = model(input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids)\n",
    "            \n",
    "            train_preds.extend(outputs.detach().sigmoid().cpu().numpy())\n",
    "            train_targets.extend(targets.detach().cpu().numpy())\n",
    "            \n",
    "            #loss = torch.nn.MultiLabelMarginLoss(outputs, targets)\n",
    "            loss = F.binary_cross_entropy_with_logits(outputs, targets, reduction=\"none\") \n",
    "            loss = (loss * LABEL_WEIGHTS[:loss.size(-1)]).mean()\n",
    "            model.zero_grad()\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            scheduler.step()\n",
    "            train_losses.append(loss.detach().cpu().item())\n",
    "        \n",
    "        model.eval()\n",
    "        valid_losses = []\n",
    "        valid_preds = []\n",
    "        valid_targets = []\n",
    "        with torch.no_grad():\n",
    "            for input_ids, token_type_ids, attention_mask, targets in tqdm(valid_dataloader, total=len(valid_dataloader)):\n",
    "                input_ids = input_ids.to(device)\n",
    "                token_type_ids = token_type_ids.to(device)\n",
    "                attention_mask = attention_mask.to(device)\n",
    "                targets = targets.to(device)\n",
    "                  \n",
    "                outputs = model(input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids)\n",
    "                \n",
    "                valid_preds.extend(outputs.sigmoid().cpu().numpy())\n",
    "                valid_targets.extend(targets.cpu().numpy())\n",
    "                \n",
    "                #loss = torch.nn.MultiLabelMarginLoss(outputs, targets)\n",
    "                loss = F.binary_cross_entropy_with_logits(outputs, targets, reduction=\"none\")\n",
    "                loss = (loss * LABEL_WEIGHTS[:loss.size(-1)]).mean()\n",
    "                valid_losses.append(loss.detach().cpu().item())\n",
    "            \n",
    "            valid_predictions.append(np.stack(valid_preds))\n",
    "            test_preds = []\n",
    "            \n",
    "            for input_ids, token_type_ids, attention_mask in tqdm(test_dataloader, total=len(test_dataloader)):\n",
    "                input_ids = input_ids.to(device)\n",
    "                token_type_ids = token_type_ids.to(device)\n",
    "                attention_mask = attention_mask.to(device)\n",
    "                \n",
    "                outputs = model(input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids)\n",
    "                \n",
    "                test_preds.extend(outputs.sigmoid().cpu().numpy())\n",
    "            \n",
    "            test_predictions.append(np.stack(test_preds))\n",
    "    \n",
    "        print(\"Epoch {}: Train Loss {}, Valid Loss {}\".format(epoch + 1, np.mean(train_losses), np.mean(valid_losses)))\n",
    "        print(\"\\t Train Spearmanr {:.4f}, Valid Spearmanr (avg) {:.4f}, Valid Spearmanr (last) {:.4f}\".format(\n",
    "            compute_spearmanr(np.stack(train_targets), np.stack(train_preds)),\n",
    "            compute_spearmanr(np.stack(valid_targets), sum(valid_predictions) / len(valid_predictions)),\n",
    "            compute_spearmanr(np.stack(valid_targets), valid_predictions[-1])\n",
    "        ))\n",
    "        print(\"\\t elapsed: {}s\".format(time.time() - start))\n",
    "\n",
    "    return valid_predictions, test_predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train, df_test = model_selection.train_test_split(data_df, shuffle=True, test_size=0.2)\n",
    "df_train = df_train.reset_index(drop=True)\n",
    "df_test= df_test.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda\n"
     ]
    }
   ],
   "source": [
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "print(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "troll \t 0.72048426\n",
      "controversiality \t 1.2795157\n"
     ]
    }
   ],
   "source": [
    "LABEL_WEIGHTS = torch.tensor(1.0 / df_train[output_categories].std().values, dtype=torch.float32).to(device)\n",
    "LABEL_WEIGHTS = LABEL_WEIGHTS / LABEL_WEIGHTS.sum() * 2\n",
    "for name, weight in zip(output_categories, LABEL_WEIGHTS.cpu().numpy()):\n",
    "    print(name, \"\\t\", weight)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Fold(object):\n",
    "    def __init__(self, n_splits=5, shuffle=True, random_state=47):\n",
    "        self.n_splits = n_splits\n",
    "        self.shuffle = shuffle\n",
    "        self.random_state = random_state\n",
    "\n",
    "    def get_groupkfold(self, train, group_name):\n",
    "        group = train[group_name]\n",
    "        unique_group = group.unique()\n",
    "        kf = KFold(\n",
    "            n_splits=self.n_splits,\n",
    "            shuffle=self.shuffle,\n",
    "            random_state=self.random_state\n",
    "        )\n",
    "        \n",
    "        folds_ids = []\n",
    "        for trn_group_idx, val_group_idx in kf.split(unique_group):\n",
    "            trn_group = unique_group[trn_group_idx]\n",
    "            val_group = unique_group[val_group_idx]\n",
    "            is_trn = group.isin(trn_group)\n",
    "            is_val = group.isin(val_group)\n",
    "            trn_idx = train[is_trn].index \n",
    "            val_idx = train[is_val].index\n",
    "            folds_ids.append((trn_idx, val_idx))\n",
    "\n",
    "        return folds_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "gkf = Fold(n_splits=3, shuffle=True, random_state=47)\n",
    "fold_ids = gkf.get_groupkfold(df_train, group_name=\"subreddit\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "distilbert_tokenizer = DistilBertTokenizer(vocab_file = '../models/bert/distilbert-transformers-pytorch/bert-base-uncased-vocab.txt')\n",
    "outputs = torch.tensor(compute_output_arrays(df_train), dtype=torch.float)\n",
    "inputs = [torch.tensor(x, dtype=torch.long) for x in compute_input_arrays(df_train, distilbert_tokenizer)]\n",
    "test_inputs = [torch.tensor(x, dtype=torch.long) for x in compute_input_arrays(df_test, distilbert_tokenizer)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_dataset = torch.utils.data.TensorDataset(*test_inputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[0., 0.],\n",
      "        [1., 0.],\n",
      "        [0., 0.],\n",
      "        ...,\n",
      "        [1., 0.],\n",
      "        [1., 0.],\n",
      "        [0., 0.]])\n"
     ]
    }
   ],
   "source": [
    "print(outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/733 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bert.embeddings.word_embeddings.weight\n",
      "bert.embeddings.position_embeddings.weight\n",
      "bert.embeddings.LayerNorm.weight\n",
      "bert.embeddings.LayerNorm.bias\n",
      "bert.transformer.layer.0.attention.q_lin.weight\n",
      "bert.transformer.layer.0.attention.q_lin.bias\n",
      "bert.transformer.layer.0.attention.k_lin.weight\n",
      "bert.transformer.layer.0.attention.k_lin.bias\n",
      "bert.transformer.layer.0.attention.v_lin.weight\n",
      "bert.transformer.layer.0.attention.v_lin.bias\n",
      "bert.transformer.layer.0.attention.out_lin.weight\n",
      "bert.transformer.layer.0.attention.out_lin.bias\n",
      "bert.transformer.layer.0.sa_layer_norm.weight\n",
      "bert.transformer.layer.0.sa_layer_norm.bias\n",
      "bert.transformer.layer.0.ffn.lin1.weight\n",
      "bert.transformer.layer.0.ffn.lin1.bias\n",
      "bert.transformer.layer.0.ffn.lin2.weight\n",
      "bert.transformer.layer.0.ffn.lin2.bias\n",
      "bert.transformer.layer.0.output_layer_norm.weight\n",
      "bert.transformer.layer.0.output_layer_norm.bias\n",
      "bert.transformer.layer.1.attention.q_lin.weight\n",
      "bert.transformer.layer.1.attention.q_lin.bias\n",
      "bert.transformer.layer.1.attention.k_lin.weight\n",
      "bert.transformer.layer.1.attention.k_lin.bias\n",
      "bert.transformer.layer.1.attention.v_lin.weight\n",
      "bert.transformer.layer.1.attention.v_lin.bias\n",
      "bert.transformer.layer.1.attention.out_lin.weight\n",
      "bert.transformer.layer.1.attention.out_lin.bias\n",
      "bert.transformer.layer.1.sa_layer_norm.weight\n",
      "bert.transformer.layer.1.sa_layer_norm.bias\n",
      "bert.transformer.layer.1.ffn.lin1.weight\n",
      "bert.transformer.layer.1.ffn.lin1.bias\n",
      "bert.transformer.layer.1.ffn.lin2.weight\n",
      "bert.transformer.layer.1.ffn.lin2.bias\n",
      "bert.transformer.layer.1.output_layer_norm.weight\n",
      "bert.transformer.layer.1.output_layer_norm.bias\n",
      "bert.transformer.layer.2.attention.q_lin.weight\n",
      "bert.transformer.layer.2.attention.q_lin.bias\n",
      "bert.transformer.layer.2.attention.k_lin.weight\n",
      "bert.transformer.layer.2.attention.k_lin.bias\n",
      "bert.transformer.layer.2.attention.v_lin.weight\n",
      "bert.transformer.layer.2.attention.v_lin.bias\n",
      "bert.transformer.layer.2.attention.out_lin.weight\n",
      "bert.transformer.layer.2.attention.out_lin.bias\n",
      "bert.transformer.layer.2.sa_layer_norm.weight\n",
      "bert.transformer.layer.2.sa_layer_norm.bias\n",
      "bert.transformer.layer.2.ffn.lin1.weight\n",
      "bert.transformer.layer.2.ffn.lin1.bias\n",
      "bert.transformer.layer.2.ffn.lin2.weight\n",
      "bert.transformer.layer.2.ffn.lin2.bias\n",
      "bert.transformer.layer.2.output_layer_norm.weight\n",
      "bert.transformer.layer.2.output_layer_norm.bias\n",
      "bert.transformer.layer.3.attention.q_lin.weight\n",
      "bert.transformer.layer.3.attention.q_lin.bias\n",
      "bert.transformer.layer.3.attention.k_lin.weight\n",
      "bert.transformer.layer.3.attention.k_lin.bias\n",
      "bert.transformer.layer.3.attention.v_lin.weight\n",
      "bert.transformer.layer.3.attention.v_lin.bias\n",
      "bert.transformer.layer.3.attention.out_lin.weight\n",
      "bert.transformer.layer.3.attention.out_lin.bias\n",
      "bert.transformer.layer.3.sa_layer_norm.weight\n",
      "bert.transformer.layer.3.sa_layer_norm.bias\n",
      "bert.transformer.layer.3.ffn.lin1.weight\n",
      "bert.transformer.layer.3.ffn.lin1.bias\n",
      "bert.transformer.layer.3.ffn.lin2.weight\n",
      "bert.transformer.layer.3.ffn.lin2.bias\n",
      "bert.transformer.layer.3.output_layer_norm.weight\n",
      "bert.transformer.layer.3.output_layer_norm.bias\n",
      "bert.transformer.layer.4.attention.q_lin.weight\n",
      "bert.transformer.layer.4.attention.q_lin.bias\n",
      "bert.transformer.layer.4.attention.k_lin.weight\n",
      "bert.transformer.layer.4.attention.k_lin.bias\n",
      "bert.transformer.layer.4.attention.v_lin.weight\n",
      "bert.transformer.layer.4.attention.v_lin.bias\n",
      "bert.transformer.layer.4.attention.out_lin.weight\n",
      "bert.transformer.layer.4.attention.out_lin.bias\n",
      "bert.transformer.layer.4.sa_layer_norm.weight\n",
      "bert.transformer.layer.4.sa_layer_norm.bias\n",
      "bert.transformer.layer.4.ffn.lin1.weight\n",
      "bert.transformer.layer.4.ffn.lin1.bias\n",
      "bert.transformer.layer.4.ffn.lin2.weight\n",
      "bert.transformer.layer.4.ffn.lin2.bias\n",
      "bert.transformer.layer.4.output_layer_norm.weight\n",
      "bert.transformer.layer.4.output_layer_norm.bias\n",
      "bert.transformer.layer.5.attention.q_lin.weight\n",
      "bert.transformer.layer.5.attention.q_lin.bias\n",
      "bert.transformer.layer.5.attention.k_lin.weight\n",
      "bert.transformer.layer.5.attention.k_lin.bias\n",
      "bert.transformer.layer.5.attention.v_lin.weight\n",
      "bert.transformer.layer.5.attention.v_lin.bias\n",
      "bert.transformer.layer.5.attention.out_lin.weight\n",
      "bert.transformer.layer.5.attention.out_lin.bias\n",
      "bert.transformer.layer.5.sa_layer_norm.weight\n",
      "bert.transformer.layer.5.sa_layer_norm.bias\n",
      "bert.transformer.layer.5.ffn.lin1.weight\n",
      "bert.transformer.layer.5.ffn.lin1.bias\n",
      "bert.transformer.layer.5.ffn.lin2.weight\n",
      "bert.transformer.layer.5.ffn.lin2.bias\n",
      "bert.transformer.layer.5.output_layer_norm.weight\n",
      "bert.transformer.layer.5.output_layer_norm.bias\n",
      "cls_token_head.1.weight\n",
      "cls_token_head.1.bias\n",
      "classifier.weight\n",
      "classifier.bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 733/733 [03:02<00:00,  4.02it/s]\n",
      "100%|██████████| 610/610 [00:47<00:00, 12.97it/s]\n",
      "100%|██████████| 336/336 [00:25<00:00, 13.06it/s]\n",
      "  0%|          | 0/733 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1: Train Loss 0.2397104445628893, Valid Loss 0.1606440605839989\n",
      "\t Train Spearmanr 0.4958, Valid Spearmanr (avg) 0.5363, Valid Spearmanr (last) 0.5363\n",
      "\t elapsed: 255.2367696762085s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 733/733 [03:02<00:00,  4.02it/s]\n",
      "100%|██████████| 610/610 [00:47<00:00, 12.97it/s]\n",
      "100%|██████████| 336/336 [00:25<00:00, 13.04it/s]\n",
      "  0%|          | 0/733 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 2: Train Loss 0.17698884611418228, Valid Loss 0.16428467410570774\n",
      "\t Train Spearmanr 0.5715, Valid Spearmanr (avg) 0.5351, Valid Spearmanr (last) 0.5225\n",
      "\t elapsed: 255.21331429481506s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 733/733 [03:02<00:00,  4.02it/s]\n",
      "100%|██████████| 610/610 [00:47<00:00, 12.97it/s]\n",
      "100%|██████████| 336/336 [00:25<00:00, 13.04it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 3: Train Loss 0.11605727733722741, Valid Loss 0.2062991289495391\n",
      "\t Train Spearmanr 0.6506, Valid Spearmanr (avg) 0.5355, Valid Spearmanr (last) 0.5295\n",
      "\t elapsed: 255.19935059547424s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/1032 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bert.embeddings.word_embeddings.weight\n",
      "bert.embeddings.position_embeddings.weight\n",
      "bert.embeddings.LayerNorm.weight\n",
      "bert.embeddings.LayerNorm.bias\n",
      "bert.transformer.layer.0.attention.q_lin.weight\n",
      "bert.transformer.layer.0.attention.q_lin.bias\n",
      "bert.transformer.layer.0.attention.k_lin.weight\n",
      "bert.transformer.layer.0.attention.k_lin.bias\n",
      "bert.transformer.layer.0.attention.v_lin.weight\n",
      "bert.transformer.layer.0.attention.v_lin.bias\n",
      "bert.transformer.layer.0.attention.out_lin.weight\n",
      "bert.transformer.layer.0.attention.out_lin.bias\n",
      "bert.transformer.layer.0.sa_layer_norm.weight\n",
      "bert.transformer.layer.0.sa_layer_norm.bias\n",
      "bert.transformer.layer.0.ffn.lin1.weight\n",
      "bert.transformer.layer.0.ffn.lin1.bias\n",
      "bert.transformer.layer.0.ffn.lin2.weight\n",
      "bert.transformer.layer.0.ffn.lin2.bias\n",
      "bert.transformer.layer.0.output_layer_norm.weight\n",
      "bert.transformer.layer.0.output_layer_norm.bias\n",
      "bert.transformer.layer.1.attention.q_lin.weight\n",
      "bert.transformer.layer.1.attention.q_lin.bias\n",
      "bert.transformer.layer.1.attention.k_lin.weight\n",
      "bert.transformer.layer.1.attention.k_lin.bias\n",
      "bert.transformer.layer.1.attention.v_lin.weight\n",
      "bert.transformer.layer.1.attention.v_lin.bias\n",
      "bert.transformer.layer.1.attention.out_lin.weight\n",
      "bert.transformer.layer.1.attention.out_lin.bias\n",
      "bert.transformer.layer.1.sa_layer_norm.weight\n",
      "bert.transformer.layer.1.sa_layer_norm.bias\n",
      "bert.transformer.layer.1.ffn.lin1.weight\n",
      "bert.transformer.layer.1.ffn.lin1.bias\n",
      "bert.transformer.layer.1.ffn.lin2.weight\n",
      "bert.transformer.layer.1.ffn.lin2.bias\n",
      "bert.transformer.layer.1.output_layer_norm.weight\n",
      "bert.transformer.layer.1.output_layer_norm.bias\n",
      "bert.transformer.layer.2.attention.q_lin.weight\n",
      "bert.transformer.layer.2.attention.q_lin.bias\n",
      "bert.transformer.layer.2.attention.k_lin.weight\n",
      "bert.transformer.layer.2.attention.k_lin.bias\n",
      "bert.transformer.layer.2.attention.v_lin.weight\n",
      "bert.transformer.layer.2.attention.v_lin.bias\n",
      "bert.transformer.layer.2.attention.out_lin.weight\n",
      "bert.transformer.layer.2.attention.out_lin.bias\n",
      "bert.transformer.layer.2.sa_layer_norm.weight\n",
      "bert.transformer.layer.2.sa_layer_norm.bias\n",
      "bert.transformer.layer.2.ffn.lin1.weight\n",
      "bert.transformer.layer.2.ffn.lin1.bias\n",
      "bert.transformer.layer.2.ffn.lin2.weight\n",
      "bert.transformer.layer.2.ffn.lin2.bias\n",
      "bert.transformer.layer.2.output_layer_norm.weight\n",
      "bert.transformer.layer.2.output_layer_norm.bias\n",
      "bert.transformer.layer.3.attention.q_lin.weight\n",
      "bert.transformer.layer.3.attention.q_lin.bias\n",
      "bert.transformer.layer.3.attention.k_lin.weight\n",
      "bert.transformer.layer.3.attention.k_lin.bias\n",
      "bert.transformer.layer.3.attention.v_lin.weight\n",
      "bert.transformer.layer.3.attention.v_lin.bias\n",
      "bert.transformer.layer.3.attention.out_lin.weight\n",
      "bert.transformer.layer.3.attention.out_lin.bias\n",
      "bert.transformer.layer.3.sa_layer_norm.weight\n",
      "bert.transformer.layer.3.sa_layer_norm.bias\n",
      "bert.transformer.layer.3.ffn.lin1.weight\n",
      "bert.transformer.layer.3.ffn.lin1.bias\n",
      "bert.transformer.layer.3.ffn.lin2.weight\n",
      "bert.transformer.layer.3.ffn.lin2.bias\n",
      "bert.transformer.layer.3.output_layer_norm.weight\n",
      "bert.transformer.layer.3.output_layer_norm.bias\n",
      "bert.transformer.layer.4.attention.q_lin.weight\n",
      "bert.transformer.layer.4.attention.q_lin.bias\n",
      "bert.transformer.layer.4.attention.k_lin.weight\n",
      "bert.transformer.layer.4.attention.k_lin.bias\n",
      "bert.transformer.layer.4.attention.v_lin.weight\n",
      "bert.transformer.layer.4.attention.v_lin.bias\n",
      "bert.transformer.layer.4.attention.out_lin.weight\n",
      "bert.transformer.layer.4.attention.out_lin.bias\n",
      "bert.transformer.layer.4.sa_layer_norm.weight\n",
      "bert.transformer.layer.4.sa_layer_norm.bias\n",
      "bert.transformer.layer.4.ffn.lin1.weight\n",
      "bert.transformer.layer.4.ffn.lin1.bias\n",
      "bert.transformer.layer.4.ffn.lin2.weight\n",
      "bert.transformer.layer.4.ffn.lin2.bias\n",
      "bert.transformer.layer.4.output_layer_norm.weight\n",
      "bert.transformer.layer.4.output_layer_norm.bias\n",
      "bert.transformer.layer.5.attention.q_lin.weight\n",
      "bert.transformer.layer.5.attention.q_lin.bias\n",
      "bert.transformer.layer.5.attention.k_lin.weight\n",
      "bert.transformer.layer.5.attention.k_lin.bias\n",
      "bert.transformer.layer.5.attention.v_lin.weight\n",
      "bert.transformer.layer.5.attention.v_lin.bias\n",
      "bert.transformer.layer.5.attention.out_lin.weight\n",
      "bert.transformer.layer.5.attention.out_lin.bias\n",
      "bert.transformer.layer.5.sa_layer_norm.weight\n",
      "bert.transformer.layer.5.sa_layer_norm.bias\n",
      "bert.transformer.layer.5.ffn.lin1.weight\n",
      "bert.transformer.layer.5.ffn.lin1.bias\n",
      "bert.transformer.layer.5.ffn.lin2.weight\n",
      "bert.transformer.layer.5.ffn.lin2.bias\n",
      "bert.transformer.layer.5.output_layer_norm.weight\n",
      "bert.transformer.layer.5.output_layer_norm.bias\n",
      "cls_token_head.1.weight\n",
      "cls_token_head.1.bias\n",
      "classifier.weight\n",
      "classifier.bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1032/1032 [04:16<00:00,  4.02it/s]\n",
      "100%|██████████| 311/311 [00:23<00:00, 12.97it/s]\n",
      "100%|██████████| 336/336 [00:25<00:00, 13.05it/s]\n",
      "  0%|          | 0/1032 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1: Train Loss 0.2115722319351278, Valid Loss 0.2039628719425853\n",
      "\t Train Spearmanr 0.5178, Valid Spearmanr (avg) 0.5307, Valid Spearmanr (last) 0.5307\n",
      "\t elapsed: 306.4821012020111s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1032/1032 [04:16<00:00,  4.02it/s]\n",
      "100%|██████████| 311/311 [00:23<00:00, 12.98it/s]\n",
      "100%|██████████| 336/336 [00:25<00:00, 13.04it/s]\n",
      "  0%|          | 0/1032 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 2: Train Loss 0.1612495779873832, Valid Loss 0.17605194478172964\n",
      "\t Train Spearmanr 0.5756, Valid Spearmanr (avg) 0.5343, Valid Spearmanr (last) 0.5293\n",
      "\t elapsed: 306.6195857524872s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1032/1032 [04:16<00:00,  4.02it/s]\n",
      "100%|██████████| 311/311 [00:23<00:00, 12.99it/s]\n",
      "100%|██████████| 336/336 [00:25<00:00, 13.05it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 3: Train Loss 0.09624801508081958, Valid Loss 0.22780874778345897\n",
      "\t Train Spearmanr 0.6498, Valid Spearmanr (avg) 0.5309, Valid Spearmanr (last) 0.5141\n",
      "\t elapsed: 306.52532935142517s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/921 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bert.embeddings.word_embeddings.weight\n",
      "bert.embeddings.position_embeddings.weight\n",
      "bert.embeddings.LayerNorm.weight\n",
      "bert.embeddings.LayerNorm.bias\n",
      "bert.transformer.layer.0.attention.q_lin.weight\n",
      "bert.transformer.layer.0.attention.q_lin.bias\n",
      "bert.transformer.layer.0.attention.k_lin.weight\n",
      "bert.transformer.layer.0.attention.k_lin.bias\n",
      "bert.transformer.layer.0.attention.v_lin.weight\n",
      "bert.transformer.layer.0.attention.v_lin.bias\n",
      "bert.transformer.layer.0.attention.out_lin.weight\n",
      "bert.transformer.layer.0.attention.out_lin.bias\n",
      "bert.transformer.layer.0.sa_layer_norm.weight\n",
      "bert.transformer.layer.0.sa_layer_norm.bias\n",
      "bert.transformer.layer.0.ffn.lin1.weight\n",
      "bert.transformer.layer.0.ffn.lin1.bias\n",
      "bert.transformer.layer.0.ffn.lin2.weight\n",
      "bert.transformer.layer.0.ffn.lin2.bias\n",
      "bert.transformer.layer.0.output_layer_norm.weight\n",
      "bert.transformer.layer.0.output_layer_norm.bias\n",
      "bert.transformer.layer.1.attention.q_lin.weight\n",
      "bert.transformer.layer.1.attention.q_lin.bias\n",
      "bert.transformer.layer.1.attention.k_lin.weight\n",
      "bert.transformer.layer.1.attention.k_lin.bias\n",
      "bert.transformer.layer.1.attention.v_lin.weight\n",
      "bert.transformer.layer.1.attention.v_lin.bias\n",
      "bert.transformer.layer.1.attention.out_lin.weight\n",
      "bert.transformer.layer.1.attention.out_lin.bias\n",
      "bert.transformer.layer.1.sa_layer_norm.weight\n",
      "bert.transformer.layer.1.sa_layer_norm.bias\n",
      "bert.transformer.layer.1.ffn.lin1.weight\n",
      "bert.transformer.layer.1.ffn.lin1.bias\n",
      "bert.transformer.layer.1.ffn.lin2.weight\n",
      "bert.transformer.layer.1.ffn.lin2.bias\n",
      "bert.transformer.layer.1.output_layer_norm.weight\n",
      "bert.transformer.layer.1.output_layer_norm.bias\n",
      "bert.transformer.layer.2.attention.q_lin.weight\n",
      "bert.transformer.layer.2.attention.q_lin.bias\n",
      "bert.transformer.layer.2.attention.k_lin.weight\n",
      "bert.transformer.layer.2.attention.k_lin.bias\n",
      "bert.transformer.layer.2.attention.v_lin.weight\n",
      "bert.transformer.layer.2.attention.v_lin.bias\n",
      "bert.transformer.layer.2.attention.out_lin.weight\n",
      "bert.transformer.layer.2.attention.out_lin.bias\n",
      "bert.transformer.layer.2.sa_layer_norm.weight\n",
      "bert.transformer.layer.2.sa_layer_norm.bias\n",
      "bert.transformer.layer.2.ffn.lin1.weight\n",
      "bert.transformer.layer.2.ffn.lin1.bias\n",
      "bert.transformer.layer.2.ffn.lin2.weight\n",
      "bert.transformer.layer.2.ffn.lin2.bias\n",
      "bert.transformer.layer.2.output_layer_norm.weight\n",
      "bert.transformer.layer.2.output_layer_norm.bias\n",
      "bert.transformer.layer.3.attention.q_lin.weight\n",
      "bert.transformer.layer.3.attention.q_lin.bias\n",
      "bert.transformer.layer.3.attention.k_lin.weight\n",
      "bert.transformer.layer.3.attention.k_lin.bias\n",
      "bert.transformer.layer.3.attention.v_lin.weight\n",
      "bert.transformer.layer.3.attention.v_lin.bias\n",
      "bert.transformer.layer.3.attention.out_lin.weight\n",
      "bert.transformer.layer.3.attention.out_lin.bias\n",
      "bert.transformer.layer.3.sa_layer_norm.weight\n",
      "bert.transformer.layer.3.sa_layer_norm.bias\n",
      "bert.transformer.layer.3.ffn.lin1.weight\n",
      "bert.transformer.layer.3.ffn.lin1.bias\n",
      "bert.transformer.layer.3.ffn.lin2.weight\n",
      "bert.transformer.layer.3.ffn.lin2.bias\n",
      "bert.transformer.layer.3.output_layer_norm.weight\n",
      "bert.transformer.layer.3.output_layer_norm.bias\n",
      "bert.transformer.layer.4.attention.q_lin.weight\n",
      "bert.transformer.layer.4.attention.q_lin.bias\n",
      "bert.transformer.layer.4.attention.k_lin.weight\n",
      "bert.transformer.layer.4.attention.k_lin.bias\n",
      "bert.transformer.layer.4.attention.v_lin.weight\n",
      "bert.transformer.layer.4.attention.v_lin.bias\n",
      "bert.transformer.layer.4.attention.out_lin.weight\n",
      "bert.transformer.layer.4.attention.out_lin.bias\n",
      "bert.transformer.layer.4.sa_layer_norm.weight\n",
      "bert.transformer.layer.4.sa_layer_norm.bias\n",
      "bert.transformer.layer.4.ffn.lin1.weight\n",
      "bert.transformer.layer.4.ffn.lin1.bias\n",
      "bert.transformer.layer.4.ffn.lin2.weight\n",
      "bert.transformer.layer.4.ffn.lin2.bias\n",
      "bert.transformer.layer.4.output_layer_norm.weight\n",
      "bert.transformer.layer.4.output_layer_norm.bias\n",
      "bert.transformer.layer.5.attention.q_lin.weight\n",
      "bert.transformer.layer.5.attention.q_lin.bias\n",
      "bert.transformer.layer.5.attention.k_lin.weight\n",
      "bert.transformer.layer.5.attention.k_lin.bias\n",
      "bert.transformer.layer.5.attention.v_lin.weight\n",
      "bert.transformer.layer.5.attention.v_lin.bias\n",
      "bert.transformer.layer.5.attention.out_lin.weight\n",
      "bert.transformer.layer.5.attention.out_lin.bias\n",
      "bert.transformer.layer.5.sa_layer_norm.weight\n",
      "bert.transformer.layer.5.sa_layer_norm.bias\n",
      "bert.transformer.layer.5.ffn.lin1.weight\n",
      "bert.transformer.layer.5.ffn.lin1.bias\n",
      "bert.transformer.layer.5.ffn.lin2.weight\n",
      "bert.transformer.layer.5.ffn.lin2.bias\n",
      "bert.transformer.layer.5.output_layer_norm.weight\n",
      "bert.transformer.layer.5.output_layer_norm.bias\n",
      "cls_token_head.1.weight\n",
      "cls_token_head.1.bias\n",
      "classifier.weight\n",
      "classifier.bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 921/921 [03:49<00:00,  4.02it/s]\n",
      "100%|██████████| 422/422 [00:32<00:00, 13.01it/s]\n",
      "100%|██████████| 336/336 [00:25<00:00, 13.05it/s]\n",
      "  0%|          | 0/921 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1: Train Loss 0.20327538275134616, Valid Loss 0.23134682852536512\n",
      "\t Train Spearmanr 0.5089, Valid Spearmanr (avg) 0.5313, Valid Spearmanr (last) 0.5313\n",
      "\t elapsed: 287.4165608882904s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 921/921 [03:49<00:00,  4.02it/s]\n",
      "100%|██████████| 422/422 [00:32<00:00, 12.99it/s]\n",
      "100%|██████████| 336/336 [00:25<00:00, 13.03it/s]\n",
      "  0%|          | 0/921 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 2: Train Loss 0.1538048578819602, Valid Loss 0.2040229191014905\n",
      "\t Train Spearmanr 0.5662, Valid Spearmanr (avg) 0.5429, Valid Spearmanr (last) 0.5442\n",
      "\t elapsed: 287.56513595581055s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 921/921 [03:49<00:00,  4.02it/s]\n",
      "100%|██████████| 422/422 [00:32<00:00, 12.99it/s]\n",
      "100%|██████████| 336/336 [00:25<00:00, 13.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 3: Train Loss 0.09910993997855001, Valid Loss 0.2615735959372002\n",
      "\t Train Spearmanr 0.6359, Valid Spearmanr (avg) 0.5450, Valid Spearmanr (last) 0.5488\n",
      "\t elapsed: 287.51194524765015s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "histories = []\n",
    "\n",
    "for fold, (train_idx, valid_idx) in enumerate(fold_ids):\n",
    "    import gc\n",
    "    gc.collect()\n",
    "    \n",
    "    train_inputs = [inputs[i][train_idx] for i in range(3)]\n",
    "    train_outputs = outputs[train_idx]\n",
    "    train_dataset = torch.utils.data.TensorDataset(*train_inputs, train_outputs)\n",
    "\n",
    "    valid_inputs = [inputs[i][valid_idx] for i in range(3)]\n",
    "    valid_outputs = outputs[valid_idx]\n",
    "    valid_dataset = torch.utils.data.TensorDataset(*valid_inputs, valid_outputs)\n",
    "   \n",
    "    history = train_and_predict(train_data=train_dataset, valid_data=valid_dataset, test_data=test_dataset, \n",
    "        epochs=3, batch_size=8, model_type = 'distilbert', device = device, fold=fold)\n",
    "\n",
    "    histories.append(history)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_preds_list = []\n",
    "n_epochs = len(histories[0][0])\n",
    "\n",
    "for epoch in range(n_epochs):\n",
    "    val_preds_one_epoch = np.zeros([len(df_train), 2])    \n",
    "\n",
    "    for fold, (train_idx, valid_idx) in enumerate(fold_ids):\n",
    "        val_pred = histories[fold][0][epoch]\n",
    "        val_preds_one_epoch[valid_idx, :] += val_pred\n",
    "\n",
    "    val_preds_list.append(val_preds_one_epoch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_preds_list = []\n",
    "\n",
    "#for epoch in range(n_epochs):\n",
    "#    test_preds_one_epoch = 0\n",
    "\n",
    "#    for fold in range(len(fold_ids)):\n",
    "#        test_preds = histories[fold][1][epoch]\n",
    "#        test_preds_one_epoch += test_preds\n",
    "\n",
    "#    test_preds_one_epoch = test_preds_one_epoch / len(fold_ids)\n",
    "#    test_preds_list.append(test_preds_one_epoch)\n",
    "\n",
    "\n",
    "for epoch in range(n_epochs):\n",
    "    test_preds_one_epoch = 0\n",
    "\n",
    "    for fold in range(len(fold_ids)):\n",
    "        test_preds = histories[fold][1][epoch]\n",
    "        test_preds_one_epoch += test_preds\n",
    "\n",
    "    test_preds_one_epoch = test_preds_one_epoch / len(fold_ids)\n",
    "    test_preds_list.append(test_preds_one_epoch)\n",
    "\n",
    "test_predictions_distil = np.zeros((n_epochs, len(df_test), len(output_categories)), dtype=np.float32)\n",
    "\n",
    "for j, name in enumerate(output_categories):\n",
    "    for epoch in range(n_epochs):\n",
    "        col = \"{}_{}\".format(epoch, name)\n",
    "        test_predictions_distil[epoch, :, j] = test_preds_list[epoch][:, j]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_preds_distil = test_preds_list[n_epochs - 1][:, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2685, 2)"
      ]
     },
     "execution_count": 175,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_preds_distil.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DescribeResult(nobs=2685, minmax=(7.356166e-08, 1.0), mean=0.49959177, variance=0.24903183, skewness=0.002104994375258684, kurtosis=-1.9977449003094976)"
      ]
     },
     "execution_count": 176,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stats.describe(test_preds_distil[:,0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DescribeResult(nobs=2685, minmax=(3.5281304e-05, 0.8151059), mean=0.07728863, variance=0.0191027, skewness=2.6136560440063477, kurtosis=7.183695039175532)"
      ]
     },
     "execution_count": 177,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stats.describe(test_preds_distil[:,1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_test_troll = np.asarray(df_test['troll'])\n",
    "y_test_contro = np.asarray(df_test['controversiality'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00      1345\n",
      "           1       1.00      1.00      1.00      1340\n",
      "\n",
      "    accuracy                           1.00      2685\n",
      "   macro avg       1.00      1.00      1.00      2685\n",
      "weighted avg       1.00      1.00      1.00      2685\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_test_troll, np.around(test_preds_distil[:,0]).astype(np.int32)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.92      0.98      0.95      2443\n",
      "           1       0.33      0.10      0.16       242\n",
      "\n",
      "    accuracy                           0.90      2685\n",
      "   macro avg       0.63      0.54      0.55      2685\n",
      "weighted avg       0.86      0.90      0.88      2685\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_test_contro, np.around(test_preds_distil[:,1]).astype(np.int32)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {},
   "outputs": [],
   "source": [
    "fpr_troll, tpr_troll, thresholds_troll = roc_curve(y_test_troll, np.around(test_preds_distil[:,0]).astype(np.int32))\n",
    "fpr_contro, tpr_contro, thresholds_contro = roc_curve(y_test_contro, np.around(test_preds_distil[:,1]).astype(np.int32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9996282527881041 0.5414195728730764\n"
     ]
    }
   ],
   "source": [
    "auc_troll = auc(fpr_troll, tpr_troll)\n",
    "auc_contro = auc(fpr_contro, tpr_contro)\n",
    "print(auc_troll, auc_contro)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-1"
      ]
     },
     "execution_count": 183,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = y_test_troll - np.around(test_preds_distil[:,0]).astype(np.int32)\n",
    "x.sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9996282527881041 0.5414195728730764\n"
     ]
    }
   ],
   "source": [
    "ba_troll = balanced_accuracy_score(y_test_troll, np.around(test_preds_distil[:,0]).astype(np.int32))\n",
    "ba_contro = balanced_accuracy_score(y_test_contro, np.around(test_preds_distil[:,1]).astype(np.int32))\n",
    "print(ba_troll, ba_contro)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "bert_tokenizer = BertTokenizer(vocab_file = '../models/bert/bert-base/bert-base-uncased-vocab.txt')\n",
    "outputs = torch.tensor(compute_output_arrays(df_train), dtype=torch.float)\n",
    "inputs = [torch.tensor(x, dtype=torch.long) for x in compute_input_arrays(df_train, bert_tokenizer)]\n",
    "test_inputs = [torch.tensor(x, dtype=torch.long) for x in compute_input_arrays(df_test, bert_tokenizer)]\n",
    "test_dataset = torch.utils.data.TensorDataset(*test_inputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/733 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bert.embeddings.word_embeddings.weight\n",
      "bert.embeddings.position_embeddings.weight\n",
      "bert.embeddings.token_type_embeddings.weight\n",
      "bert.embeddings.LayerNorm.weight\n",
      "bert.embeddings.LayerNorm.bias\n",
      "bert.encoder.layer.0.attention.self.query.weight\n",
      "bert.encoder.layer.0.attention.self.query.bias\n",
      "bert.encoder.layer.0.attention.self.key.weight\n",
      "bert.encoder.layer.0.attention.self.key.bias\n",
      "bert.encoder.layer.0.attention.self.value.weight\n",
      "bert.encoder.layer.0.attention.self.value.bias\n",
      "bert.encoder.layer.0.attention.output.dense.weight\n",
      "bert.encoder.layer.0.attention.output.dense.bias\n",
      "bert.encoder.layer.0.attention.output.LayerNorm.weight\n",
      "bert.encoder.layer.0.attention.output.LayerNorm.bias\n",
      "bert.encoder.layer.0.intermediate.dense.weight\n",
      "bert.encoder.layer.0.intermediate.dense.bias\n",
      "bert.encoder.layer.0.output.dense.weight\n",
      "bert.encoder.layer.0.output.dense.bias\n",
      "bert.encoder.layer.0.output.LayerNorm.weight\n",
      "bert.encoder.layer.0.output.LayerNorm.bias\n",
      "bert.encoder.layer.1.attention.self.query.weight\n",
      "bert.encoder.layer.1.attention.self.query.bias\n",
      "bert.encoder.layer.1.attention.self.key.weight\n",
      "bert.encoder.layer.1.attention.self.key.bias\n",
      "bert.encoder.layer.1.attention.self.value.weight\n",
      "bert.encoder.layer.1.attention.self.value.bias\n",
      "bert.encoder.layer.1.attention.output.dense.weight\n",
      "bert.encoder.layer.1.attention.output.dense.bias\n",
      "bert.encoder.layer.1.attention.output.LayerNorm.weight\n",
      "bert.encoder.layer.1.attention.output.LayerNorm.bias\n",
      "bert.encoder.layer.1.intermediate.dense.weight\n",
      "bert.encoder.layer.1.intermediate.dense.bias\n",
      "bert.encoder.layer.1.output.dense.weight\n",
      "bert.encoder.layer.1.output.dense.bias\n",
      "bert.encoder.layer.1.output.LayerNorm.weight\n",
      "bert.encoder.layer.1.output.LayerNorm.bias\n",
      "bert.encoder.layer.2.attention.self.query.weight\n",
      "bert.encoder.layer.2.attention.self.query.bias\n",
      "bert.encoder.layer.2.attention.self.key.weight\n",
      "bert.encoder.layer.2.attention.self.key.bias\n",
      "bert.encoder.layer.2.attention.self.value.weight\n",
      "bert.encoder.layer.2.attention.self.value.bias\n",
      "bert.encoder.layer.2.attention.output.dense.weight\n",
      "bert.encoder.layer.2.attention.output.dense.bias\n",
      "bert.encoder.layer.2.attention.output.LayerNorm.weight\n",
      "bert.encoder.layer.2.attention.output.LayerNorm.bias\n",
      "bert.encoder.layer.2.intermediate.dense.weight\n",
      "bert.encoder.layer.2.intermediate.dense.bias\n",
      "bert.encoder.layer.2.output.dense.weight\n",
      "bert.encoder.layer.2.output.dense.bias\n",
      "bert.encoder.layer.2.output.LayerNorm.weight\n",
      "bert.encoder.layer.2.output.LayerNorm.bias\n",
      "bert.encoder.layer.3.attention.self.query.weight\n",
      "bert.encoder.layer.3.attention.self.query.bias\n",
      "bert.encoder.layer.3.attention.self.key.weight\n",
      "bert.encoder.layer.3.attention.self.key.bias\n",
      "bert.encoder.layer.3.attention.self.value.weight\n",
      "bert.encoder.layer.3.attention.self.value.bias\n",
      "bert.encoder.layer.3.attention.output.dense.weight\n",
      "bert.encoder.layer.3.attention.output.dense.bias\n",
      "bert.encoder.layer.3.attention.output.LayerNorm.weight\n",
      "bert.encoder.layer.3.attention.output.LayerNorm.bias\n",
      "bert.encoder.layer.3.intermediate.dense.weight\n",
      "bert.encoder.layer.3.intermediate.dense.bias\n",
      "bert.encoder.layer.3.output.dense.weight\n",
      "bert.encoder.layer.3.output.dense.bias\n",
      "bert.encoder.layer.3.output.LayerNorm.weight\n",
      "bert.encoder.layer.3.output.LayerNorm.bias\n",
      "bert.encoder.layer.4.attention.self.query.weight\n",
      "bert.encoder.layer.4.attention.self.query.bias\n",
      "bert.encoder.layer.4.attention.self.key.weight\n",
      "bert.encoder.layer.4.attention.self.key.bias\n",
      "bert.encoder.layer.4.attention.self.value.weight\n",
      "bert.encoder.layer.4.attention.self.value.bias\n",
      "bert.encoder.layer.4.attention.output.dense.weight\n",
      "bert.encoder.layer.4.attention.output.dense.bias\n",
      "bert.encoder.layer.4.attention.output.LayerNorm.weight\n",
      "bert.encoder.layer.4.attention.output.LayerNorm.bias\n",
      "bert.encoder.layer.4.intermediate.dense.weight\n",
      "bert.encoder.layer.4.intermediate.dense.bias\n",
      "bert.encoder.layer.4.output.dense.weight\n",
      "bert.encoder.layer.4.output.dense.bias\n",
      "bert.encoder.layer.4.output.LayerNorm.weight\n",
      "bert.encoder.layer.4.output.LayerNorm.bias\n",
      "bert.encoder.layer.5.attention.self.query.weight\n",
      "bert.encoder.layer.5.attention.self.query.bias\n",
      "bert.encoder.layer.5.attention.self.key.weight\n",
      "bert.encoder.layer.5.attention.self.key.bias\n",
      "bert.encoder.layer.5.attention.self.value.weight\n",
      "bert.encoder.layer.5.attention.self.value.bias\n",
      "bert.encoder.layer.5.attention.output.dense.weight\n",
      "bert.encoder.layer.5.attention.output.dense.bias\n",
      "bert.encoder.layer.5.attention.output.LayerNorm.weight\n",
      "bert.encoder.layer.5.attention.output.LayerNorm.bias\n",
      "bert.encoder.layer.5.intermediate.dense.weight\n",
      "bert.encoder.layer.5.intermediate.dense.bias\n",
      "bert.encoder.layer.5.output.dense.weight\n",
      "bert.encoder.layer.5.output.dense.bias\n",
      "bert.encoder.layer.5.output.LayerNorm.weight\n",
      "bert.encoder.layer.5.output.LayerNorm.bias\n",
      "bert.encoder.layer.6.attention.self.query.weight\n",
      "bert.encoder.layer.6.attention.self.query.bias\n",
      "bert.encoder.layer.6.attention.self.key.weight\n",
      "bert.encoder.layer.6.attention.self.key.bias\n",
      "bert.encoder.layer.6.attention.self.value.weight\n",
      "bert.encoder.layer.6.attention.self.value.bias\n",
      "bert.encoder.layer.6.attention.output.dense.weight\n",
      "bert.encoder.layer.6.attention.output.dense.bias\n",
      "bert.encoder.layer.6.attention.output.LayerNorm.weight\n",
      "bert.encoder.layer.6.attention.output.LayerNorm.bias\n",
      "bert.encoder.layer.6.intermediate.dense.weight\n",
      "bert.encoder.layer.6.intermediate.dense.bias\n",
      "bert.encoder.layer.6.output.dense.weight\n",
      "bert.encoder.layer.6.output.dense.bias\n",
      "bert.encoder.layer.6.output.LayerNorm.weight\n",
      "bert.encoder.layer.6.output.LayerNorm.bias\n",
      "bert.encoder.layer.7.attention.self.query.weight\n",
      "bert.encoder.layer.7.attention.self.query.bias\n",
      "bert.encoder.layer.7.attention.self.key.weight\n",
      "bert.encoder.layer.7.attention.self.key.bias\n",
      "bert.encoder.layer.7.attention.self.value.weight\n",
      "bert.encoder.layer.7.attention.self.value.bias\n",
      "bert.encoder.layer.7.attention.output.dense.weight\n",
      "bert.encoder.layer.7.attention.output.dense.bias\n",
      "bert.encoder.layer.7.attention.output.LayerNorm.weight\n",
      "bert.encoder.layer.7.attention.output.LayerNorm.bias\n",
      "bert.encoder.layer.7.intermediate.dense.weight\n",
      "bert.encoder.layer.7.intermediate.dense.bias\n",
      "bert.encoder.layer.7.output.dense.weight\n",
      "bert.encoder.layer.7.output.dense.bias\n",
      "bert.encoder.layer.7.output.LayerNorm.weight\n",
      "bert.encoder.layer.7.output.LayerNorm.bias\n",
      "bert.encoder.layer.8.attention.self.query.weight\n",
      "bert.encoder.layer.8.attention.self.query.bias\n",
      "bert.encoder.layer.8.attention.self.key.weight\n",
      "bert.encoder.layer.8.attention.self.key.bias\n",
      "bert.encoder.layer.8.attention.self.value.weight\n",
      "bert.encoder.layer.8.attention.self.value.bias\n",
      "bert.encoder.layer.8.attention.output.dense.weight\n",
      "bert.encoder.layer.8.attention.output.dense.bias\n",
      "bert.encoder.layer.8.attention.output.LayerNorm.weight\n",
      "bert.encoder.layer.8.attention.output.LayerNorm.bias\n",
      "bert.encoder.layer.8.intermediate.dense.weight\n",
      "bert.encoder.layer.8.intermediate.dense.bias\n",
      "bert.encoder.layer.8.output.dense.weight\n",
      "bert.encoder.layer.8.output.dense.bias\n",
      "bert.encoder.layer.8.output.LayerNorm.weight\n",
      "bert.encoder.layer.8.output.LayerNorm.bias\n",
      "bert.encoder.layer.9.attention.self.query.weight\n",
      "bert.encoder.layer.9.attention.self.query.bias\n",
      "bert.encoder.layer.9.attention.self.key.weight\n",
      "bert.encoder.layer.9.attention.self.key.bias\n",
      "bert.encoder.layer.9.attention.self.value.weight\n",
      "bert.encoder.layer.9.attention.self.value.bias\n",
      "bert.encoder.layer.9.attention.output.dense.weight\n",
      "bert.encoder.layer.9.attention.output.dense.bias\n",
      "bert.encoder.layer.9.attention.output.LayerNorm.weight\n",
      "bert.encoder.layer.9.attention.output.LayerNorm.bias\n",
      "bert.encoder.layer.9.intermediate.dense.weight\n",
      "bert.encoder.layer.9.intermediate.dense.bias\n",
      "bert.encoder.layer.9.output.dense.weight\n",
      "bert.encoder.layer.9.output.dense.bias\n",
      "bert.encoder.layer.9.output.LayerNorm.weight\n",
      "bert.encoder.layer.9.output.LayerNorm.bias\n",
      "bert.encoder.layer.10.attention.self.query.weight\n",
      "bert.encoder.layer.10.attention.self.query.bias\n",
      "bert.encoder.layer.10.attention.self.key.weight\n",
      "bert.encoder.layer.10.attention.self.key.bias\n",
      "bert.encoder.layer.10.attention.self.value.weight\n",
      "bert.encoder.layer.10.attention.self.value.bias\n",
      "bert.encoder.layer.10.attention.output.dense.weight\n",
      "bert.encoder.layer.10.attention.output.dense.bias\n",
      "bert.encoder.layer.10.attention.output.LayerNorm.weight\n",
      "bert.encoder.layer.10.attention.output.LayerNorm.bias\n",
      "bert.encoder.layer.10.intermediate.dense.weight\n",
      "bert.encoder.layer.10.intermediate.dense.bias\n",
      "bert.encoder.layer.10.output.dense.weight\n",
      "bert.encoder.layer.10.output.dense.bias\n",
      "bert.encoder.layer.10.output.LayerNorm.weight\n",
      "bert.encoder.layer.10.output.LayerNorm.bias\n",
      "bert.encoder.layer.11.attention.self.query.weight\n",
      "bert.encoder.layer.11.attention.self.query.bias\n",
      "bert.encoder.layer.11.attention.self.key.weight\n",
      "bert.encoder.layer.11.attention.self.key.bias\n",
      "bert.encoder.layer.11.attention.self.value.weight\n",
      "bert.encoder.layer.11.attention.self.value.bias\n",
      "bert.encoder.layer.11.attention.output.dense.weight\n",
      "bert.encoder.layer.11.attention.output.dense.bias\n",
      "bert.encoder.layer.11.attention.output.LayerNorm.weight\n",
      "bert.encoder.layer.11.attention.output.LayerNorm.bias\n",
      "bert.encoder.layer.11.intermediate.dense.weight\n",
      "bert.encoder.layer.11.intermediate.dense.bias\n",
      "bert.encoder.layer.11.output.dense.weight\n",
      "bert.encoder.layer.11.output.dense.bias\n",
      "bert.encoder.layer.11.output.LayerNorm.weight\n",
      "bert.encoder.layer.11.output.LayerNorm.bias\n",
      "bert.pooler.dense.weight\n",
      "bert.pooler.dense.bias\n",
      "cls_token_head.1.weight\n",
      "cls_token_head.1.bias\n",
      "classifier.weight\n",
      "classifier.bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 733/733 [05:51<00:00,  2.09it/s]\n",
      "100%|██████████| 610/610 [01:33<00:00,  6.55it/s]\n",
      "100%|██████████| 336/336 [00:51<00:00,  6.56it/s]\n",
      "  0%|          | 0/733 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1: Train Loss 0.25398403245544043, Valid Loss 0.1635389211725016\n",
      "\t Train Spearmanr 0.4851, Valid Spearmanr (avg) 0.5402, Valid Spearmanr (last) 0.5402\n",
      "\t elapsed: 495.59016609191895s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 733/733 [05:51<00:00,  2.09it/s]\n",
      "100%|██████████| 610/610 [01:33<00:00,  6.54it/s]\n",
      "100%|██████████| 336/336 [00:51<00:00,  6.56it/s]\n",
      "  0%|          | 0/733 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 2: Train Loss 0.18372701434609143, Valid Loss 0.15774763936451713\n",
      "\t Train Spearmanr 0.5645, Valid Spearmanr (avg) 0.5421, Valid Spearmanr (last) 0.5394\n",
      "\t elapsed: 495.6176085472107s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 733/733 [05:51<00:00,  2.09it/s]\n",
      "100%|██████████| 610/610 [01:33<00:00,  6.54it/s]\n",
      "100%|██████████| 336/336 [00:51<00:00,  6.56it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 3: Train Loss 0.11620855016734243, Valid Loss 0.1842280607876658\n",
      "\t Train Spearmanr 0.6470, Valid Spearmanr (avg) 0.5440, Valid Spearmanr (last) 0.5301\n",
      "\t elapsed: 495.6487867832184s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/1032 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bert.embeddings.word_embeddings.weight\n",
      "bert.embeddings.position_embeddings.weight\n",
      "bert.embeddings.token_type_embeddings.weight\n",
      "bert.embeddings.LayerNorm.weight\n",
      "bert.embeddings.LayerNorm.bias\n",
      "bert.encoder.layer.0.attention.self.query.weight\n",
      "bert.encoder.layer.0.attention.self.query.bias\n",
      "bert.encoder.layer.0.attention.self.key.weight\n",
      "bert.encoder.layer.0.attention.self.key.bias\n",
      "bert.encoder.layer.0.attention.self.value.weight\n",
      "bert.encoder.layer.0.attention.self.value.bias\n",
      "bert.encoder.layer.0.attention.output.dense.weight\n",
      "bert.encoder.layer.0.attention.output.dense.bias\n",
      "bert.encoder.layer.0.attention.output.LayerNorm.weight\n",
      "bert.encoder.layer.0.attention.output.LayerNorm.bias\n",
      "bert.encoder.layer.0.intermediate.dense.weight\n",
      "bert.encoder.layer.0.intermediate.dense.bias\n",
      "bert.encoder.layer.0.output.dense.weight\n",
      "bert.encoder.layer.0.output.dense.bias\n",
      "bert.encoder.layer.0.output.LayerNorm.weight\n",
      "bert.encoder.layer.0.output.LayerNorm.bias\n",
      "bert.encoder.layer.1.attention.self.query.weight\n",
      "bert.encoder.layer.1.attention.self.query.bias\n",
      "bert.encoder.layer.1.attention.self.key.weight\n",
      "bert.encoder.layer.1.attention.self.key.bias\n",
      "bert.encoder.layer.1.attention.self.value.weight\n",
      "bert.encoder.layer.1.attention.self.value.bias\n",
      "bert.encoder.layer.1.attention.output.dense.weight\n",
      "bert.encoder.layer.1.attention.output.dense.bias\n",
      "bert.encoder.layer.1.attention.output.LayerNorm.weight\n",
      "bert.encoder.layer.1.attention.output.LayerNorm.bias\n",
      "bert.encoder.layer.1.intermediate.dense.weight\n",
      "bert.encoder.layer.1.intermediate.dense.bias\n",
      "bert.encoder.layer.1.output.dense.weight\n",
      "bert.encoder.layer.1.output.dense.bias\n",
      "bert.encoder.layer.1.output.LayerNorm.weight\n",
      "bert.encoder.layer.1.output.LayerNorm.bias\n",
      "bert.encoder.layer.2.attention.self.query.weight\n",
      "bert.encoder.layer.2.attention.self.query.bias\n",
      "bert.encoder.layer.2.attention.self.key.weight\n",
      "bert.encoder.layer.2.attention.self.key.bias\n",
      "bert.encoder.layer.2.attention.self.value.weight\n",
      "bert.encoder.layer.2.attention.self.value.bias\n",
      "bert.encoder.layer.2.attention.output.dense.weight\n",
      "bert.encoder.layer.2.attention.output.dense.bias\n",
      "bert.encoder.layer.2.attention.output.LayerNorm.weight\n",
      "bert.encoder.layer.2.attention.output.LayerNorm.bias\n",
      "bert.encoder.layer.2.intermediate.dense.weight\n",
      "bert.encoder.layer.2.intermediate.dense.bias\n",
      "bert.encoder.layer.2.output.dense.weight\n",
      "bert.encoder.layer.2.output.dense.bias\n",
      "bert.encoder.layer.2.output.LayerNorm.weight\n",
      "bert.encoder.layer.2.output.LayerNorm.bias\n",
      "bert.encoder.layer.3.attention.self.query.weight\n",
      "bert.encoder.layer.3.attention.self.query.bias\n",
      "bert.encoder.layer.3.attention.self.key.weight\n",
      "bert.encoder.layer.3.attention.self.key.bias\n",
      "bert.encoder.layer.3.attention.self.value.weight\n",
      "bert.encoder.layer.3.attention.self.value.bias\n",
      "bert.encoder.layer.3.attention.output.dense.weight\n",
      "bert.encoder.layer.3.attention.output.dense.bias\n",
      "bert.encoder.layer.3.attention.output.LayerNorm.weight\n",
      "bert.encoder.layer.3.attention.output.LayerNorm.bias\n",
      "bert.encoder.layer.3.intermediate.dense.weight\n",
      "bert.encoder.layer.3.intermediate.dense.bias\n",
      "bert.encoder.layer.3.output.dense.weight\n",
      "bert.encoder.layer.3.output.dense.bias\n",
      "bert.encoder.layer.3.output.LayerNorm.weight\n",
      "bert.encoder.layer.3.output.LayerNorm.bias\n",
      "bert.encoder.layer.4.attention.self.query.weight\n",
      "bert.encoder.layer.4.attention.self.query.bias\n",
      "bert.encoder.layer.4.attention.self.key.weight\n",
      "bert.encoder.layer.4.attention.self.key.bias\n",
      "bert.encoder.layer.4.attention.self.value.weight\n",
      "bert.encoder.layer.4.attention.self.value.bias\n",
      "bert.encoder.layer.4.attention.output.dense.weight\n",
      "bert.encoder.layer.4.attention.output.dense.bias\n",
      "bert.encoder.layer.4.attention.output.LayerNorm.weight\n",
      "bert.encoder.layer.4.attention.output.LayerNorm.bias\n",
      "bert.encoder.layer.4.intermediate.dense.weight\n",
      "bert.encoder.layer.4.intermediate.dense.bias\n",
      "bert.encoder.layer.4.output.dense.weight\n",
      "bert.encoder.layer.4.output.dense.bias\n",
      "bert.encoder.layer.4.output.LayerNorm.weight\n",
      "bert.encoder.layer.4.output.LayerNorm.bias\n",
      "bert.encoder.layer.5.attention.self.query.weight\n",
      "bert.encoder.layer.5.attention.self.query.bias\n",
      "bert.encoder.layer.5.attention.self.key.weight\n",
      "bert.encoder.layer.5.attention.self.key.bias\n",
      "bert.encoder.layer.5.attention.self.value.weight\n",
      "bert.encoder.layer.5.attention.self.value.bias\n",
      "bert.encoder.layer.5.attention.output.dense.weight\n",
      "bert.encoder.layer.5.attention.output.dense.bias\n",
      "bert.encoder.layer.5.attention.output.LayerNorm.weight\n",
      "bert.encoder.layer.5.attention.output.LayerNorm.bias\n",
      "bert.encoder.layer.5.intermediate.dense.weight\n",
      "bert.encoder.layer.5.intermediate.dense.bias\n",
      "bert.encoder.layer.5.output.dense.weight\n",
      "bert.encoder.layer.5.output.dense.bias\n",
      "bert.encoder.layer.5.output.LayerNorm.weight\n",
      "bert.encoder.layer.5.output.LayerNorm.bias\n",
      "bert.encoder.layer.6.attention.self.query.weight\n",
      "bert.encoder.layer.6.attention.self.query.bias\n",
      "bert.encoder.layer.6.attention.self.key.weight\n",
      "bert.encoder.layer.6.attention.self.key.bias\n",
      "bert.encoder.layer.6.attention.self.value.weight\n",
      "bert.encoder.layer.6.attention.self.value.bias\n",
      "bert.encoder.layer.6.attention.output.dense.weight\n",
      "bert.encoder.layer.6.attention.output.dense.bias\n",
      "bert.encoder.layer.6.attention.output.LayerNorm.weight\n",
      "bert.encoder.layer.6.attention.output.LayerNorm.bias\n",
      "bert.encoder.layer.6.intermediate.dense.weight\n",
      "bert.encoder.layer.6.intermediate.dense.bias\n",
      "bert.encoder.layer.6.output.dense.weight\n",
      "bert.encoder.layer.6.output.dense.bias\n",
      "bert.encoder.layer.6.output.LayerNorm.weight\n",
      "bert.encoder.layer.6.output.LayerNorm.bias\n",
      "bert.encoder.layer.7.attention.self.query.weight\n",
      "bert.encoder.layer.7.attention.self.query.bias\n",
      "bert.encoder.layer.7.attention.self.key.weight\n",
      "bert.encoder.layer.7.attention.self.key.bias\n",
      "bert.encoder.layer.7.attention.self.value.weight\n",
      "bert.encoder.layer.7.attention.self.value.bias\n",
      "bert.encoder.layer.7.attention.output.dense.weight\n",
      "bert.encoder.layer.7.attention.output.dense.bias\n",
      "bert.encoder.layer.7.attention.output.LayerNorm.weight\n",
      "bert.encoder.layer.7.attention.output.LayerNorm.bias\n",
      "bert.encoder.layer.7.intermediate.dense.weight\n",
      "bert.encoder.layer.7.intermediate.dense.bias\n",
      "bert.encoder.layer.7.output.dense.weight\n",
      "bert.encoder.layer.7.output.dense.bias\n",
      "bert.encoder.layer.7.output.LayerNorm.weight\n",
      "bert.encoder.layer.7.output.LayerNorm.bias\n",
      "bert.encoder.layer.8.attention.self.query.weight\n",
      "bert.encoder.layer.8.attention.self.query.bias\n",
      "bert.encoder.layer.8.attention.self.key.weight\n",
      "bert.encoder.layer.8.attention.self.key.bias\n",
      "bert.encoder.layer.8.attention.self.value.weight\n",
      "bert.encoder.layer.8.attention.self.value.bias\n",
      "bert.encoder.layer.8.attention.output.dense.weight\n",
      "bert.encoder.layer.8.attention.output.dense.bias\n",
      "bert.encoder.layer.8.attention.output.LayerNorm.weight\n",
      "bert.encoder.layer.8.attention.output.LayerNorm.bias\n",
      "bert.encoder.layer.8.intermediate.dense.weight\n",
      "bert.encoder.layer.8.intermediate.dense.bias\n",
      "bert.encoder.layer.8.output.dense.weight\n",
      "bert.encoder.layer.8.output.dense.bias\n",
      "bert.encoder.layer.8.output.LayerNorm.weight\n",
      "bert.encoder.layer.8.output.LayerNorm.bias\n",
      "bert.encoder.layer.9.attention.self.query.weight\n",
      "bert.encoder.layer.9.attention.self.query.bias\n",
      "bert.encoder.layer.9.attention.self.key.weight\n",
      "bert.encoder.layer.9.attention.self.key.bias\n",
      "bert.encoder.layer.9.attention.self.value.weight\n",
      "bert.encoder.layer.9.attention.self.value.bias\n",
      "bert.encoder.layer.9.attention.output.dense.weight\n",
      "bert.encoder.layer.9.attention.output.dense.bias\n",
      "bert.encoder.layer.9.attention.output.LayerNorm.weight\n",
      "bert.encoder.layer.9.attention.output.LayerNorm.bias\n",
      "bert.encoder.layer.9.intermediate.dense.weight\n",
      "bert.encoder.layer.9.intermediate.dense.bias\n",
      "bert.encoder.layer.9.output.dense.weight\n",
      "bert.encoder.layer.9.output.dense.bias\n",
      "bert.encoder.layer.9.output.LayerNorm.weight\n",
      "bert.encoder.layer.9.output.LayerNorm.bias\n",
      "bert.encoder.layer.10.attention.self.query.weight\n",
      "bert.encoder.layer.10.attention.self.query.bias\n",
      "bert.encoder.layer.10.attention.self.key.weight\n",
      "bert.encoder.layer.10.attention.self.key.bias\n",
      "bert.encoder.layer.10.attention.self.value.weight\n",
      "bert.encoder.layer.10.attention.self.value.bias\n",
      "bert.encoder.layer.10.attention.output.dense.weight\n",
      "bert.encoder.layer.10.attention.output.dense.bias\n",
      "bert.encoder.layer.10.attention.output.LayerNorm.weight\n",
      "bert.encoder.layer.10.attention.output.LayerNorm.bias\n",
      "bert.encoder.layer.10.intermediate.dense.weight\n",
      "bert.encoder.layer.10.intermediate.dense.bias\n",
      "bert.encoder.layer.10.output.dense.weight\n",
      "bert.encoder.layer.10.output.dense.bias\n",
      "bert.encoder.layer.10.output.LayerNorm.weight\n",
      "bert.encoder.layer.10.output.LayerNorm.bias\n",
      "bert.encoder.layer.11.attention.self.query.weight\n",
      "bert.encoder.layer.11.attention.self.query.bias\n",
      "bert.encoder.layer.11.attention.self.key.weight\n",
      "bert.encoder.layer.11.attention.self.key.bias\n",
      "bert.encoder.layer.11.attention.self.value.weight\n",
      "bert.encoder.layer.11.attention.self.value.bias\n",
      "bert.encoder.layer.11.attention.output.dense.weight\n",
      "bert.encoder.layer.11.attention.output.dense.bias\n",
      "bert.encoder.layer.11.attention.output.LayerNorm.weight\n",
      "bert.encoder.layer.11.attention.output.LayerNorm.bias\n",
      "bert.encoder.layer.11.intermediate.dense.weight\n",
      "bert.encoder.layer.11.intermediate.dense.bias\n",
      "bert.encoder.layer.11.output.dense.weight\n",
      "bert.encoder.layer.11.output.dense.bias\n",
      "bert.encoder.layer.11.output.LayerNorm.weight\n",
      "bert.encoder.layer.11.output.LayerNorm.bias\n",
      "bert.pooler.dense.weight\n",
      "bert.pooler.dense.bias\n",
      "cls_token_head.1.weight\n",
      "cls_token_head.1.bias\n",
      "classifier.weight\n",
      "classifier.bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1032/1032 [08:14<00:00,  2.09it/s]\n",
      "100%|██████████| 311/311 [00:47<00:00,  6.55it/s]\n",
      "100%|██████████| 336/336 [00:51<00:00,  6.56it/s]\n",
      "  0%|          | 0/1032 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1: Train Loss 0.21441988131733497, Valid Loss 0.18289941694240094\n",
      "\t Train Spearmanr 0.5187, Valid Spearmanr (avg) 0.5200, Valid Spearmanr (last) 0.5200\n",
      "\t elapsed: 593.1157283782959s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1032/1032 [08:14<00:00,  2.09it/s]\n",
      "100%|██████████| 311/311 [00:47<00:00,  6.55it/s]\n",
      "100%|██████████| 336/336 [00:51<00:00,  6.56it/s]\n",
      "  0%|          | 0/1032 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 2: Train Loss 0.1638477302901808, Valid Loss 0.1893603660321983\n",
      "\t Train Spearmanr 0.5727, Valid Spearmanr (avg) 0.5297, Valid Spearmanr (last) 0.5335\n",
      "\t elapsed: 593.2233009338379s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1032/1032 [08:14<00:00,  2.09it/s]\n",
      "100%|██████████| 311/311 [00:47<00:00,  6.55it/s]\n",
      "100%|██████████| 336/336 [00:51<00:00,  6.56it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 3: Train Loss 0.09988516832606202, Valid Loss 0.23029252088859295\n",
      "\t Train Spearmanr 0.6448, Valid Spearmanr (avg) 0.5300, Valid Spearmanr (last) 0.5048\n",
      "\t elapsed: 593.1130738258362s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/921 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bert.embeddings.word_embeddings.weight\n",
      "bert.embeddings.position_embeddings.weight\n",
      "bert.embeddings.token_type_embeddings.weight\n",
      "bert.embeddings.LayerNorm.weight\n",
      "bert.embeddings.LayerNorm.bias\n",
      "bert.encoder.layer.0.attention.self.query.weight\n",
      "bert.encoder.layer.0.attention.self.query.bias\n",
      "bert.encoder.layer.0.attention.self.key.weight\n",
      "bert.encoder.layer.0.attention.self.key.bias\n",
      "bert.encoder.layer.0.attention.self.value.weight\n",
      "bert.encoder.layer.0.attention.self.value.bias\n",
      "bert.encoder.layer.0.attention.output.dense.weight\n",
      "bert.encoder.layer.0.attention.output.dense.bias\n",
      "bert.encoder.layer.0.attention.output.LayerNorm.weight\n",
      "bert.encoder.layer.0.attention.output.LayerNorm.bias\n",
      "bert.encoder.layer.0.intermediate.dense.weight\n",
      "bert.encoder.layer.0.intermediate.dense.bias\n",
      "bert.encoder.layer.0.output.dense.weight\n",
      "bert.encoder.layer.0.output.dense.bias\n",
      "bert.encoder.layer.0.output.LayerNorm.weight\n",
      "bert.encoder.layer.0.output.LayerNorm.bias\n",
      "bert.encoder.layer.1.attention.self.query.weight\n",
      "bert.encoder.layer.1.attention.self.query.bias\n",
      "bert.encoder.layer.1.attention.self.key.weight\n",
      "bert.encoder.layer.1.attention.self.key.bias\n",
      "bert.encoder.layer.1.attention.self.value.weight\n",
      "bert.encoder.layer.1.attention.self.value.bias\n",
      "bert.encoder.layer.1.attention.output.dense.weight\n",
      "bert.encoder.layer.1.attention.output.dense.bias\n",
      "bert.encoder.layer.1.attention.output.LayerNorm.weight\n",
      "bert.encoder.layer.1.attention.output.LayerNorm.bias\n",
      "bert.encoder.layer.1.intermediate.dense.weight\n",
      "bert.encoder.layer.1.intermediate.dense.bias\n",
      "bert.encoder.layer.1.output.dense.weight\n",
      "bert.encoder.layer.1.output.dense.bias\n",
      "bert.encoder.layer.1.output.LayerNorm.weight\n",
      "bert.encoder.layer.1.output.LayerNorm.bias\n",
      "bert.encoder.layer.2.attention.self.query.weight\n",
      "bert.encoder.layer.2.attention.self.query.bias\n",
      "bert.encoder.layer.2.attention.self.key.weight\n",
      "bert.encoder.layer.2.attention.self.key.bias\n",
      "bert.encoder.layer.2.attention.self.value.weight\n",
      "bert.encoder.layer.2.attention.self.value.bias\n",
      "bert.encoder.layer.2.attention.output.dense.weight\n",
      "bert.encoder.layer.2.attention.output.dense.bias\n",
      "bert.encoder.layer.2.attention.output.LayerNorm.weight\n",
      "bert.encoder.layer.2.attention.output.LayerNorm.bias\n",
      "bert.encoder.layer.2.intermediate.dense.weight\n",
      "bert.encoder.layer.2.intermediate.dense.bias\n",
      "bert.encoder.layer.2.output.dense.weight\n",
      "bert.encoder.layer.2.output.dense.bias\n",
      "bert.encoder.layer.2.output.LayerNorm.weight\n",
      "bert.encoder.layer.2.output.LayerNorm.bias\n",
      "bert.encoder.layer.3.attention.self.query.weight\n",
      "bert.encoder.layer.3.attention.self.query.bias\n",
      "bert.encoder.layer.3.attention.self.key.weight\n",
      "bert.encoder.layer.3.attention.self.key.bias\n",
      "bert.encoder.layer.3.attention.self.value.weight\n",
      "bert.encoder.layer.3.attention.self.value.bias\n",
      "bert.encoder.layer.3.attention.output.dense.weight\n",
      "bert.encoder.layer.3.attention.output.dense.bias\n",
      "bert.encoder.layer.3.attention.output.LayerNorm.weight\n",
      "bert.encoder.layer.3.attention.output.LayerNorm.bias\n",
      "bert.encoder.layer.3.intermediate.dense.weight\n",
      "bert.encoder.layer.3.intermediate.dense.bias\n",
      "bert.encoder.layer.3.output.dense.weight\n",
      "bert.encoder.layer.3.output.dense.bias\n",
      "bert.encoder.layer.3.output.LayerNorm.weight\n",
      "bert.encoder.layer.3.output.LayerNorm.bias\n",
      "bert.encoder.layer.4.attention.self.query.weight\n",
      "bert.encoder.layer.4.attention.self.query.bias\n",
      "bert.encoder.layer.4.attention.self.key.weight\n",
      "bert.encoder.layer.4.attention.self.key.bias\n",
      "bert.encoder.layer.4.attention.self.value.weight\n",
      "bert.encoder.layer.4.attention.self.value.bias\n",
      "bert.encoder.layer.4.attention.output.dense.weight\n",
      "bert.encoder.layer.4.attention.output.dense.bias\n",
      "bert.encoder.layer.4.attention.output.LayerNorm.weight\n",
      "bert.encoder.layer.4.attention.output.LayerNorm.bias\n",
      "bert.encoder.layer.4.intermediate.dense.weight\n",
      "bert.encoder.layer.4.intermediate.dense.bias\n",
      "bert.encoder.layer.4.output.dense.weight\n",
      "bert.encoder.layer.4.output.dense.bias\n",
      "bert.encoder.layer.4.output.LayerNorm.weight\n",
      "bert.encoder.layer.4.output.LayerNorm.bias\n",
      "bert.encoder.layer.5.attention.self.query.weight\n",
      "bert.encoder.layer.5.attention.self.query.bias\n",
      "bert.encoder.layer.5.attention.self.key.weight\n",
      "bert.encoder.layer.5.attention.self.key.bias\n",
      "bert.encoder.layer.5.attention.self.value.weight\n",
      "bert.encoder.layer.5.attention.self.value.bias\n",
      "bert.encoder.layer.5.attention.output.dense.weight\n",
      "bert.encoder.layer.5.attention.output.dense.bias\n",
      "bert.encoder.layer.5.attention.output.LayerNorm.weight\n",
      "bert.encoder.layer.5.attention.output.LayerNorm.bias\n",
      "bert.encoder.layer.5.intermediate.dense.weight\n",
      "bert.encoder.layer.5.intermediate.dense.bias\n",
      "bert.encoder.layer.5.output.dense.weight\n",
      "bert.encoder.layer.5.output.dense.bias\n",
      "bert.encoder.layer.5.output.LayerNorm.weight\n",
      "bert.encoder.layer.5.output.LayerNorm.bias\n",
      "bert.encoder.layer.6.attention.self.query.weight\n",
      "bert.encoder.layer.6.attention.self.query.bias\n",
      "bert.encoder.layer.6.attention.self.key.weight\n",
      "bert.encoder.layer.6.attention.self.key.bias\n",
      "bert.encoder.layer.6.attention.self.value.weight\n",
      "bert.encoder.layer.6.attention.self.value.bias\n",
      "bert.encoder.layer.6.attention.output.dense.weight\n",
      "bert.encoder.layer.6.attention.output.dense.bias\n",
      "bert.encoder.layer.6.attention.output.LayerNorm.weight\n",
      "bert.encoder.layer.6.attention.output.LayerNorm.bias\n",
      "bert.encoder.layer.6.intermediate.dense.weight\n",
      "bert.encoder.layer.6.intermediate.dense.bias\n",
      "bert.encoder.layer.6.output.dense.weight\n",
      "bert.encoder.layer.6.output.dense.bias\n",
      "bert.encoder.layer.6.output.LayerNorm.weight\n",
      "bert.encoder.layer.6.output.LayerNorm.bias\n",
      "bert.encoder.layer.7.attention.self.query.weight\n",
      "bert.encoder.layer.7.attention.self.query.bias\n",
      "bert.encoder.layer.7.attention.self.key.weight\n",
      "bert.encoder.layer.7.attention.self.key.bias\n",
      "bert.encoder.layer.7.attention.self.value.weight\n",
      "bert.encoder.layer.7.attention.self.value.bias\n",
      "bert.encoder.layer.7.attention.output.dense.weight\n",
      "bert.encoder.layer.7.attention.output.dense.bias\n",
      "bert.encoder.layer.7.attention.output.LayerNorm.weight\n",
      "bert.encoder.layer.7.attention.output.LayerNorm.bias\n",
      "bert.encoder.layer.7.intermediate.dense.weight\n",
      "bert.encoder.layer.7.intermediate.dense.bias\n",
      "bert.encoder.layer.7.output.dense.weight\n",
      "bert.encoder.layer.7.output.dense.bias\n",
      "bert.encoder.layer.7.output.LayerNorm.weight\n",
      "bert.encoder.layer.7.output.LayerNorm.bias\n",
      "bert.encoder.layer.8.attention.self.query.weight\n",
      "bert.encoder.layer.8.attention.self.query.bias\n",
      "bert.encoder.layer.8.attention.self.key.weight\n",
      "bert.encoder.layer.8.attention.self.key.bias\n",
      "bert.encoder.layer.8.attention.self.value.weight\n",
      "bert.encoder.layer.8.attention.self.value.bias\n",
      "bert.encoder.layer.8.attention.output.dense.weight\n",
      "bert.encoder.layer.8.attention.output.dense.bias\n",
      "bert.encoder.layer.8.attention.output.LayerNorm.weight\n",
      "bert.encoder.layer.8.attention.output.LayerNorm.bias\n",
      "bert.encoder.layer.8.intermediate.dense.weight\n",
      "bert.encoder.layer.8.intermediate.dense.bias\n",
      "bert.encoder.layer.8.output.dense.weight\n",
      "bert.encoder.layer.8.output.dense.bias\n",
      "bert.encoder.layer.8.output.LayerNorm.weight\n",
      "bert.encoder.layer.8.output.LayerNorm.bias\n",
      "bert.encoder.layer.9.attention.self.query.weight\n",
      "bert.encoder.layer.9.attention.self.query.bias\n",
      "bert.encoder.layer.9.attention.self.key.weight\n",
      "bert.encoder.layer.9.attention.self.key.bias\n",
      "bert.encoder.layer.9.attention.self.value.weight\n",
      "bert.encoder.layer.9.attention.self.value.bias\n",
      "bert.encoder.layer.9.attention.output.dense.weight\n",
      "bert.encoder.layer.9.attention.output.dense.bias\n",
      "bert.encoder.layer.9.attention.output.LayerNorm.weight\n",
      "bert.encoder.layer.9.attention.output.LayerNorm.bias\n",
      "bert.encoder.layer.9.intermediate.dense.weight\n",
      "bert.encoder.layer.9.intermediate.dense.bias\n",
      "bert.encoder.layer.9.output.dense.weight\n",
      "bert.encoder.layer.9.output.dense.bias\n",
      "bert.encoder.layer.9.output.LayerNorm.weight\n",
      "bert.encoder.layer.9.output.LayerNorm.bias\n",
      "bert.encoder.layer.10.attention.self.query.weight\n",
      "bert.encoder.layer.10.attention.self.query.bias\n",
      "bert.encoder.layer.10.attention.self.key.weight\n",
      "bert.encoder.layer.10.attention.self.key.bias\n",
      "bert.encoder.layer.10.attention.self.value.weight\n",
      "bert.encoder.layer.10.attention.self.value.bias\n",
      "bert.encoder.layer.10.attention.output.dense.weight\n",
      "bert.encoder.layer.10.attention.output.dense.bias\n",
      "bert.encoder.layer.10.attention.output.LayerNorm.weight\n",
      "bert.encoder.layer.10.attention.output.LayerNorm.bias\n",
      "bert.encoder.layer.10.intermediate.dense.weight\n",
      "bert.encoder.layer.10.intermediate.dense.bias\n",
      "bert.encoder.layer.10.output.dense.weight\n",
      "bert.encoder.layer.10.output.dense.bias\n",
      "bert.encoder.layer.10.output.LayerNorm.weight\n",
      "bert.encoder.layer.10.output.LayerNorm.bias\n",
      "bert.encoder.layer.11.attention.self.query.weight\n",
      "bert.encoder.layer.11.attention.self.query.bias\n",
      "bert.encoder.layer.11.attention.self.key.weight\n",
      "bert.encoder.layer.11.attention.self.key.bias\n",
      "bert.encoder.layer.11.attention.self.value.weight\n",
      "bert.encoder.layer.11.attention.self.value.bias\n",
      "bert.encoder.layer.11.attention.output.dense.weight\n",
      "bert.encoder.layer.11.attention.output.dense.bias\n",
      "bert.encoder.layer.11.attention.output.LayerNorm.weight\n",
      "bert.encoder.layer.11.attention.output.LayerNorm.bias\n",
      "bert.encoder.layer.11.intermediate.dense.weight\n",
      "bert.encoder.layer.11.intermediate.dense.bias\n",
      "bert.encoder.layer.11.output.dense.weight\n",
      "bert.encoder.layer.11.output.dense.bias\n",
      "bert.encoder.layer.11.output.LayerNorm.weight\n",
      "bert.encoder.layer.11.output.LayerNorm.bias\n",
      "bert.pooler.dense.weight\n",
      "bert.pooler.dense.bias\n",
      "cls_token_head.1.weight\n",
      "cls_token_head.1.bias\n",
      "classifier.weight\n",
      "classifier.bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 921/921 [07:20<00:00,  2.09it/s]\n",
      "100%|██████████| 422/422 [01:04<00:00,  6.58it/s]\n",
      "100%|██████████| 336/336 [00:51<00:00,  6.59it/s]\n",
      "  0%|          | 0/921 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1: Train Loss 0.20821738861507283, Valid Loss 0.24535326843737001\n",
      "\t Train Spearmanr 0.5133, Valid Spearmanr (avg) 0.5052, Valid Spearmanr (last) 0.5052\n",
      "\t elapsed: 555.4103755950928s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 921/921 [07:20<00:00,  2.09it/s]\n",
      "100%|██████████| 422/422 [01:04<00:00,  6.57it/s]\n",
      "100%|██████████| 336/336 [00:51<00:00,  6.59it/s]\n",
      "  0%|          | 0/921 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 2: Train Loss 0.15871176949320978, Valid Loss 0.2078020681861029\n",
      "\t Train Spearmanr 0.5615, Valid Spearmanr (avg) 0.5555, Valid Spearmanr (last) 0.5571\n",
      "\t elapsed: 555.9400916099548s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 921/921 [07:21<00:00,  2.09it/s]\n",
      "100%|██████████| 422/422 [01:04<00:00,  6.56it/s]\n",
      "100%|██████████| 336/336 [00:51<00:00,  6.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 3: Train Loss 0.10767920500518963, Valid Loss 0.24470226979434878\n",
      "\t Train Spearmanr 0.6274, Valid Spearmanr (avg) 0.5530, Valid Spearmanr (last) 0.5338\n",
      "\t elapsed: 556.6648147106171s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "bert_histories = []\n",
    "\n",
    "for fold, (train_idx, valid_idx) in enumerate(fold_ids):\n",
    "    import gc\n",
    "    gc.collect()\n",
    "    \n",
    "    train_inputs = [inputs[i][train_idx] for i in range(3)]\n",
    "    train_outputs = outputs[train_idx]\n",
    "    train_dataset = torch.utils.data.TensorDataset(*train_inputs, train_outputs)\n",
    "\n",
    "    valid_inputs = [inputs[i][valid_idx] for i in range(3)]\n",
    "    valid_outputs = outputs[valid_idx]\n",
    "    valid_dataset = torch.utils.data.TensorDataset(*valid_inputs, valid_outputs)\n",
    "   \n",
    "    history = train_and_predict(train_data=train_dataset, valid_data=valid_dataset, test_data=test_dataset, \n",
    "        epochs=3, batch_size=8, model_type = 'bert', device = device, fold=fold)\n",
    "\n",
    "    bert_histories.append(history)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs = len(bert_histories[0][0])\n",
    "test_bert_preds_list = []\n",
    "test_predictions_bert = np.zeros((n_epochs, len(df_test), len(output_categories)), dtype=np.float32)\n",
    "\n",
    "for epoch in range(n_epochs):\n",
    "    test_preds_one_epoch_bert = []\n",
    "\n",
    "    for fold in range(len(fold_ids)):\n",
    "        test_preds = bert_histories[fold][1][epoch]\n",
    "        test_preds_one_epoch_bert.append(test_preds)\n",
    "   \n",
    "    test_preds_one_epoch_bert = np.array([np.array([rankdata(c) for c in p.T]).T \n",
    "                                          for p in test_preds_one_epoch_bert]).mean(axis=0)\n",
    "    max_epoch_rank_value = (test_preds_one_epoch_bert).max() + 1\n",
    "    test_preds_one_epoch_bert = test_preds_one_epoch_bert/max_epoch_rank_value\n",
    "    test_bert_preds_list.append(test_preds_one_epoch_bert)\n",
    "\n",
    "\n",
    "for j, name in enumerate(output_categories):\n",
    "    for epoch in range(n_epochs):\n",
    "        col = \"{}_{}\".format(epoch, name)\n",
    "        test_predictions_bert[epoch, :, j] = test_bert_preds_list[epoch][:, j]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3, 2685, 2)"
      ]
     },
     "execution_count": 172,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_predictions_bert.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_preds_bert = test_bert_preds_list[n_epochs - 1][:, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DescribeResult(nobs=2685, minmax=(0.0014903129657228018, 0.9909960258320915), mean=0.5003725782414307, variance=0.07692651083590676, skewness=-0.03004033354239729, kurtosis=-1.3721339349705053)"
      ]
     },
     "execution_count": 170,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stats.describe(test_preds_bert[:,0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DescribeResult(nobs=2685, minmax=(0.001366120218579235, 0.9996274217585693), mean=0.5003725782414307, variance=0.06407081454996681, skewness=0.08559423095304955, kurtosis=-1.0506793559002334)"
      ]
     },
     "execution_count": 158,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stats.describe(test_preds_bert[:,1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00      1345\n",
      "           1       1.00      1.00      1.00      1340\n",
      "\n",
      "    accuracy                           1.00      2685\n",
      "   macro avg       1.00      1.00      1.00      2685\n",
      "weighted avg       1.00      1.00      1.00      2685\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_test_troll, np.around(test_preds_bert[:,0]).astype(np.int32)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.97      0.55      0.70      2443\n",
      "           1       0.16      0.84      0.26       242\n",
      "\n",
      "    accuracy                           0.58      2685\n",
      "   macro avg       0.56      0.70      0.48      2685\n",
      "weighted avg       0.90      0.58      0.66      2685\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_test_contro, np.around(test_preds_bert[:,1]).astype(np.int32)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [],
   "source": [
    "fpr_troll, tpr_troll, thresholds_troll = roc_curve(y_test_troll, np.around(test_preds_bert[:,0]).astype(np.int32))\n",
    "fpr_contro, tpr_contro, thresholds_contro = roc_curve(y_test_contro, np.around(test_preds_bert[:,1]).astype(np.int32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9973963824002664 0.6963545701498293\n"
     ]
    }
   ],
   "source": [
    "auc_troll = auc(fpr_troll, tpr_troll)\n",
    "auc_contro = auc(fpr_contro, tpr_contro)\n",
    "print(auc_troll, auc_contro)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-5"
      ]
     },
     "execution_count": 184,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = y_test_troll - np.around(test_preds_bert[:,0]).astype(np.int32)\n",
    "x.sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9973963824002663 0.6963545701498293\n"
     ]
    }
   ],
   "source": [
    "ba_troll = balanced_accuracy_score(y_test_troll, np.around(test_preds_bert[:,0]).astype(np.int32))\n",
    "ba_contro = balanced_accuracy_score(y_test_contro, np.around(test_preds_bert[:,1]).astype(np.int32))\n",
    "print(ba_troll, ba_contro)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "roberta_tokenizer_merges_file = '../models/bert/roberta-transformers-pytorch/roberta-base/roberta-base-merges.txt'\n",
    "roberta_tokenizer = RobertaTokenizer(vocab_file = '../models/bert/roberta-transformers-pytorch/roberta-base/roberta-base-vocab.json', merges_file=roberta_tokenizer_merges_file)\n",
    "outputs = torch.tensor(compute_output_arrays(df_train), dtype=torch.float)\n",
    "inputs = [torch.tensor(x, dtype=torch.long) for x in compute_input_arrays(df_train, roberta_tokenizer)]\n",
    "test_inputs = [torch.tensor(x, dtype=torch.long) for x in compute_input_arrays(df_test, roberta_tokenizer)]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}