(Model 6) ElmoEmbedding + Interpretability.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import csv\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "from allennlp.modules.elmo import Elmo, batch_to_ids\n",
    "\n",
    "import pickle\n",
    "\n",
    "\n",
    "\n",
    "DS = pd.read_csv(\"ObjectSubset150SentenceLength.csv\")\n",
    "DS['tag'].fillna('text', inplace=True)\n",
    "words = list(set(DS[\"word\"].values))\n",
    "n_words = len(words)\n",
    "tags = list(set(DS[\"tag\"].values))\n",
    "n_tags = len(tags)\n",
    "\n",
    "class SentenceGetter(object):\n",
    "    def __init__(self, data):\n",
    "        self.n_sent = 1\n",
    "        self.data = data\n",
    "        self.empty = False\n",
    "        agg_func = lambda s: [(w, t) for w, t in zip(s[\"word\"].values.tolist(),\n",
    "                                                           s[\"tag\"].values.tolist())]\n",
    "        self.grouped = self.data.groupby(\"text_ID\").apply(agg_func)\n",
    "        self.sentences = [s for s in self.grouped]\n",
    "\n",
    "getter = SentenceGetter(DS)\n",
    "sentences = getter.sentences\n",
    "\n",
    "max_len = 10\n",
    "tags2 = [\"PAD\",\"object\",\"text\"]\n",
    "tag2idx = {t: i  for i, t in enumerate(tags2)}\n",
    "\n",
    "X = [[w[0] for w in s] for s in sentences]\n",
    "\n",
    "new_X = []\n",
    "for seq in X:\n",
    "    new_seq = []\n",
    "    for i in range(max_len):\n",
    "        try:\n",
    "            new_seq.append(seq[i])\n",
    "        except:\n",
    "            new_seq.append(\"PAD\")\n",
    "    new_X.append(new_seq)\n",
    "X = new_X\n",
    "\n",
    "y = [[tag2idx[w[1]] for w in s] for s in sentences]\n",
    "\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "y = pad_sequences(maxlen=max_len, sequences=y, padding=\"post\", value=tag2idx[\"PAD\"], truncating='post')\n",
    "\n",
    "idx2tag = {i: w for w, i in tag2idx.items()}\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "X_tr, X_te, y_tr, y_te = train_test_split(X, y, test_size=0.2, random_state=2018)\n",
    "\n",
    "\n",
    "\n",
    "batch_size = 32\n",
    "\n",
    "\n",
    "# options_file = \"options.json\"\n",
    "# weight_file = \"swb_weights.hdf5\"\n",
    "# elmo_model  = Elmo(options_file, weight_file, 2, dropout=0)\n",
    "\n",
    "\n",
    "\n",
    "# X_tr_character_ids = batch_to_ids(X_tr)\n",
    "# X_tr_embeddings = elmo_model(X_tr_character_ids)\n",
    "# pickle.dump(X_tr_embeddings, open('X_tr_embeddings.pickle', 'wb'))\n",
    "\n",
    "# X_te_character_ids = batch_to_ids(X_te)\n",
    "# X_te_embeddings = elmo_model(X_te_character_ids)\n",
    "# pickle.dump(X_te_embeddings, open('X_te_embeddings.pickle', 'wb'))\n",
    "\n",
    "\n",
    "\n",
    "# pickle.dump(y_tr, open('y_tr.pickle', 'wb'))\n",
    "# pickle.dump(y_te, open('y_te.pickle', 'wb'))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_len = 150\n",
    "word2idx = {w: i + 2 for i, w in enumerate(words)}\n",
    "\n",
    "word2idx[\"UNK\"] = 1\n",
    "word2idx[\"PAD\"] = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from allennlp.modules.elmo import Elmo, batch_to_ids\n",
    "options_file = \"elmoembedings/options.json\"\n",
    "weight_file = \"swb_weights.hdf5\"\n",
    "elmo_model  = Elmo(options_file, weight_file, 2, dropout=0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from keras.models import Model, Input\n",
    "# from keras.layers.merge import add\n",
    "from keras.layers import LSTM, Embedding, Dense, TimeDistributed, Dropout, Bidirectional, Lambda\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# X_tr_character_ids = batch_to_ids(X_tr)\n",
    "# X_tr_embeddings = elmo_model(X_tr_character_ids)\n",
    "\n",
    "# X_te_character_ids = batch_to_ids(X_te)\n",
    "# X_te_embeddings = elmo_model(X_te_character_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load pickle files that was prepared in UT server.\n",
    "with open('elmoembedings\\X_tr_embeddings.pickle', 'rb') as f:\n",
    "    X_tr_embeddings = pickle.load(f)\n",
    "\n",
    "with open('elmoembedings\\X_te_embeddings.pickle', 'rb') as f:\n",
    "    X_te_embeddings = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10914, 35, 1024)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_tr_embeddings['elmo_representations'][1].detach().numpy().shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sess.run(tf.compat.v1.global_variables_initializer())\n",
    "# sess.run(tf.compat.v1.tables_initializer())\n",
    "# def ElmoEmbedding(x):\n",
    "#     return elmo_model(inputs={\n",
    "#                             \"tokens\": tf.squeeze(tf.cast(x, tf.string)),\n",
    "#                             \"sequence_len\": tf.constant(batch_size*[max_len])\n",
    "#                       },\n",
    "# #                       signature=\"tokens\",\n",
    "#                       as_dict=True)[\"elmo\"]\n",
    "\n",
    "# from keras.models import Model, Input\n",
    "# from keras.layers.merge import add\n",
    "# from keras.layers import LSTM, Embedding, Dense, TimeDistributed, Dropout, Bidirectional, Lambda\n",
    "\n",
    "# input_text = Input(shape=(max_len,), dtype=tf.string)\n",
    "# embedding = Lambda(ElmoEmbedding, output_shape=(None, 1024))(input_text)\n",
    "# x = Bidirectional(LSTM(units=512, return_sequences=True,\n",
    "#                        recurrent_dropout=0.2, dropout=0.2))(embedding)\n",
    "# x_rnn = Bidirectional(LSTM(units=512, return_sequences=True,\n",
    "#                            recurrent_dropout=0.2, dropout=0.2))(x)\n",
    "# x = add([x, x_rnn])  # residual connection to the first biLSTM\n",
    "# out = TimeDistributed(Dense(n_tags, activation=\"softmax\"))(x)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([10914, 35, 1024])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_tr_embeddings[\"elmo_representations\"][1].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load pickle files that was prepared in UT server.\n",
    "with open('elmoembedings/y_tr.pickle', 'rb') as f:\n",
    "    y_tr = pickle.load(f)\n",
    "\n",
    "with open('elmoembedings/y_te.pickle', 'rb') as f:\n",
    "    y_te = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10914, 35)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_tr.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import LSTM, core, Activation, Dense\n",
    "import numpy as np\n",
    "from keras.layers import Bidirectional, concatenate, SpatialDropout1D, GlobalMaxPooling1D\n",
    "\n",
    "\n",
    "model = Sequential()\n",
    "# model.add(TimeDistributed(LSTM(units=20,input_shape=(10, 1024), return_sequences=False,\n",
    "#                                 recurrent_dropout=0.5)))\n",
    "\n",
    "# # main LSTM\n",
    "# model.add(SpatialDropout1D(0.3))\n",
    "\n",
    "model.add(Bidirectional(LSTM(20, input_shape=(10, 1024), return_sequences=True,recurrent_dropout=0.5)))\n",
    "\n",
    "model.add(TimeDistributed(Dense(3, activation='softmax')))\n",
    "model.compile(loss='sparse_categorical_crossentropy', optimizer='adam', metrics=[\"accuracy\"])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 9822 samples, validate on 1092 samples\n",
      "Epoch 1/10\n",
      "9822/9822 [==============================] - 14s 1ms/step - loss: 0.0355 - accuracy: 0.9889 - val_loss: 0.0065 - val_accuracy: 0.9986\n",
      "Epoch 2/10\n",
      "9822/9822 [==============================] - 13s 1ms/step - loss: 0.0052 - accuracy: 0.9990 - val_loss: 0.0037 - val_accuracy: 0.9992\n",
      "Epoch 3/10\n",
      "9822/9822 [==============================] - 13s 1ms/step - loss: 0.0034 - accuracy: 0.9992 - val_loss: 0.0025 - val_accuracy: 0.9993\n",
      "Epoch 4/10\n",
      "9822/9822 [==============================] - 13s 1ms/step - loss: 0.0026 - accuracy: 0.9994 - val_loss: 0.0024 - val_accuracy: 0.9993\n",
      "Epoch 5/10\n",
      "9822/9822 [==============================] - 13s 1ms/step - loss: 0.0020 - accuracy: 0.9995 - val_loss: 0.0020 - val_accuracy: 0.9993\n",
      "Epoch 6/10\n",
      "9822/9822 [==============================] - 13s 1ms/step - loss: 0.0017 - accuracy: 0.9996 - val_loss: 0.0022 - val_accuracy: 0.9992\n",
      "Epoch 7/10\n",
      "9822/9822 [==============================] - 13s 1ms/step - loss: 0.0014 - accuracy: 0.9996 - val_loss: 0.0015 - val_accuracy: 0.9996\n",
      "Epoch 8/10\n",
      "9822/9822 [==============================] - 13s 1ms/step - loss: 0.0012 - accuracy: 0.9997 - val_loss: 0.0014 - val_accuracy: 0.9997\n",
      "Epoch 9/10\n",
      "9822/9822 [==============================] - 13s 1ms/step - loss: 0.0010 - accuracy: 0.9998 - val_loss: 0.0013 - val_accuracy: 0.9997\n",
      "Epoch 10/10\n",
      "9822/9822 [==============================] - 13s 1ms/step - loss: 8.9693e-04 - accuracy: 0.9998 - val_loss: 0.0013 - val_accuracy: 0.9997\n"
     ]
    }
   ],
   "source": [
    "history = model.fit(X_tr_embeddings['elmo_representations'][1].detach().numpy(), y_tr.reshape(y_tr.shape[0],35,1),validation_split=0.1 ,verbose=1,epochs=10,batch_size=32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAs8AAAKrCAYAAAAOF2D7AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nOzda5Ab533v+V/jOgPMHRjO8H6XRNnU7VC+ULEdkTza9Son4mZd3uTEXpelrC3RrFScHK03KsdvIipKSSrmuGQrqZgl5kilaLNey0m25KyW9MbelU6JskXakmVLoi5WSA4HQGOumAsG6GdfNAYzmAtnSGKmG8D3UzUFoNGYfppsib/5z7+fxzLGGAEAAABYUsDrAQAAAAC1gvAMAAAALBPhGQAAAFgmwjMAAACwTIRnAAAAYJkIzwAAAMAyhbwewOW6cOHCqh8zmUwqk8ms+nFRG7g+sBiuDSyGawOL4drwj3Xr1i24ncozAAAAsEyEZwAAAGCZCM8AAADAMhGeAQAAgGUiPAMAAADLRHgGAAAAlonwDAAAACwT4RkAAABYJsIzAAAAsEyEZwAAAGCZCM8AAADAMhGeAQAAgGUiPAMAAADLFFpqh29/+9t69dVX1d7erscee2ze+8YYPfnkkzp9+rSi0agOHTqkbdu2SZLOnDmjJ598Uo7jaP/+/Tp48KAkaXR0VEePHlU6nVZ3d7e++tWvqqWlpcqnBgAAAFTXkpXn3/zN39QDDzyw6PunT5/WxYsX9c1vflNf+tKX9J3vfEeS5DiOjh07pgceeEBHjx7Viy++qHPnzkmSvv/972v37t365je/qd27d+v73/9+lU4HAAAAWDlLhufrr7/+klXhn/zkJ/rkJz8py7J0zTXXKJfLaWBgQGfPnlVvb696enoUCoW0d+9evfLKK5KkV155RZ/61KckSZ/61KfK2wEAAAA/u+qe52w2q2QyWX6dSCSUzWaVzWaVSCTmbZekoaEhdXZ2SpI6Ozs1PDx8tcMAAAAAVtySPc9LMcbM22ZZ1qLbL9eJEyd04sQJSdLDDz9cEdRXSygU8uS4qA1cH1gM1wYWw7WBxXBt+N9Vh+dEIqFMJlN+bdu2Ojs7VSgUZNv2vO2S1N7eroGBAXV2dmpgYEBtbW2Lfv8DBw7owIED5dezj7VaksmkJ8dFbeD6wGK4NrAYrg0shmvDP9atW7fg9qtu29izZ49+/OMfyxijt956S7FYTJ2dndq+fbv6+vqUSqVUKBT00ksvac+ePeXP/OhHP5Ik/ehHP9Ktt956tcMAAAAAVtySlee/+qu/0htvvKGRkRHde++9+uxnP6tCoSBJuuOOO3TzzTfr1Vdf1R/+4R8qEono0KFDkqRgMKi7775bR44ckeM4uv3227Vx40ZJ0sGDB3X06FH98Ic/VDKZ1B//8R+v4CkCAAAA1WGZhZqTfezChQurfkx+hYJL4frAYrg2sBiuDSyGa8M/VqxtAwAAAGgUhGcAAABgmQjPAAAAwDIRngEAAIBlIjwDAAAAy3TVi6QAAACgcRhjpGJRKhakQkEqTrmPhcLMtkW2m8LUrM9del8VC1IsrsBnvuj1KVcgPAMAAPiAMUYmPykzlpsVLJcOm+Yy9p293Uxvr9i39H3mhti532slWJYUCkuhkBQMuY/J3pU51lUgPAMAAFwlY4yUn5TGc9L4mDTmPprxsZltsx7d7fPfSxWL1R9cKCQFS6F0djCdfgyFZ543NZWfWwt95pLfK1z63PL3nf19rUCw+ue+AgjPAACgoRljpMmJRQLudBieeW7mheExaWLMbWW4FMuSmmJS8/RXXGrvktW7UYq52+JdSeXyU/PCp7VA2FwwmM7eHgxJwaAsy1qdP8gGQXgGAAA1yw2+46Vwe6mAO6cSPCsMa2JMcpxLH8gKSM3NbuCdDr+dSVnrYrO2zbxnxeZui0vRJlmBS8/VEE8mNV4nKwwaY+QYqeCYiq+iU9pmjApFo6JZ4L3SVyRo6SMbWr0+lQqEZwAA4AlTLLoV34nx5QXfsVxltXc8J42PS2Y5wXdWtTcWkxLdspo3zw+4zTFZc8NwLCZFmz2p4BpjVHDkBsxiKXA6RkXHaKoUNmeeLx5C5wbU+ftPP9fM64pjuceZf6yFxzb93tXqbQkTngEAQG0wjiNN5d2AOznh9vTmJ0uvJ2XyE+Xn7nuzX0/IzNrXfW/W5/MTy7vxbMHgu0ZWxbaZsGstEIaL4agcaX44dJYRREeNisMFFZzhBYLmnCBaUUld+DgVYzBzxyAZndVUwSl/z6JZ8b9mhQJW6UsKlp+XvixLwYAqtkVDAYVK2+buHwxYClkz74Xn7DP7ewUtS6Gge4xQ0FLQ0rz9I0H/tZwQngEAqFHGGHdmhOlgO7lAgJ0dXKdD6/S+k7MC7rzHCSmfv/xBRaJStGnmMdqkqUizch2dykVblYvGlAvFlAs1KxdqUi4Y1WQwomIwomIorEIwrGIgpKlASMVAUEUrMFPhnPvr/+mQmjcqThhN2WZWdXRCRWdcBcdopfNnYDoslsPg/GA581yKhCzFA4GZbbMCakusWYX8ZMX+8wJtwA2a80LrQgF1OsDOHluw9DpgKWCJnujLRHgGAGCFmULB7csdH5MmxpXP9MmkLrrV2+mgWlGVnSxXe01FtXd+9XfJloW5QuFSqI26ATdSet7SKnV1y4pE3dfRpvJ7hXBUY+GYckE37OYCUeWssHIKK6egRk1AuWJAuaI0NuVoNO8oly+6X1OO8guVTwulr+lhBTQr4BmFAgUFA8UFK5zBgKVYKDAvCC5WQa0IonP2DZa2LxpES8dwA+7cSqp7nEAVw2cymVSmTnqe6xXhGQCABcxMPVa6oWx83H2cGHdvOpso9d1OjpffM6VwPB2S3f3H5lVwBy514ECgIriWH6NNUluHrNmvI9GKCq8iUVnlz8x8vhiJatyKKKeQxorSaL6oXN5Rbsp9HJ0OurO2lbcPFTVRWKh2ayS55xWwpJZIUPFIQPGw+5iIRdUy63U8ElQ8HCjtV7ktErSofqJmEJ4BAHWlsso7HXrHZSbGFgzCGh9z35uY9ZmJ8eXdiCa504E1N7tTkDXF3OftnbJ61rl9t9PbmmJSU7Os5pja1vRoeCI/p/rrhl0rFJ5/TsZovOCUQu2swDs16/VUKQAPOeWKby5f1Gh+RGNTQ5c8BUtSLFIKtmE31K5tCysebnID8JxgXBGAw0E1hQi/aByEZwCA58rz7E5Xa6fD7fiYzMTMc03M3T47JJfen1pmn260eSbUNrvBVu2dsqLN80Nvc0xWU/O8bWpqlkLhywqOo/mi+tSsC+lsqbI7HXbHNJofWaAC7AZhZ4nG3eZQoCLodsfD2tIRnakIzwrGc6vEzeFAVVsPgHpGeAYALMo4RWlqyr0pbSrvPp9+nLPNTOVL26akwpx9p/Kzqr/jC1aGl1/lLYXW2VXe3vXlym5l4C3tUxGIY8uab7daio7R2/aETveN6nTfmN62xxcMwtGgVQq2bsDtbAppQ9vsSm9lC8TsKnEs7N58BmDlEZ4BwKeM48yE0am8+1UozHo+VX40c4Nqxedmh9kpmcJi+80OxaXPVGOp4Onlf2dXa6ervHO3zQ295UDsBl8rPL+lwY/6R/M63ZfTmb6cfn5xTLkpR5akHYkmfeZDCd26rUeazJVDcSwcVNiHU3IBmI/wDABXyRjjVk4Hs9KgLTOUlQazGrUkZ3iosjo7N9QuVq0tTC1vDtylBEvBNRyWwpGZx9nbmuNSOOL22l5qv3BYCrmPVvl5xA3H4cjCnwuFV63C66WxqaJe6x/TmVJgvjAyJUlKxkLau6lVN6+N64beuFqjQXd7skuZTBVWkACw6gjPAHAJZjoUD2VlSo9uSM6WQ7IGs+6sDHPkAsFSmJwJnW6onBUym2NSKCSrInhGpHBoJpzOCaTlfed+r9lBt/Q9rEDQgz+1+ld0jN4dmChXl3+VHlfRuK0Xu3ti+u+u6dTNa+Na3xbhRjqgzhCeATQkMzkpDdnS4EBlCJ4OxdPbJsbnfzgSlTq6pI4uWZt3SDe6z9XeJav0qI5OdW/YxHytdSQzNqUzfTmd7svpZxfHNDLptrRs74rq4K4u3bQ2rl3dzQoH67/SDjQywjOAumKm8uVKsQZL1eLBrDQ0JySP5+Z/OByZCcEbtkofumUmJLd3SR0J93VTM9XEBjBRcPSL/jGdvpjT6Qs5nRt2Z/HobA7p1vVx3dQb141r4+po4p9SoJHwXzyAmmCmpqThgZnq8OyAPDsUj43O/3AoVKoGd0lrN8radeOsSnGn1F4KxbE4obiBOcbo/YHJcnX5jbS7tHMkaOn6NTH9+x3tuqk3rs0dUa4ToIERngF4yhQKM6F4aGBWpdiuqBprdHj+h4NBqb3TDcY962Rd++FySLamw3JHlxRvJexgQQPjhXJYPnMxp6EJtxVjc0dUv3Vtp25aG9f13c2KhmjFAOAiPANYEaZYlEYGF6kUD0iDtvve6LBk5kx6GwhIbZ1uME72yNqxa1alOOFu7+iSWtoaYiYHVE++6OiN1Hj5Rr/3B90bPdujQd24Nq6b18Z1Y29MiVhtTIkHYPURngFcETM2KmX6pUy/TLr0mE3P9BsPD81f9MKypLYOtzrcmZS19ZpyELamWyc6uqTWNmaJQFUYY/RvQ+6cy6f7cvpFakz5olEoYOn67mb9Tzd16+a1cW3pjLLCHoBlITwDWJDJT0p2yg3F0yG59KhMvzQ254a75riU6JY6ErI2baucfWJ6Boq2DllBQjFW1vBEQWcujpWry9lxd77sDW0R/Tc7OnTT2rg+3BNTE60YAK4A4RloUKZYlLLpWaG4FJTtUjgeGqj8QDgiJda4bRTbrnMfkz1S6cuKt3hzImh4U0WjX2XGdKbPDczvZidkJLVEArqx123FuGltXN1xWjEAXD3CM3CZJguO7LGC7PEpDYwXlRyyVJwoLbMbdpfabQ4HPP8VsDFGGh6U0hcrKsbl5wMZyZnVVhEISJ1JNwh/+N/NhOLpgNzWQX/xIqaKRrmponJ5R7l8Ubkp97FjUArkx9TWFFRHNKR4JMCNi1VgjNH5kXx5Nb/X+sc0UTAKWtK1yWb9xxuSumltXNu7mhQM8OcNoLoIz0CJY4yGJ4qyxwuyx6ZkjxWUHS+UgnJB2bEp2eMF5fJzl9S9MO97WZJikUApUAcUj7ihOh4OqiVS+To+vd+sbU0ha1khq6LvuPzoVpBl90v5fOUHpm/A276rFI7XzITjzqSsUGP+L6HoGI1NORrNlwLwVNENwfm529xQPDrn9WTRLPKdK6+NUEBqjYbU0RRUezSotqaQ2kvP28vPS49NQTWHCNvTRieL+ll/rhyYUzm3FWNta1i3b23XzWvj2t0bUyxMWxCAldWY/1Ki4UwWnJkgXArB2VIotsfcYDwwUVBhTi4OWFJ7U0iJ5pDWtkb0oTUxJWIhJWJhdTWH1NUcUktbu86l7HK1sRy4yq/dbX3DUxqdmlAu72hi7oHmCFpyw3TYUtxyFDd5xQoTasmPKj4+rFhuQPHhjOJjQ2opjCteGFe8MKFYyFK8o1XR3vWyPnxLZeU40SMrGl3BP2XvOMZofMqZ9WdfWQUuB+BZfy+j+Zm/r/El/j4Clip/CIoE1dkcnfnBZ9Z7s1+3tLfr1xdtDU0UNDRRdB8nixqaKGp4sqCLmXENThQXvR7CAascpGdCdagUvIPqKAXutqj7vJ6mUys6Rm9lxssLlJzNTsgxUiwc0A29Mf3O9W47Rm9rxOuhAmgwhGfUNGOMhieL5SpxZk7FOFtqrxidVy2WmkKWuprDSsRC+tCamLpiITcYN4fLzzubQkv+2jeZbFFXYOKyxl1wjMbyRY1OTCmXsZWzsxoZHFJuOKfc6LhyE3mNThY15ljKhZqVCzXJDjVrNBzXWDipfFdI6lr8+4cDluLBgOK5oOL5gOIZSy2RTCmQl4LdrCA4sy2gWDiocHB1q53GGE0UZlofRmf90FG5beEK8Fje0WK132nuOc6cb29LWPFIkxt4w7P+LBZ4faUV4GSyRV3W0tfGZMHRcClUzwTsUuCe9fzc8KQGJ4rKL1LpjgatWRXsWdXsueG79L7flpG+OJIvz7f884tjGptyFLCknYkmffbDCd20Nq5rEs20YgDwFOEZvpUvOpXV4fGpUuV4VjgeL6jgVAYJS1JHU1BdsbB6WsO6fk2zG4abSxXj0vNYeOV/JV7uO54zU4WV6Vc8fVHxS/Udl2/G65jXd5wvOhrLOxqdGyQrKt4zFfDRfFH9o/nytkW7DEqiQWvBSmq59SQ603ISn9XrHQlaGitVgN0xLTye2cF4etzOEmNqCgUqgm0iFtKmjmA5DM/9YaBl1nibwwFfB65oKKDuUGDZN7RNFBwNTRQ0OFHU8ERRQ5OzKtulwJ0dL+i9gUkNTRbn/TcyLRYOqK0UsjtKFeyFw3dIbdGgQlX+MxybKuq10qwYp/tyujg6JUlaEw/pE5vbdNPamG7oiaslSisGAP8gPGPVGWM0MlmcFYoX7jEemSzO+2w0aCkRC6krFtau7mb3eXOoopWiszlU9X/kL8XJjch88E51+o67upc1lVskGFCkOaCO5sv/T9gYo3zRzGtlWPT1VFEDEwWdG86X31sq6C48Zquiyt3eFNS61khlxXeRFohYpPrBrZY1hQJqaomoZxkTnBjj9nMPTxY1WArX08+HJ9xq9+BkQf2jU3orM66hycV/kGmJBMptI7NDdUfpsb3UStLWFFRrJDjvB5aiY/ROdqK8ot+vMuNyjPtboN09cf32dV26aW1c61rD9HoD8C3CM6pqqji7t3hWMJ7VY5wdK2hqgWpxe1NQiVhI3fGwrutuVqI5VGqfCJefx1ehWnwpJjcivfuWzHtvyrz3lvT+20qPjlTuFIu7QXjtBlm7/92cvuM1siLe9h1blqVoyFI0FFAidvmfN8ZovDBTXc7NqoDni45ic26KbAkHFYsEFPFZi0CjsKzp3yIEtXYZ/cGOMcrlnUu2jwxNFnV+OK83UuManiwu2DJjSWqdFbKjQUtvZsY1mndkSdre1aTfuT6hm9fGdW2yedVbhQDgShGecVmGJgo6a0/MCsOVFePhBarFkVK1ONEc0rXJ5lL7RCkYl3qOV7tavBymMCWde1/m3Tel996SefctKVWaPcEKSOs2yrrpY4rvuFZjzS0zU7vF6nu+Y8uyFAsHFQsHmTe3DgUsS63RoFqjQW1Yxv5Fx/0txlBF+8ic5xMFZSaL+uiGVt20Nq6bemNqa+KfHwC1if97YVlG80U990ZW//yrbMW0XO1NQSWaQ0rGQrom0Vxqn5hupXArxrUwt60xRrJTbjW5VFnWr9+RCm4Pptq7pK3XyPqNA7K2XStt3iGrqVmSFE8mNZ7JeDh6wDvBwPRNiiFJ9TmbCwDMRnjGJU0WHP3zmwP63hu2xvKOPrGlTf/tjg51x8PqbA7V7K9azfiY9P7bMu+W2i/efVMaGXLfjESkTTtk7btT1tZrpG3XunMg+/wHAAAAsPIIz1jQVNHo/35nUP/wWkYDE0Xduj6uz93YrS2dTV4P7bKZYlG68IFbTX73LTcs9/2bZEoV9N717op6266RtfVaaf3mhl0sBAAAXBoJARWKjtH/++th/f3PM7o4OqXru5v1tU90a9eaK7izzCNm0JbefVNmOij/+qw0WZprt6VV2nqtrFt/ww3KW3bKitd3jzIAAKgewjMkuT2/r5wf1dNnMvr10KS2dkb1jd/coFvWxX3drmAmJ6Vfn3Vnv3j3Lem9t6SBUv9xMCRt2ibrtgPStmvdFozuXl+fDwAA8DfCM/R6/5j+y5m03syMa11rWP/ptnW6bXOrAj4LmcZxpP7zbkh+9023DeP8r2cWGenulbXzevfGvm3XShu3yQozGwQAAKgewnMDeyc7oafOpHW6L6dEc0hf+Wiv9m1r982UcWZkaGZO5XfflN4/K43n3Deb49LWnbI+/Rm3/WLbNbJa270dMAAAqHuE5wZ0bnhSz/wsoxc/GFFrNKgv3tKtT+/sVDTk3SIWZmpK+uAdt0f5vVKvcvqi+2YgIG3YIusjn3D7lbddI/WslxVg0Q0AALC6CM8NJJ2b0rOvZfTDd4cUCQb0P+5O6OCuLsXCSy8HXU3GGCl9cWbxkffekv7tXalQcHfoTLqV5E992u1T3rxDVpT5YwEAgPcIzw1gaKKg7/7C1g/eGpSRdOc1nfrMhxPqWKUVvszYqPTerDmV33tLGh1234w2ueF4/2+7fcrbrpHVkViVcQEAAFwuwnMdG5sq6h9/mdX3fzmgfNHR7Vvb9bu7k1rTsnI30ZlCQTr/69KcyqWwfPG8+6ZlSWs3yrrxI25I3nattHaTrODqVr4BAACuFOG5DuWLjn7w1qD+91/YGpks6uMbW/X7Nya1sb26rQ/GGCmbkUo39LlzKr8jTeXdHdo63JkvPna7G5S37JTVXDvzRQMAAMxFeK4jRcfo5LtDeva1jOyxgm7qjelzN3VrZ6K5ascwqT6Zn75YmlP5TWlowH0jFJY2b5f1qU+XVuq7RkqsYU5lAABQVwjPdcAxRv/1gxE9/bOMLozkdU2iSX/08bW6oTde/WM9/W3plz+T1qyTteummaC8YYusEHMqAwCA+kZ4rmHGGJ3uy+npn6X1TnZSm9ojeuCT6/WRDS0rV/FN9cn6yCcV+J//08p8fwAAAB8jPNeoX6bH9NSZtH6RGteaeFh/9PG1+uSWNgVXcIETUyy6S18ne1bsGAAAAH5GeK4x7w9M6OmfZfTK+VF1NAX1pT09umNHh8LBVegtHsy6S2Enulf+WAAAAD5EeK4RF0fyeubnGf34/WHFwgF97sak/sN1XWpazVUB7X5JkpWg8gwAABoT4dnnsuMF/cNrGb1wdlDBgKX//vou/c71CbVGV39uZGOn3SeJNat+bAAAAD8gPPvU6GRR/8cbtv7PNwdUdIzu2NGhz+5OqqvZw7+yUuWZtg0AANCoCM8+M1Fw9M+/yuq5N7Iam3L0yS1t+r0bklrbGvF6aFImJbV3ygr7YCwAAAAeIDz7xFTR6IWzg/qH1zManCjq1vUt+tyNSW3pbPJ6aGUmm5a6qDoDAIDGRXj2WNEx+vH7w/r71zLqH53Sh9c063/9ZLd2dftwGetMv6wtO70eBQAAgGcIzx4xxujUuVE9/bO0PhjKa3tXVPfevkE3r437cklr4zhSNiPdstfroQAAAHiG8OyBn1/M6akzab1lT2hda0T/y2+s08c3tSrgw9BcNjQgFQtSkpk2AABA4yI8r6K37XE9fSatMxfHlIiF9JWP9mr/tvYVXRWwauyUJMlimjoAANDACM+r4NzQpJ7+WUb/9d9G1BoN6u5b1ujT13QoElzFBU6ukimFZ+Z4BgAAjYzwvILSuSn9/c8z+n/eG1IkGNDv7k7orl1dioVXf4GTq0Z4BgAAIDyvhMGJgr77uq0fvD0oS9JvXdupz3woofamGv7jtlNSS5usqH+mzgMAAFhtNZzm/CeXL+r7v8zqn341oHzR0b5t7frd3Ul1x8NeD+2qGTtF1RkAADQ8wnMVTBYc/eDtAX33dVsjeUe3bWrVf7wxqQ1tUa+HVj12Slq32etRAAAAeIrwfBUKjtHJd4b0v72WkT1e0E1r4/r8jd3akaiv1gZjjGSnZd1wq9dDAQAA8BTh+Qo4xuj/+/WI/v7naV0YmdK1ySZ99ba12t0T93poK2NkUJrKS120bQAAgMZGeL4Mxhi9eiGnp36W1nsDk9rcHtUDn1qvj6xv8eWqgFWTKc3xzAIpAACgwRGel+mXqTH9lzNpvZEeV09LWF/du1af2NxWGwucXCVjp90niW5vBwIAAOAxwvMSPhic1F+++Au99P6AOpuC+vKtPfr32zsUDtZ/aC6z+91H2jYAAECDIzwvoW8kr9f6hvX5m7r1W9d2qilUO6sCVo2dlmItsmJ12tMNAACwTITnJXxkQ4u+e/2tmhgZ9HoonnHneKZlAwAAoAHLqJfHsiy1RBv8Zww7JSV6vB4FAACA5wjPuCR3jueULCrPAAAAhGcsYXREmpyQmKYOAACA8IwlZEtzPDPTBgAAAOEZSygtkELlGQAAgPCMJRi7FJ4ThGcAAADCMy7NTklNzVKsxeuRAAAAeI7wjEty53heI8tqoBUVAQAAFkF4xqWVwjMAAAAIz1iKnZZFeAYAAJBEeMYlmLFRaTxH5RkAAKCE8IzFlaaps5imDgAAQBLhGZdSWiBFLJACAAAgifCMSzAskAIAAFCB8IzF2WkpEpFa2rweCQAAgC8QnrEoY/dLiR7meAYAACghPGNxdlpKdHs9CgAAAN8gPGNxdoo5ngEAAGYhPGNBZmJMyo1IiR6vhwIAAOAbhGcszE67j7RtAAAAlBGesbDpBVJo2wAAACgjPGNBZnqBFMIzAABAGeEZC8ukpFBYauvweiQAAAC+QXjGwuyU1NUtK8AlAgAAMI1khAUZO8Wy3AAAAHMQnrEw5ngGAACYJ7Scnc6cOaMnn3xSjuNo//79OnjwYMX7o6OjeuKJJ9Tf369wOKz77rtPmzZtkiQ9//zzOnnypIwx2r9/v+68805J0vvvv6+//du/VT6fVzAY1B/8wR9ox44dVT49XAkzOSmNDHGzIAAAwBxLVp4dx9GxY8f0wAMP6OjRo3rxxRd17ty5in2ee+45bdmyRY8++qgOHz6s48ePS5I++OADnTx5Ug899JAeeeQRvfrqq+rr65MkPf300/rMZz6jRx55RJ/97Gf19NNPV//scGWy03M8E54BAABmWzI8nz17Vr29verp6VEoFNLevXv1yiuvVOxz7tw57d69W5K0fv16pdNpDQ4O6vz589q5c6ei0aiCwaB27dqlU6dOSZIsy9L4+LgkaWxsTJ2dndU+N1wpu18SczwDAADMtWTbRjabVSKRKL9OJBJ6++23K/bZvHmzXn75ZV133XU6e/as0um0stmsNm7cqGeffVYjIyOKRCI6ffq0tm/fLkn6whe+oCNHjuipp56S4zh68MEHFzz+iRMndOLECUnSww8/rGQyecUnewS+WTQAACAASURBVKVCoZAnx/XK2MSYRiR17bxOwQY67yvVaNcHlo9rA4vh2sBiuDb8b8nwbIyZt82yrIrXBw8e1PHjx3X//fdr06ZN2rp1qwKBgDZs2KC77rpLDz74oJqamrR582YFSlOfvfDCC/rCF76gj33sY3rppZf013/91/qzP/uzecc6cOCADhw4UH6dyWQu+ySvVjKZ9OS4XnE+eFcKBpV1jKwGOu8r1WjXB5aPawOL4drAYrg2/GPdunULbl8yPCcSCdm2XX5t2/a8FotYLKZDhw5JcsP24cOHtWaN+yv/ffv2ad++fZKkZ555plzF/tGPfqQvfvGLkqSPf/zj+pu/+ZvLPSeslMz0HM9Br0cCAADgK0v2PG/fvl19fX1KpVIqFAp66aWXtGfPnop9crmcCoWCJOnkyZPatWuXYrGYJGloaEiSWzE+deqUbrvtNklSV1eX3njjDUnS66+/rt7e3uqdFa6Kyaalrm6vhwEAAOA7S1aeg8Gg7r77bh05ckSO4+j222/Xxo0b9cILL0iS7rjjDp0/f16PP/54uVXj3nvvLX/+scce08jIiEKhkO655x61tLRIkr785S+Xp78Lh8P68pe/vEKniMuWScn68M1ejwIAAMB3LLNQU7OPXbhwYdWP2Uj9R2ZqSs6h/0HWf/g9BX7797weTk1opOsDl4drA4vh2sBiuDb8Y7GeZ1YYRKXpOZ5ZmhsAAGAewjMq2SlJkpXo8XggAAAA/kN4RgVTCs9KcMMgAADAXIRnVLJTUiAgdTJBOwAAwFyEZ1SyU1JHQlaQOZ4BAADmIjyjgsmkuFkQAABgEYRnVMqmZHURngEAABZCeEaZKRSkgSyVZwAAgEUQnjFjICMZh6W5AQAAFkF4xozpOZ6TzPEMAACwEMIzyoxdWl0wQdsGAADAQgjPmGH3S5YldTHHMwAAwEIIz5hhp6X2LlmhsNcjAQAA8CXCM8qMnWJZbgAAgEsgPGNGpl9WgpsFAQAAFkN4hiTJFIvSoE3lGQAA4BIIz3ANZqVikQVSAAAALoHwDNf0HM8szQ0AALAowjMklW4WlKg8AwAAXALhGa7p8MzS3AAAAIsiPMNlp6S2DlmRqNcjAQAA8C3CMyRNz/FMywYAAMClEJ7hslOyCM8AAACXRHiGjONI2TSVZwAAgCUQniEND0iFAuEZAABgCYRnSJnSHM9MUwcAAHBJhGfMzPHMAikAAACXRHjGzBzPCeZ4BgAAuBTCMyQ7LbW0yWpq9nokAAAAvkZ4hozdz82CAAAAy0B4hlt5JjwDAAAsifDc4IwxpQVS6HcGAABYCuG50Y0MSlN5KdHj9UgAAAB8j/Dc6Oy0JFF5BgAAWAbCc4MzpQVSxAIpAAAASyI8N7osC6QAAAAsF+G50WVSUiwuKxb3eiQAAAC+R3hucMZOMU0dAADAMhGeGx3hGQAAYNkIzw3MneM5LYvwDAAAsCyE50aWG5Emx6k8AwAALBPhuZGV53gmPAMAACwH4bmR2f3uI+EZAABgWQjPDYwFUgAAAC4P4bmRZdNStFmKtXg9EgAAgJpAeG5gJtMvJdfIsiyvhwIAAFATCM+NzE7T7wwAAHAZCM+NzE7JSnR7PQoAAICaQXhuUGZsVBrPSYker4cCAABQMwjPjao8xzOVZwAAgOUiPDcquzRNHZVnAACAZSM8NyhTDs9UngEAAJaL8NyoMikpEpFa270eCQAAQM0gPDcok01JXczxDAAAcDkIz40qk2JZbgAAgMtEeG5U2ZSsLsIzAADA5SA8NyAzMS6NjlB5BgAAuEyE50ZUmuOZpbkBAAAuD+G5Edn9kiSL8AwAAHBZCM8NyFB5BgAAuCKE50Zk90uhkNTW4fVIAAAAagrhuRFlSnM8B/jrBwAAuBykpwZksmmW5QYAALgChOdGlOmXlezxehQAAAA1h/DcYEx+UhoZkrqoPAMAAFwuwnOjmZ5pgwVSAAAALhvhudHYKUmSlaBtAwAA4HIRnhuMKYVnbhgEAAC4fITnRmOnpGBQ6ujyeiQAAAA1h/DcaOyU1JmUFQh6PRIAAICaQ3huMMZOsSw3AADAFSI8Nxo7JYvwDAAAcEUIzw3ETE1Jg1kqzwAAAFeI8NxIBkpzPBOeAQAArgjhuZFkSnM8s0AKAADAFSE8N5CZOZ4JzwAAAFeC8NxI7JRkBaSOhNcjAQAAqEmE50Zip6XOhKxQyOuRAAAA1CTCcwMxdj/LcgMAAFwFwnMjsVOyEj1ejwIAAKBmEZ4bhCkUpIEslWcAAICrQHhuFAMZyTjMtAEAAHAVCM+NIusukMLS3AAAAFeO8NwgTGmBFLFACgAAwBUjPDcKOyVZltRJzzMAAMCVIjw3CjsltXfKCoe9HgkAAEDNIjw3CGOnuFkQAADgKhGeG4Wd4mZBAACAq0R4bgDGKbpT1RGeAQAArgrhuREMZqVikfAMAABwlQjPjaA0TR1tGwAAAFeH8NwATLY0xzPhGQAA4KoQnhvB9AIpCeZ4BgAAuBqE50aQTUttHbIiUa9HAgAAUNMIzw3AZPpp2QAAAKgCwnMjsNPcLAgAAFAFhOc6ZxxHyqbodwYAAKgCwnO9Gx6UCgUp0eP1SAAAAGoe4bne2dNzPFN5BgAAuFqE5zpnMv3uEyrPAAAAVy20nJ3OnDmjJ598Uo7jaP/+/Tp48GDF+6Ojo3riiSfU39+vcDis++67T5s2bZIkPf/88zp58qSMMdq/f7/uvPPO8ud+8IMf6F/+5V8UDAZ1yy236HOf+1wVTw2S3GnqJHqeAQAAqmDJ8Ow4jo4dO6avf/3rSiQS+tM//VPt2bNHGzZsKO/z3HPPacuWLbr//vt1/vx5HTt2TN/4xjf0wQcf6OTJk3rooYcUCoX00EMP6ZZbbtHatWv1+uuv6yc/+YkeffRRhcNhDQ0NreiJNqxMSmppldXU7PVIAAAAat6SbRtnz55Vb2+venp6FAqFtHfvXr3yyisV+5w7d067d++WJK1fv17pdFqDg4M6f/68du7cqWg0qmAwqF27dunUqVOSpBdeeEF33XWXwuGwJKm9vb3a5waVluamZQMAAKAqlqw8Z7NZJRKJ8utEIqG33367Yp/Nmzfr5Zdf1nXXXaezZ88qnU4rm81q48aNevbZZzUyMqJIJKLTp09r+/btkqS+vj796le/0rPPPqtwOKzPf/7z2rFjx7zjnzhxQidOnJAkPfzww0omk1d1wlciFAp5ctxqyAzYCm3cqo4aHX8tqOXrAyuLawOL4drAYrg2/G/J8GyMmbfNsqyK1wcPHtTx48d1//33a9OmTdq6dasCgYA2bNigu+66Sw8++KCampq0efNmBQJusdtxHI2OjurIkSN65513dPToUT3++OPzvveBAwd04MCB8utMJnNFJ3o1ksmkJ8e9WsYYOek+OdffVJPjrxW1en1g5XFtYDFcG1gM14Z/rFu3bsHtS4bnRCIh27bLr23bVmdnZ8U+sVhMhw4dkuQGtsOHD2vNGndFu3379mnfvn2SpGeeeaZcxe7q6tJHP/pRWZalHTt2KBAIaGRkRG1tbVdweljQyJCUz7M0NwAAQJUs2fO8fft29fX1KZVKqVAo6KWXXtKePXsq9snlcioUCpKkkydPateuXYrFYpJUvhEwk8no1KlTuu222yRJt956q15//XVJ0oULF1QoFNTa2lq9M4NkuzNtsDQ3AABAdSxZeQ4Gg7r77rt15MgROY6j22+/XRs3btQLL7wgSbrjjjt0/vx5Pf744+VWjXvvvbf8+ccee0wjIyMKhUK655571NLSIsmtSH/729/Wn/zJnygUCukrX/nKvJYNXCV7eo5nwjMAAEA1WGahpmYfu3Dhwqofs1b7j5z/63sy3z2uwH9+Rlasxevh1K1avT6w8rg2sBiuDSyGa8M/Fut5ZoXBemanpOY4wRkAAKBKCM91zGRStGwAAABUEeG5nmXTUpLwDAAAUC2E5zpljJEyKWbaAAAAqCLCc70aG5Umx2nbAAAAqCLCc73KpCRJVqLb44EAAADUD8JzvbLd8KxEj7fjAAAAqCOE5zplyuGZyjMAAEC1EJ7rlZ2Sos1SnCXPAQAAqoXwXKeMnZIS3Sx5DgAAUEWE53rFAikAAABVR3iuV9mULBZIAQAAqCrCcx0yYzlpLEflGQAAoMoIz/UoOz3HM+EZAACgmgjP9SgzPU0d4RkAAKCaCM91yNhp9wnhGQAAoKoIz/XI7pciEam13euRAAAA1BXCcx0ydlrqWsMczwAAAFVGeK5HpQVSAAAAUF2E53pk98tK9Hg9CgAAgLpDeK4zZmJcGh2h8gwAALACCM/1hpk2AAAAVgzhud5ML5CSpG0DAACg2gjPdcaUF0ihbQMAAKDaCM/1xk5JoZDU1un1SAAAAOoO4bne2Cmpq1tWgL9aAACAaiNh1Rljp7hZEAAAYIUQnuuNnZJFeAYAAFgRhOc6YvKT0vAglWcAAIAVQniuJ1nmeAYAAFhJhOd6UpqmjrYNAACAlUF4riOmtECKkoRnAACAlUB4rieZlBQMSh1dXo8EAACgLhGe64mdljqTsgJBr0cCAABQlwjPdcTY/dwsCAAAsIIIz/XETnOzIAAAwAoiPNcJMzUlDWWlRLfXQwEAAKhbhOd6MZCWjJESPV6PBAAAoG4RnuuF7S6QYlF5BgAAWDGE5zphMv3uE3qeAQAAVgzhuV5k05IVkDqTXo8EAACgbhGe60UmJXV2yQqFvB4JAABA3SI81wmTTdGyAQAAsMIIz/Uik2KOZwAAgBVGeK4DpliUBm0qzwAAACuM8FwPBjKS4xCeAQAAVhjhuR7YKUmibQMAAGCFEZ7rgCmFZyrPAAAAK4vwXA8ypfDcxeqCAAAAK4nwXA+yKamjS1Y47PVIAAAA6hrhuQ6YDHM8AwAArAbCcz3IprlZEAAAYBUQnmuccYpSNi0l6HcGAABYaYTnWjc4IBWLUqLH65EAAADUPcJzrSvP8UzlGQAAYKURnmucsfvdJ1SeAQAAVhzhudbZafeROZ4BAABWHOG51tkpqbVdVjTq9UgAAADqHuG5xhk7JSVp2QAAAFgNhOdal0nJomUDAABgVRCea5hxHHeO5yQLpAAAAKwGwnMtGx6UClMszQ0AALBKCM+1rDzHM+EZAABgNRCea5gphWcqzwAAAKuD8FzLpud4ZnVBAACAVUF4rmV2vxRvldUU83okAAAADYHwXMOMnaJlAwAAYBURnmuZzTR1AAAAq4nwXKOMMZLdL6uL8AwAALBaCM+1anRYyuepPAMAAKwiwnOtykzP8cxMGwAAAKuF8FyrstNzPPd4Ow4AAIAGQniuUSYzHZ6pPAMAAKwWwnOtslNSc1xWrMXrkQAAADQMwnONcud4puoMAACwmgjPtYoFUgAAAFYd4bkGuXM8p2QluVkQAABgNRGea9HYqDQxLnXRtgEAALCaCM+1yC7N8cwCKQAAAKuK8FyLytPUEZ4BAABWE+G5Bpks4RkAAMALhOdalElJ0SYp3ur1SAAAABoK4bkGGTstJdbIsiyvhwIAANBQCM+1yO6nZQMAAMADhOdaZKdkEZ4BAABWHeG5xpixnDSWk5imDgAAYNURnmvN9EwbXYRnAACA1UZ4rjV2WhILpAAAAHiB8FxjTHmBFJbmBgAAWG2E51qTTUnhiNTa4fVIAAAAGg7hucaYTEpKdDPHMwAAgAcIz7XGTjHHMwAAgEcIz7WGOZ4BAAA8Q3iuIWZyQhodpvIMAADgEcJzLbGnZ9ogPAMAAHiB8FxLSuGZtg0AAABvEJ5riJmuPLNACgAAgCcIz7Ukk5JCIamt0+uRAAAANCTCcy3JpqWublkB/toAAAC8QAqrISbTz82CAAAAHgotZ6czZ87oySeflOM42r9/vw4ePFjx/ujoqJ544gn19/crHA7rvvvu06ZNmyRJzz//vE6ePCljjPbv368777yz4rP/9E//pKefflrf+c531NbWVqXTqlPZtKzde7weBQAAQMNasvLsOI6OHTumBx54QEePHtWLL76oc+fOVezz3HPPacuWLXr00Ud1+PBhHT9+XJL0wQcf6OTJk3rooYf0yCOP6NVXX1VfX1/5c5lMRq+99pqSyWR1z6oOmfykNDQgJbq9HgoAAEDDWjI8nz17Vr29verp6VEoFNLevXv1yiuvVOxz7tw57d69W5K0fv16pdNpDQ4O6vz589q5c6ei0aiCwaB27dqlU6dOlT/3d3/3d/r93/99WZZV5dOqQ9m0+5jo8XYcAAAADWzJto1sNqtEIlF+nUgk9Pbbb1fss3nzZr388su67rrrdPbsWaXTaWWzWW3cuFHPPvusRkZGFIlEdPr0aW3fvl2S9JOf/ERdXV3asmXLJY9/4sQJnThxQpL08MMPe1KlDoVCnlfHJ8+9o0FJHdt2KEKl3lf8cH3An7g2sBiuDSyGa8P/lgzPxph52+ZWig8ePKjjx4/r/vvv16ZNm7R161YFAgFt2LBBd911lx588EE1NTVp8+bNCgQCmpyc1Pe+9z19/etfX3KABw4c0IEDB8qvM5nMcs6rqpLJpCfHnc151/2BZSjUJMvjsaCSH64P+BPXBhbDtYHFcG34x7p16xbcvmR4TiQSsm27/Nq2bXV2Vs4zHIvFdOjQIUlu2D58+LDWrHFnhdi3b5/27dsnSXrmmWeUSCTU39+vVCql+++/v/w9v/a1r+kv/uIv1NHRcQWn1wDstBQMSh1dXo8EAACgYS3Z87x9+3b19fUplUqpUCjopZde0p49lTM+5HI5FQoFSdLJkye1a9cuxWIxSdLQ0JAkt2J86tQp3Xbbbdq0aZO+853v6Fvf+pa+9a1vKZFI6C//8i8JzpeSSUkdCVnBoNcjAQAAaFhLVp6DwaDuvvtuHTlyRI7j6Pbbb9fGjRv1wgsvSJLuuOMOnT9/Xo8//ni5VePee+8tf/6xxx7TyMiIQqGQ7rnnHrW0tKzc2dQxk01JSW4WBAAA8JJlFmpq9rELFy6s+jH90H9UvP+LsnbdqMDdf+TpODCfH64P+BPXBhbDtYHFcG34x2I9z6wwWANMYUoaykpJVhcEAADwEuG5FmQzkjEszQ0AAOAxwnMtsFOSJIvwDAAA4CnCcw0wpfBM5RkAAMBbhOdaYKckKyB1JpbeFwAAACuG8FwL7JTU2SUrFPZ6JAAAAA2N8FwDjJ2SumjZAAAA8BrhuRbYaVlMUwcAAOA5wrPPmWJRGshQeQYAAPABwrPfDdqS47BACgAAgA8Qnv0uMz3Hc7fHAwEAAADh2edm5nju8XYgAAAAIDz73nR47kp6Ow4AAAAQnn3P7pfau2SFI16PBAAAoOERnn3O2GluFgQAAPAJwrPf2SlZXdwsCAAA4AeEZx8zTlHKZqg8AwAA+ATh2c8GB6RigQVSAAAAfILw7GfZ0hzPVJ4BAAB8gfDsYyYzPccz4RkAAMAPCM9+Vp7jmfAMAADgB4RnP7NTUmu7rGjU65EAAABAhGdfM3aKlg0AAAAfITz7mZ2WRXgGAADwDcKzTxnHcds2CM8AAAC+QXj2q5EhqTDFAikAAAA+Qnj2q0y/JMlipg0AAADfIDz7lMmm3SdUngEAAHyD8OxX5QVSur0dBwAAAMoIz36VTUnxVllNMa9HAgAAgBLCs0+ZTIqqMwAAgM8Qnv2KaeoAAAB8h/DsQ8YYyU7JSvR4PRQAAADMQnj2o9FhKT9J2wYAAIDPEJ79yHZn2rCYpg4AAMBXCM9+VArPYoEUAAAAXyE8+5CZDs9UngEAAHyF8OxHmZTUHJMVa/F6JAAAAJiF8OxDJptmmjoAAAAfIjz7Uaaf8AwAAOBDhGefMcZI2bQswjMAAIDvEJ79ZiwnjY9ReQYAAPAhwrPf2P2SROUZAADAhwjPfmOn3UemqQMAAPAdwrPPmFLlmQVSAAAA/Ifw7Dd2Woo2SS2tXo8EAAAAcxCefcZkUlJXtyzL8nooAAAAmIPw7DfZlJTs8XoUAAAAWADh2W8yKVmJbq9HAQAAgAUQnn3EjI9JY6PM8QwAAOBThGc/sVPuY4K2DQAAAD8iPPtJKTzTtgEAAOBPhGcfMdOVZxZIAQAA8CXCs5/YKSkckVo7vB4JAAAAFkB49hFjp6QEczwDAAD4FeHZTzIpluUGAADwMcKzn2TTsuh3BgAA8C3Cs0+YyQlpZEjqYqYNAAAAvyI8+0U27T6yNDcAAIBvEZ79IjM9xzNtGwAAAH5FePYJY/e7TwjPAAAAvkV49gs7LQVDUnun1yMBAADAIgjPfmGnpK6krAB/JQAAAH5FUvMJY6e4WRAAAMDnCM9+YadkMU0dAACArxGefcBM5aWhAYkFUgAAAHyN8OwHdmmOZ5bmBgAA8DXCsx9kS3M8U3kGAADwNcKzD5jSAinM8QwAAOBvhGc/sFNSICB1JLweCQAAAC6B8OwHdkrqTMoKBr0eCQAAAC6B8OwDxk7RsgEAAFADCM9+YKdlEZ4BAAB8j/DsMVOYkgZtKs8AAAA1gPDstQFbMoYFUgAAAGoA4dlrmX5JYmluAACAGkB49pjJllYXTPZ4OxAAAAAsifDstUxKsiypkzmeAQAA/I7w7DW7X+pIyAqFvR4JAAAAlkB49pix08y0AQAAUCMIz16zU7IS3CwIAABQCwjPHjLFojSQkRLcLAgAAFALCM9eGrQlx5GoPAMAANQEwrOX7JQkyWKBFAAAgJpAePaQybjhWV2EZwAAgFpAePZSthSeadsAAACoCYRnL2VSUnunrHDE65EAAABgGQjPHjJZ5ngGAACoJYRnL2X6ZRGeAQAAagbh2SPGcaRshsozAABADSE8e2VoQCoWCM8AAAA1hPDsFbtfkmjbAAAAqCGEZ48YO+0+YYEUAACAmkF49krGrTyrizmeAQAAagXh2SvZtNTaLiva5PVIAAAAsEyEZ4+YTIqqMwAAQI0hPHslm6LfGQAAoMYQnj1gjJHstKxEj9dDAQAAwGUgPHtheFCayksJ2jYAAABqCeHZC3ZKkqg8AwAA1JjQcnY6c+aMnnzySTmOo/379+vgwYMV74+OjuqJJ55Qf3+/wuGw7rvvPm3atEmS9Pzzz+vkyZMyxmj//v268847JUlPPfWUfvrTnyoUCqmnp0eHDh1SPB6v8un5kymFZyrPAAAAtWXJyrPjODp27JgeeOABHT16VC+++KLOnTtXsc9zzz2nLVu26NFHH9Xhw4d1/PhxSdIHH3ygkydP6qGHHtIjjzyiV199VX19fZKkG264QY899pgeffRRrV27Vs8991z1z86vyuGZGwYBAABqyZLh+ezZs+rt7VVPT49CoZD27t2rV155pWKfc+fOaffu3ZKk9evXK51Oa3BwUOfPn9fOnTsVjUYVDAa1a9cunTp1SpJ04403KhgMSpKuueYaZbPZap+bf9kpKdYiqznm9UgAAABwGZZs28hms0okEuXXiURCb7/9dsU+mzdv1ssvv6zrrrtOZ8+eVTqdVjab1caNG/Xss89qZGREkUhEp0+f1vbt2+cd44c//KH27t274PFPnDihEydOSJIefvhhJZPJyzrBagiFQlU97sDIoJzedUp4cC6ovmpfH6gfXBtYDNcGFsO14X9LhmdjzLxtlmVVvD548KCOHz+u+++/X5s2bdLWrVsVCAS0YcMG3XXXXXrwwQfV1NSkzZs3KxCoLHZ/73vfUzAY1Cc+8YkFj3/gwAEdOHCg/DqTySzrxKopmUxW9bjFC+eknvWenAuqr9rXB+oH1wYWw7WBxXBt+Me6desW3L5keE4kErJtu/zatm11dnZW7BOLxXTo0CFJbtg+fPiw1qxx+3n37dunffv2SZKeeeaZiir2v/7rv+qnP/2pvvGNb8wL5PXKGCNl07I+dLPXQwEAAMBlWrLnefv27err61MqlVKhUNBLL72kPXv2VOyTy+VUKBQkSSdPntSuXbsUi7n9vENDQ5LcivGpU6d02223SXJn8PjHf/xHfe1rX1M0Gq3qSfna6Ig0OcHNggAAADVoycpzMBjU3XffrSNHjshxHN1+++3auHGjXnjhBUnSHXfcofPnz+vxxx8vt2rce++95c8/9thjGhkZUSgU0j333KOWlhZJ0rFjx1QoFPTnf/7nkqSdO3fqS1/60kqco7/Y/ZIki/AMAABQcyyzUFOzj124cGHVj1nN/iPz05fk/PXDCvzZX8natK0q3xPeoj8Ni+HawGK4NrAYrg3/WKznmRUGV5kpVZ5p2wAAAKg9hOfVZqel5pgUa4zVFAEAAOoJ4XmVGTsldXU3zOwiAAAA9YTwvNrslJTs8XoU+P/bu7/YLO/6/+Ovm3aFMQTau8AyBpsgxh3gjMFsmT8zB2QH/omcumiy4IngYqJmEY2Hzp9REUOczoMFfifGM0k0MzFjyZZI4tC5BI2ZkBB1G/ZurwIC48/a+/oeFPr9TWHcLffdq73vx+OkLSvj3eXK8syHz/X5AADMgniea0UjtaFVVU8BAMAsiOc5VL51Prn4VjLsZUEAgIVIPM+lsUaSpFa3bQMAYCESz3OpmIrn1G3bAABYiMTzHCqn49nKMwDAQiSe51LRSAYWJ8veU/UkAADMgnieQ2XRSOqrnfEMALBAiee5dDWeAQBYmMTzXCpGU3NMHQDAgiWe50h56a3kwrlkSDwDACxU4nmuFKNTH608AwAsWOJ5rkxfkCKeAQAWKvE8R8rxa2c8i2cAgIVKPM+VsUZy20CyfGXVkwAAMEvieY6UxUgytMoZzwAAC5h4nivFqC0bAAALnHieK0UjtfqqqqcAAOAWiOc5UF6+nJw7a+UZAGCBE89zwUkbAABdQTzPheLqGc8uSAEAWNDE8xwox66tPK+pdhAAAG6JeJ4L442krz9ZMVj1JAAA3ALxPBfGGsnQcGqL/OcGAFjI1NwcKIuGlwUBALqAeJ4LxWhq4hkA9NRdZAAAD0BJREFUYMETzx1Wvn0lOTtu5RkAoAuI504bH5v6KJ4BABY88dxpxUiS2LYBANAFxHOHlcXo1CcuSAEAWPDEc6eNNZJFi5KV9aonAQDgFonnThtvJIPDqfX1VT0JAAC3SDx3WDnmjGcAgG4hnjutaKRWX1X1FAAAtIF47qBy4u3kzHhSX1P1KAAAtIF47qTTRVI2EyvPAABdQTx3UtFI4oxnAIBuIZ47qLwaz14YBADoDuK5k4pGUqslQ8NVTwIAQBuI504aayQrhlLrv63qSQAAaAPx3EHl+KhruQEAuoh47qSxES8LAgB0EfHcIeXkZHJ6zMuCAABdRDx3ypnxpNkUzwAAXUQ8d0oxksQZzwAA3UQ8d0hZjE59Ip4BALqGeO6UqyvPruYGAOge4rlTitFkxWBqtw1UPQkAAG0injukLBrJkFVnAIBuIp47pWikNrym6ikAAGgj8dwBZbM5tW3Dy4IAAF1FPHfC2dPJ5ISXBQEAuox47oSikSSp1W3bAADoJuK5A8qr8WzlGQCgu4jnTpiOZ3ueAQC6iXjuhKKRLFue2uIlVU8CAEAbiecOKIuGVWcAgC4knjtBPAMAdCXx3GZlWSbFaGrD4hkAoNuI53Y7dyZ5+4qVZwCALiSe260YTZLUxDMAQNcRz21WjjmmDgCgW4nnditGpj6KZwCAriOe260YTZYuS+32pVVPAgBAm4nnNps649m13AAA3Ug8t1vRSOprqp4CAIAOEM9tNHXGcyM1K88AAF1JPLfThXPJ5UuJC1IAALqSeG6nYuqYupptGwAAXUk8t1Nx7Yxn2zYAALqReG6j/70gxcozAEA3Es/tVDSSJbcnS++oehIAADpAPLfR1BnPq1Or1aoeBQCADhDP7XQ1ngEA6E7iuZ2K0dTEMwBA1xLPbVK+dT65eMHKMwBAFxPP7VKMJklqLkgBAOha4rldipGpj1aeAQC6lnhuk/LqyrN4BgDoXuK5XcYaycDiZNnyqicBAKBDxHOblMWIM54BALqceG6XYtSWDQCALiee26VopFZfVfUUAAB0kHhug/LSW8mFc0l9TdWjAADQQeK5HaZP2rDyDADQzcRzOxSNJHE1NwBAlxPPbVBejecM27YBANDNxHM7FI2k/7bkPSuqngQAgA4Sz+0w1pg643mR/5wAAN1M7bVBWTS8LAgA0APEczsUDS8LAgD0APF8i8rLl5NzZ90uCADQA8TzrRq/dsazeAYA6Hbi+VYVI0mc8QwA0AvE8y0qCyvPAAC9QjzfqmIk6etPVg5WPQkAAB0mnm9VMZoMDae2qK/qSQAA6LD+Vr7p1VdfzYEDB9JsNrNt27bs2LHjHf/8/Pnz+elPf5qRkZHcdttt2bVrV9avX58kee6553L48OGUZZlt27blk5/85PTv2bdvX0ZHR7Nq1ap85StfybJly9r843Xe1BnPtmwAAPSCm648N5vNPPvss/nmN7+Zffv25Xe/+11ef/31d3zPL3/5y9x77735wQ9+kCeeeCIHDx5MkvzjH//I4cOH853vfCff//7388orr+TUqVNJkkOHDmXz5s3Zv39/Nm/enEOHDrX/p5sLY43UXJACANATbhrPJ06cyJ133pk1a9akv78/Dz30UI4ePfqO73n99dezefPmJMnatWszOjqaM2fO5I033simTZuyePHi9PX15b777svLL7+cJDl69GgefvjhJMnDDz/8X//OhaB8++3k7HhSX1P1KAAAzIGbbtsYHx9PvV6f/rper+f48ePv+J577rknv//97/OBD3wgJ06cyOjoaMbHx7Nu3br84he/yLlz5zIwMJA//elP2bhxY5Lk7NmzGRyceslucHAw//73v6/75z///PN5/vnnkyTf/e53Mzw8PLuf9Bb09/df98+dePOfKZK8554Nub2CuZgfbvR8gGeDG/FscCOejfnvpvFcluV//VqtVnvH1zt27MjBgwfz5JNPZv369Xnve9+bRYsW5e67785nPvOZfPvb386SJUtyzz33ZNGimb2juH379mzfvn3667GxsRn9/nYYHh6+7p9bnngtSXJ+8dJcqGAu5ocbPR/g2eBGPBvciGdj/rjrrruu++s3jed6vZ6iKKa/LopiesX4mqVLl2b37t1JpmL7iSeeyOrVUy/Rbd26NVu3bk2S/PznP59exV6xYkVOnz6dwcHBnD59OsuXL5/Fj1WtsmhMfWLPMwBAT7jpMvDGjRtz6tSpNBqNTExM5MiRI9myZcs7vufChQuZmJhIkhw+fDj33Xdfli5dmmRqe0YytWL88ssv56Mf/WiSZMuWLXnxxReTJC+++GI+8pGPtO+nmitFI1m0KBn01ysAAL3gpivPfX192blzZ5566qk0m8088sgjWbduXX77298mSR599NG88cYb+fGPfzy9VeOLX/zi9O/fu3dvzp07l/7+/nzhC1+YPo5ux44d2bdvX1544YUMDw/nq1/9aod+xA4qGsngcGp9zngGAOgFtfJ6m5rnsTfffHPO/8wb7T+a/N6epFZL35P/d85nYv6wP40b8WxwI54NbsSzMX/caM+zGwZvRdFIzQUpAAA9QzzPUjkxkZwed7sgAEAPEc+zdXosKZviGQCgh4jn2bp6TJ1tGwAAvUM8z1JZjE59Ip4BAHqGeJ6tYiSp1ZIhZzwDAPQK8TxbxWiyYii1/tuqngQAgDkinmepLBrJsC0bAAC9RDzPVtFIbUg8AwD0EvE8C2VzcuqoOivPAAA9RTzPxpnxZHIyqa+qehIAAOaQeJ6NsWtnPK+peBAAAOaSeJ6F8uoFKVaeAQB6i3iejWvxPCSeAQB6iXiejaKRLF+Z2sDiqicBAGAOiedZKIuGa7kBAHqQeJ6NopGaeAYA6DnieYbKZjMZH7XyDADQg8TzTP37dDIx4YIUAIAeJJ5nqhhNEts2AAB6kHieoXJsZOoT8QwA0HPE80xNX5AingEAeo14nqliNFm2PLXFS6qeBACAOSaeZ6gsRqw6AwD0KPE8U4Vj6gAAepV4noGyLJPxRmr1VVWPAgBABcTzTJw7m1y5ktTXVD0JAAAVEM8zcfWkjZoLUgAAepJ4nonpY+ps2wAA6EXieQbKa/E8ZOUZAKAXieeZGGskS+9IbekdVU8CAEAFxPMMlEXDMXUAAD1MPM+EeAYA6GniuUVlWSbFaGriGQCgZ4nnVl04l1y+aOUZAKCHiedWFaNJYuUZAKCHiedWFSNTH12QAgDQs8Rzi8qrK8+2bQAA9C7x3KqikSy5PVm6rOpJAACoiHhu0bUznmu1WtWjAABQEfHcqjFnPAMA9Drx3KqikVp9VdVTAABQIfHcguaFc8nFC0l9TdWjAABQIfHcgsnGv5LEyjMAQI8Tzy2YHJ2KZyvPAAC9TTy3oNk4NfWJC1IAAHqaeG7B5Oi/koGBZNnyqkcBAKBC4rkFk41/JfU1zngGAOhx4rkFk6P/SrwsCADQ88RzCyYbp1JzQQoAQM8TzzdRXrqY8txZtwsCACCeb6oYnfoongEAep54vpliJEls2wAAQDzfTHn5cmrLV3phEAAA8Xwziz7yf7L6/z2X2sp61aMAAFAx8QwAAC0SzwAA0CLxDAAALRLPAADQIvEMAAAtEs8AANAi8QwAAC0SzwAA0CLxDAAALRLPAADQIvEMAAAtEs8AANAi8QwAAC0SzwAA0CLxDAAALRLPAADQIvEMAAAtEs8AANAi8QwAAC0SzwAA0CLxDAAALRLPAADQIvEMAAAtEs8AANAi8QwAAC0SzwAA0CLxDAAALRLPAADQolpZlmXVQwAAwEJg5bkFe/bsqXoE5jHPBzfi2eBGPBvciGdj/hPPAADQIvEMAAAtEs8t2L59e9UjMI95PrgRzwY34tngRjwb858XBgEAoEVWngEAoEXiGQAAWtRf9QDz3auvvpoDBw6k2Wxm27Zt2bFjR9UjMQ+MjY3l6aefzpkzZ1Kr1bJ9+/Z84hOfqHos5pFms5k9e/ZkaGjI0VO8w4ULF/LMM8/kn//8Z2q1Wnbt2pX3v//9VY/FPPDrX/86L7zwQmq1WtatW5fdu3dnYGCg6rH4D+L5XTSbzTz77LP51re+lXq9nm984xvZsmVL7r777qpHo2J9fX35/Oc/nw0bNuTixYvZs2dPPvjBD3o2mPbcc89l7dq1uXjxYtWjMM8cOHAgH/rQh/K1r30tExMTuXz5ctUjMQ+Mj4/nN7/5Tfbt25eBgYH88Ic/zJEjR/Lxj3+86tH4D7ZtvIsTJ07kzjvvzJo1a9Lf35+HHnooR48erXos5oHBwcFs2LAhSXL77bdn7dq1GR8fr3gq5ouiKPLKK69k27ZtVY/CPPPWW2/lr3/9a7Zu3Zok6e/vzx133FHxVMwXzWYzV65cyeTkZK5cuZLBwcGqR+I6rDy/i/Hx8dTr9emv6/V6jh8/XuFEzEeNRiMnT57M+973vqpHYZ44ePBgPve5z1l15r80Go0sX748P/nJT/L3v/89GzZsyOOPP54lS5ZUPRoVGxoayqc//ens2rUrAwMDuf/++3P//fdXPRbXYeX5XVzvFL9arVbBJMxXly5dyt69e/P4449n6dKlVY/DPPDHP/4xK1asmP6bCfj/TU5O5uTJk3n00Ufzve99L4sXL86hQ4eqHot54Pz58zl69Giefvrp/OxnP8ulS5fy0ksvVT0W1yGe30W9Xk9RFNNfF0Xhr1CYNjExkb179+ZjH/tYHnjggarHYZ547bXX8oc//CFf+tKX8qMf/Sh//vOfs3///qrHYp6o1+up1+vZtGlTkuTBBx/MyZMnK56K+eDYsWNZvXp1li9fnv7+/jzwwAP529/+VvVYXIdtG+9i48aNOXXqVBqNRoaGhnLkyJF8+ctfrnos5oGyLPPMM89k7dq1+dSnPlX1OMwjjz32WB577LEkyV/+8pf86le/8v8Npq1cuTL1ej1vvvlm7rrrrhw7dsyLxiRJhoeHc/z48Vy+fDkDAwM5duxYNm7cWPVYXId4fhd9fX3ZuXNnnnrqqTSbzTzyyCNZt25d1WMxD7z22mt56aWXsn79+jz55JNJks9+9rP58Ic/XPFkwHy3c+fO7N+/PxMTE1m9enV2795d9UjMA5s2bcqDDz6Yr3/96+nr68u9997rqu55yvXcAADQInueAQCgReIZAABaJJ4BAKBF4hkAAFokngEAoEXiGQAAWiSeAQCgRf8Dfh2lhnTxkHQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 864x864 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "hist = pd.DataFrame(history.history)\n",
    "import matplotlib.pyplot as plt\n",
    "plt.style.use(\"ggplot\")\n",
    "plt.figure(figsize=(12,12))\n",
    "plt.plot(hist[\"accuracy\"])\n",
    "plt.plot(hist[\"val_accuracy\"])\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2729/2729 [==============================] - 1s 442us/step\n"
     ]
    }
   ],
   "source": [
    "y_pred = model.predict(X_te_embeddings['elmo_representations'][1].detach().numpy(),verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1.00827765e-05, 3.60560916e-05, 9.99953866e-01],\n",
       "       [5.35005893e-06, 1.57046743e-05, 9.99978900e-01],\n",
       "       [3.97064832e-06, 3.07798728e-05, 9.99965191e-01],\n",
       "       [5.12398128e-06, 6.85779887e-05, 9.99926329e-01],\n",
       "       [2.79996166e-05, 1.69808525e-04, 9.99802172e-01],\n",
       "       [1.06894462e-04, 9.92597103e-01, 7.29608303e-03],\n",
       "       [1.01237143e-04, 9.82100427e-01, 1.77982841e-02],\n",
       "       [5.99759442e-05, 5.55738399e-04, 9.99384284e-01],\n",
       "       [1.64341018e-05, 6.80143858e-05, 9.99915600e-01],\n",
       "       [5.23938024e-06, 6.10688403e-06, 9.99988675e-01],\n",
       "       [9.07158847e-06, 5.37968708e-06, 9.99985576e-01],\n",
       "       [9.99949455e-01, 4.39571704e-06, 4.61376148e-05],\n",
       "       [9.99974251e-01, 1.46868399e-06, 2.42922342e-05],\n",
       "       [9.99983549e-01, 1.03436457e-06, 1.53696637e-05],\n",
       "       [9.99985933e-01, 8.72866281e-07, 1.32077266e-05],\n",
       "       [9.99986529e-01, 8.16391605e-07, 1.26719051e-05],\n",
       "       [9.99986649e-01, 7.86850705e-07, 1.25212300e-05],\n",
       "       [9.99987006e-01, 7.68453560e-07, 1.23087020e-05],\n",
       "       [9.99987125e-01, 7.58966564e-07, 1.21378607e-05],\n",
       "       [9.99987245e-01, 7.53317465e-07, 1.19844681e-05],\n",
       "       [9.99987364e-01, 7.50664071e-07, 1.18802336e-05],\n",
       "       [9.99987483e-01, 7.48382547e-07, 1.17997952e-05],\n",
       "       [9.99987602e-01, 7.46887451e-07, 1.17386362e-05],\n",
       "       [9.99987602e-01, 7.47100501e-07, 1.16792089e-05],\n",
       "       [9.99987602e-01, 7.48738159e-07, 1.16247375e-05],\n",
       "       [9.99987721e-01, 7.50710910e-07, 1.15840348e-05],\n",
       "       [9.99987721e-01, 7.51549010e-07, 1.15651919e-05],\n",
       "       [9.99987721e-01, 7.51303219e-07, 1.15651255e-05],\n",
       "       [9.99987721e-01, 7.49742128e-07, 1.15945450e-05],\n",
       "       [9.99987602e-01, 7.45955674e-07, 1.16602223e-05],\n",
       "       [9.99987483e-01, 7.42898578e-07, 1.17553609e-05],\n",
       "       [9.99987483e-01, 7.58523981e-07, 1.18418438e-05],\n",
       "       [9.99987125e-01, 8.18393914e-07, 1.20136701e-05],\n",
       "       [9.99984622e-01, 1.02178080e-06, 1.42994049e-05],\n",
       "       [9.99955058e-01, 3.44811247e-06, 4.15105642e-05]], dtype=float32)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "bidirectional_1 (Bidirection (None, 35, 40)            167200    \n",
      "_________________________________________________________________\n",
      "time_distributed_1 (TimeDist (None, 35, 3)             123       \n",
      "=================================================================\n",
      "Total params: 167,323\n",
      "Trainable params: 167,323\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAn8AAAEnCAYAAADYeyX8AAAABmJLR0QA/wD/AP+gvaeTAAAgAElEQVR4nO3df2gb6Z0/8Ldud9s7yiHfctjZTXG4spdl4Xra3ULqvVJCnMCStKO0cE4t+9zcH4qR2R+034jjzpUIwcbbAxmW7R8xkv64IByJ9R7sSnTzT2yIKbGzcCBxtxwxxT0JupwEx2luobDbbp/vH+4zmRmNpBl5pNGP9wtE4vnxPM88GlkfP888z+MTQggQERER0Sh4/Y+8LgERERER9Q6DPyIiIqIRwuCPiIiIaIQw+CMiIiIaIU96XYBhsby8jF/+8pdeF4OIiGgoLSwsQFEUr4sxFNjy55K33noLW1tbXheDiLqoUqnwc27Tw4cP8fDhQ6+LQUNia2sLuVzO62IMDbb8uWhzcxNzc3NeF4OIuuTOnTuYn5/Hu+++63VR+t78/DyAo9+LRMcl7ydyB1v+iIiIiEYIgz8iIiKiEcLgj4iIiGiEMPgjIiIiGiEM/oiIiIhGCIM/IiIPxONxxONxr4vRt2q1GtbX170uBrlkfX0dqqp6XQz6AwZ/REQjSFVV+Hw+r4thqVar4caNG3jppZfg8/ng8/maBspyv/7Vr2q1GuLxuFbOZvPWlUolw/UsLS11lJ+qqtjf30cqlUIwGGx6XKFQQDAYRDAYRKFQaNhfqVSwtLSklWVnZ6dt3qlUyvBeXLhwAQsLC6jVah1dC7mLwR8RkQdWVlawsrLiWf67u7ue5d2KqqoIh8O4evUqpqenUa/Xkc1msbq6ahkACiFQrVYBANVqFUKIXhfZllqthsPDQ6ysrEAIgWw2i1AoZNm6+dFHHxl+vnTpUkd5JhIJ/PznP8fi4qJlUAcAuVwOqVQKmUwGmUwGH374IVKplLZfVVWUSiXcunUL9XodZ8+exfnz55umBxwFr4uLi4ZtgUAAy8vLCIfDbAHsAwz+iIhGjKqqhi/4fpJOpxEIBDA1NQUA8Pv9mJ2dBQCsrq5atpaNj48b/u1Hh4eH2jUB0K4pGo02HHvixAkIIbRXp0uatfsDo1KpIBQKYXl5GX6/H36/H5FIBIuLiyiVSgCO/kiQ+evfi2Ytiaqq4r333rPcNzU1hZMnTyKdTnd0PeQeBn9ERD1Wq9WQy+W0L1Dzz4VCAT6fD8FgEJVKRTtGds8Bj7vVlpaWcHBwoKVt1f1p3pZIJLSWG/12r59DrNVqiEajOHfunOX+RCKBUChke5kvVVWRy+W0a0ylUoZuRzv1rj92fX1d22+n61NPH/jJsgFALBYzbK9UKggGg4jH49jf33eUh1MPHjwAADz77LPatmeeeQbA49bHZoFnJBKx3J5Op/HGG280zXNmZgbRaJTdv14T5AoAYnNz0+tiEFEXbW5uCjd+bSqKIgBoael/3tvbE0IIUS6XBQARiUSEEELbrz+mXq+LSCQiAIhHjx4JIYSoVquGtPVp6beZfxZCiFgsJmKx2LGvTwgh5ubmxNzcnKNz8vm8ACDK5XLDPlnWWCwmAIhisWi5X09RFJFMJoUQR/WiKIpQFEXU63Vtf7t615+bzWaFEEJsb29blsGucrmsXYd838x1IF+KoohqtdpRPpLVey2E0O4dq+MVRbFMq16vCwAin8837Nve3tbqsVmesn6tzm+lk/uJmnqNwZ9LGPwRDT+3gj8hGr8crb4s7RxTLBYFAJFIJI6dlps6+bKWAZEVub1er2tBmz5wMp8nAzR94LS3tycAaEGcPK9dXWWzWctjOgmU9YG4+X2T6vW6KBaLWn3IALZTzd5rp9uFOKpXfQAtVatVQzmbpSGDR6vrboXBn6teY7cvEdEACwQCAKyfHRs0q6urbY/x+/3aM2Otug+3trYAGJ8DfOGFFwAAd+7ccVQueby5+9xOec0mJychhECxWEQsFkM0Gm14/tLv9yMQCGBlZQXJZLLl4Ipee/vtt7VnBPU++OADXLt2re358rxhuF8HGYM/IiIaKOPj4ygWiygUCk1Hj25sbDRsk4GH02BKHi90gzDkq1OBQAALCwsA0DAyVu/KlStdC/5aDSSxeqYvl8tBUZSG5xcLhQJeffVV18tH3cPgj4hoCDR7AH9YBQIB5PN5FAoFJBKJhv0ysLFqGey0rvQDa9xw+vTptsfIEbjdYFVHcqDLyy+/bDi2VCrh448/tmzdCwaDOHXqVNPBRtR/GPwREQ0wGZB0OhdcP5FBnN154BRF0eYANJubmwNwNMWKJNOdmZlxVK5kMgkAyGQyWhpurEAi08pmsy2PcVpeu2Rrnb6OPvnkE8M+4Oha7927Z5g2plQqaZNPt2oRbdY6ah7lTL3F4I+IqMfM043of5YBgT4AMrdeyalOVFVFJpOBoiiGLjzZUiQDQ/2UIfILW9/qI4MYr6d6kS1h5uBPXr9VK97s7KxlIHHx4kUoioK1tTXtvLt37yISiWB6erohvVb1fvnyZQBHz/iNjY3B5/NhYmJCC8rkFDBybjwrwWAQ6+vrWsuaqqpIJBKIxWLa3Hm5XM4whUylUsHu7q5WXslOfubrMv8fOHr+MJlM4vbt21BVFaqq4vbt20gmk5icnNTqIBwOIxqNGlr2XnzxxY7+4JDXf+bMGcfnkou8GWgyfMDRvkRDz63RvtCN9rR6WR2j31YsFrURr8lksmHkZblc1vbLKTXkVCVy9KscJRyLxbRtXk/1IqepkdOFCGFdV1aspiaRI1Dledls1lBXdutdCOP0LJFIxDAdTSwWE5FIpOn0KEI0TuGSSCQM12k+JhaLNZ1Kxk5+VtfSrP5kvoqiiO3tbcM+OR2M1cs8TY1V3mZyxLXT6Ws42tdVr/mE6NO1cAaMz+fD5uam1tVARMPnzp07mJ+f92wJMfn81CD82p6fnwcAbG5uOjpPtkJev37d0XmqqjaMQO21YDCIfD4/tPm5IR6PY2xszPH72+n9RJZeZ7cvERH1jXA4jPv37zte3cLrwG9/fx/Ly8tDm58bSqUSSqUSwuGw10UZeQz+iIgGgPk5wWEl5/FbW1uz9UxbP9jZ2cHTTz/dMAXKsOTnhoODA2xsbCCdTnseqBODP+oiJw+Pm9fY9IrXD7wTNTMxMWH5/2E0Pj6OTCaDe/fueV0UW6anp21N2zKo+bmhUCjg5s2bhkm3yTtPel0AGg6qqmJsbKzjZ5Fu3LhhOSmr3bz/8z//E//+7/+OQqEwcM/A6HVSj83m0fLiuTBz+fupbINu1OrM7/c7fi6M+hffy/7C4I9csbu727BNPydUO7du3eo4+JNzg3Wy1JKZkzJ3g1U9tiOE0IIuAKjX6551q5jLL4RArVbTWqq8LBsRER1h8EfHpqpqw9qUvSQDNjeCPy8dpx71AZVXwVWz8uu7eRj4ERF5j8/8eUhO1JlKpVCr1Rq6yOTkqz6fD8Fg0DD5pySfkwsGg9jf30ehUNDSabbUjnlbq7zMz+LJ9IPBoDZZZyKR0NaelGlbPcMngwN5TDwe76sH181ltnPttVoNhUJBO0Ze39LSkmEpKDvvhVU9Ap0/h9gv5Xei2T0i70350q+soN+nv65m97O8XlVVsbS0xGc8iWj0eDO/4PCBw0meE4mENklovV7XJg+VqtWqNimrEEJsb29rk7tKsVhMKIqiTZYpj5HpyAlT9emWy+WGba3ykhPFQjfxqkwjEokYrl+fpv48SU4WWq1WbaXh1HHPN5fZzrXL/fpj6vW6dq1yElS774XVNdideLfVe+Bl+VttN2t1j8jJYfX3jP5a5efAyf1cLBYt02vGrUmeRwEn5SU38X5y1Wv8LeYSp8Gf/IKT5JerlM1mG75k8IdZ34WwDgblMe2+jM3b2uVlJw07x8hZ6Z2k4cRxz7dbJjvHyNUTEonEsdPqtOz9VH6719XuHkkkEgKAYXWFYrGoBXpC2L+fzati2MHgzz5+WZObeD+5isGfW5wGf7KFw7zckKRvoTC/9OdblcNp8NIuL7eCP6lcLmtf4sMa/LmdVidl76fyO72uZveIDEqTyaS2Td+KLkRn97NdMvjjiy++ev9i8Oea1zjgwyM//vGP8etf/xqhUAjA0fNS+qHw8tkp0WR6h05Hxlppl5ebUqkUCoUCEokEotFo1/OjwdPqHgkEAohEIlhcXMSVK1cAAL/85S+1ReiB3tzP7777btfSHhbvvPMOAODNN9/0uCQ0DOT9RO5g8OeR06dPI5/Po1QqYWNjQ/uSM8+FdHBw0LPJPLudVy6Xw+LiIsrlsuHLephFIhGvi3AsvSr/0tISbt26ZeseiUQi2NjYwN27d/GVr3wFV69etTyum/fzzMxMV9IdJu+//z4A1hW5Q95P5A6O9vWIz+eDqqoIBAK4desWisWioZUjmUwCADKZDFRVBfB4BCPweG47N5Y/apeXW2Qr5ygEfnKk7KVLlzwuSWd6Wf79/X2cPXsWgL17RLb+hUIhpFKphiWuenU/ExENKgZ/HkokEtrUFH/2Z3+mBXQAcPnyZQBHc9eNjY3B5/NhYmJC+ytaURQAR9OAyDRkd5eebLmRX+b6xdKXlpba5qWfikV+kcp/gcdrjMryyC9Zq3VI5TGVSsUwjUitVjv2uqX6Mun/74S5DHavXcrlctoxmUwGiqJo1wzYey/M9QjYm+rF6vr7pfyt3s/9/X288soreOGFFwznW90jerK1T18+ye79TEQ0sjx+6HBoAJ2N9pUPtetHVUrlclmbAiYSiRgeahdCaNNUAEcPwNfrde3BWH0a8gH4fD4vhBDaNBj60cbN8oLpgdtm2+SD+LFYzDAtSKtj5MhO/ZQh5vLbrUurl1PN0ml37fL/+qlE5PuhZ+e9MNeREO2nemlXbi/Lb7dsMq9W94iZoijaVDRmdu5nRVGa1mkzHO1rH0dnkpt4P7nqNZ8QI7ZgZJf4fD5sbm5ibm7O83IAvRm8QUcGvc4HsfyqquIf//EfcevWrZ7me+fOHczPzw9UXXllfn4eALC5uelxSWgY8H5y1evs9iWigfPuu+9yIAERUYcY/A2R4z43R84Nep0PUvnj8bhhGbfp6Wmvi0RdxEE6w2V9fb3j57HJfQz+hsjExITl/weRfh3XVq9ep2U26HU+SOWXI4CTySRWVlY8Lo03VFXt+F7th/TtqtVquHHjBl566SXDOs9W3Pos90KtVjP8ESMHWZmVSiXD9cgBVU6pqor9/X2kUinDOutmcr3rYDBoOXCwUqlgaWlJK4vVOvNmco1u6cKFC1hYWOj7PzJHBYO/ISKEMLwGmflamr16nVa7tAfNIJX/2rVrEELg2rVrXhfFM7u7uwOdvh2qqiIcDuPq1auYnp5GvV5HNpvF6uqqZQAohEC1WgUAVKvVvr2Pa7UaDg8PsbKyAiEEstksQqGQZevmRx99ZPi50ymXEokEfv7zn2NxcdEyqAOORvmnUilkMhlkMhl8+OGHSKVS2n5VVVEqlXDr1i3U63WcPXsW58+fb5oecBS8Li4uGrYFAgEsLy8jHA6zBbAPMPgjIhoAqqoavpQHLX270uk0AoGANn+j3+/H7OwsgKPpe6xay8bHxw3/9qPDw0PDnJTymqxWOjpx4oThjzKrKY3sWFlZadlKXqlUEAqFsLy8DL/fD7/fr62gI+eQ3d3d1fLXvxfNWhJVVcV7771nuW9qagonT55EOp3u6HrIPQz+iIi6TFVV5HI5rRsvlUoZur+suizN2xKJhNbaIrfXajWtyw543NW2tLRkmCex0/QBe/NMuqVWqyEajeLcuXOW+xOJBEKhUNPuUrN29V6r1ZDL5bT6KxQK8Pl8CAaD2vyp+mPX19e1/Xa6PvXMk5HL1q9YLGbYXqlUEAwGEY/HDXNpdsODBw8AAM8++6y27ZlnngHwuPWxWeDZbPWfdDqNN954o2meMzMziEaj7P71GIM/IqIuW1hYwKeffqp1URYKBUP3l+y21CuXy4af9S04skVoYmJCe05rf38f165dQ71eBwA8//zzWgDYafq99vDhQwDAc889Z7n/+vXriMViCIVCtlY3alfv4XAYoVBIqz9FUVAul1EoFPDWW29p6dRqNYTDYZw8eRJCCPzoRz/C+fPnO15hqVKpaJP6LywsGPbJNFdXV/HKK68gGAx2LVC6f/8+AOOKOrL1tFm3rqw7q67onZ0dfOtb32rZAivfW/lek0fcnTdwdMHhJM9ENHg6meR5e3tbm9Rd2tvbEwBENpvVtsFicnLzNjvHCPF4smz95PGdpt+pTibllRNzW5Hb6/W6Ntm4fpJv83lu1ns2m7U8ptXk682YJ7W3muC/Xq+LYrGo1UcymXScj7msVvXqdLsQR/WqKErDJPDVatVQzmZpyMUIrK67FU7y7KrX2PJHRNRFW1tbAIzPo8nl7O7cudOVPAOBAADr58n62erqattj/H6/9sxYq+5DN+tdHm/uKrdTXrPJyUkIIVAsFhGLxRCNRhuetfT7/QgEAlhZWUEymWw5uKLX3n77be0ZQb0PPvjA1mAsed6g3ZvDhsEfEVEXbWxsNGyTX4D99KU+SMbHx1EsFhu6cfXcrHd5vHBplgDgKECXXb7mkbF6V65c6dp90mogidUzfblcDoqiNDy/WCgU8Oqrr7pePuoeBn9ERF0kv2CtWqiaPTTvlm6n76VAIIB8Po9CoaA9P6fXjXrXD6Jxw+nTp9seI0fgdoNVHcmBLi+//LLh2FKphI8//tiydS8YDOLUqVNNBxZR/2HwR0TURXK978PDQ22bbKnq1hJ1MkjpdH44r8ggzu48cIqiaHMAmrlZ78lkEgCQyWS0NNxYgUSmlc1mWx7TrftEttbp6+iTTz4x7AOOrvXevXuGQUGlUkmbfLpVi2iz1lHzKGfqLQZ/RERddPHiRSiKgrW1Na2F5e7du4hEIoYl6mTrjgzc9NN8yC9ZfUuNOfCQ05+oqopMJgNFUQzdep2m38upXmRLmDn4k/Vm1Yo3OztrGUjYqXd9ejJPfd5y/+XLlwEcPeM3NjYGn8+HiYkJLSiTU8C0Gv0bDAaxvr6utaypqopEIoFYLKbNnZfL5QxTyFQqFezu7jYsZWgnP/N1mf8PHD1/mEwmcfv2baiqClVVcfv2bSSTSW0EsBzpHI1GDS17L774Ykd/XMjrP3PmjONzyUXeDDQZPuBoX6Kh18loXyEej4TEH0ZAZrPZhtGS5XJZG8Waz+eFEEIoiiKy2aw2YlWO4o3FYto2mWaxWNTOTyaTrqUfi8U6GtXayejMarUqAIi9vT1tG3QjY9FiFKqiKJbptap3q3Sb5VUul7XRt5FIRJTLZW1fLBYTkUjEsgxSPp9vGOWrv07zMbFYTBSLRcu07ORndS3N6k/mqyiK2N7eNuyLRCJN09GPtm6Wt5kcca0fhW0HR/u66jWfEH26Fs6A8fl82Nzc1LoaiGj43LlzB/Pz8321hJh8pqqfygQA8/PzAIDNzU1H58kWx+vXrzs6T1XVhhGovRYMBpHP54c2PzfE43GMjY05fn87vZ/I0uvs9iUior4RDodx//59x6tbeB347e/vY3l5eWjzc0OpVEKpVEI4HPa6KCOPwR8R0YAyL1U2DOQ8fmtrax2voNFrOzs7ePrppxumQBmW/NxwcHCAjY0NpNNpzwN1YvBHRDSwJiYmLP8/6MbHx5HJZHDv3j2vi2LL9PS0rWlbBjU/NxQKBdy8ebPl0m/UO096XQAiIupMvz3n5ya/3+/4uTDqX3wv+wtb/oiIiIhGCIM/IiIiohHC4I+IiIhohDD4IyIiIhohHPDhoq2tLTz11FNeF4OIuuThw4cAjj7r1Jpcxot1RW7Y2trq2hrHo4grfLjky1/+Mj7//HOvi0FERDSUfvKTn2B1ddXrYgyD19ny55LPPvvM6yIQkU39uEwbEVGv8Jk/IiIiohHC4I+IiIhohDD4IyIiIhohDP6IiIiIRgiDPyIiIqIRwuCPiIiIaIQw+CMiIiIaIQz+iIiIiEYIgz8iIiKiEcLgj4iIiGiEMPgjIiIiGiEM/oiIiIhGCIM/IiIiohHC4I+IiIhohDD4IyIiIhohDP6IiIiIRgiDPyIiIqIRwuCPiIiIaIQw+CMiIiIaIQz+iIiIiEYIgz8iIiKiEcLgj4iIiGiEMPgjIiIiGiEM/oiIiIhGCIM/IiIiohHC4I+IiIhohDD4IyIiIhohDP6IiIiIRgiDPyIiIqIRwuCPiIiIaIQw+CMiIiIaIQz+iIiIiEYIgz8iIiKiEfKk1wUgIuq2d999F7/61a+0n4vFIgDgn//5nw3Hfec738Ff/dVf9bRsRES95hNCCK8LQUTUTT6fDwDw5S9/uekxn332Gf7hH/6hISAkIhoyr7Pbl4iG3uuvv44vfelL+Oyzz5q+AODSpUsel5SIqPsY/BHR0JudncXnn3/e8pgTJ07g29/+do9KRETkHQZ/RDT0/uZv/gbPPvts0/1f+tKXMD8/jz/6I/5KJKLhx990RDT0fD4ffvjDH+Kpp56y3P/5558jFAr1uFRERN5g8EdEI2Fubg6//e1vLff9xV/8Bb7xjW/0uERERN5g8EdEI+HrX/86/vIv/7Jh+1NPPYW///u/732BiIg8wuCPiEbG1atXG7p+f/vb37LLl4hGCoM/IhoZoVAIv/vd77SffT4f/vqv/9qyRZCIaFgx+COikfG1r30NL7/8sjbp8xNPPIGrV696XCoiot5i8EdEI2VhYQFPPPEEAOCLL77A7OysxyUiIuotBn9ENFJ+8IMf4Pe//z0A4Nvf/nbL+f+IiIYRgz8iGiknTpzQpnWZn5/3uDRERL3nE0KIXmb45S9/ue0yS0RERESj4Cc/+QlWV1d7meXrT/YyN+BoJv3vfe97mJub63XWREQAACEE/u///g9+v9/rovSFX/ziF3jnnXfw7rvvel2UvvfOO+8AAN58802PS0LDYH5+Hr/61a96nm/Pgz8AmJmZwczMjBdZExGRiVz5hL+X23v//fcBsK7IHfJ+6jU+80dEREQ0Qhj8EREREY0QBn9EREREI4TBHxEREdEIYfBHRERENEIY/BERkWvi8Tji8bjXxehbtVoN6+vrXheDXLK+vg5VVb0uhmMM/oiIaGioqgqfz+d1MSzVajXcuHEDL730Enw+H3w+X9NAWe7Xv/pVrVZDPB7XypnL5SyPK5VKhutZWlrqKD9VVbG/v49UKoVgMNj0uEKhgGAwiGAwiEKh0LC/UqlgaWlJK8vOzk7bvFOplOG9uHDhAhYWFlCr1Tq6Fq8w+CMiItesrKxgZWXFs/x3d3c9y7sVVVURDodx9epVTE9Po16vI5vNYnV11TIAFEKgWq0CAKrVKnq8GJdttVoNh4eHWFlZgRAC2WwWoVDIsnXzo48+Mvx86dKljvJMJBL4+c9/jsXFRcugDgByuRxSqRQymQwymQw+/PBDpFIpbb+qqiiVSrh16xbq9TrOnj2L8+fPN00POApeFxcXDdsCgQCWl5cRDocHqgWQwR8REQ0FVVUNX/D9JJ1OIxAIYGpqCgDg9/sxOzsLAFhdXbVsLRsfHzf8248ODw+1awKgXVM0Gm049sSJExBCaC9FUTrKs90fGJVKBaFQCMvLy/D7/fD7/YhEIlhcXESpVAJw9EeCzF//XjRrSVRVFe+9957lvqmpKZw8eRLpdLqj6/ECgz8iInJFrVZDLpfTvkDNPxcKBfh8PgSDQVQqFe0Y2T0HPO5WW1pawsHBgZa2VfeneVsikdBabvTbvX4OsVarIRqN4ty5c5b7E4kEQqFQ0+5SM1VVkcvltGtMpVKGbkc79a4/dn19Xdtvp+tTTx/4ybIBQCwWM2yvVCoIBoOIx+PY3993lIdTDx48AAA8++yz2rZnnnkGwOPWx2aBZyQSsdyeTqfxxhtvNM1zZmYG0Wh0cLp/RY8BEJubm73OloiImtjc3BRufB0oiiIAaGnpf97b2xNCCFEulwUAEYlEhBBC268/pl6vi0gkIgCIR48eCSGEqFarhrT1aem3mX8WQohYLCZisdixr08IIebm5sTc3Jyjc/L5vAAgyuVywz5Z1lgsJgCIYrFouV9PURSRTCaFEEf1oiiKUBRF1Ot1bX+7etefm81mhRBCbG9vW5bBrnK5rF2HfN/MdSBfiqKIarXaUT6S1XsthNDuHavjFUWxTKterwsAIp/PN+zb3t7W6rFZnrJ+rc5vpZP7yQWvMfgjIhpxbgV/QjR+OVp9Wdo5plgsCgAikUgcOy03dfJlLQMiK3J7vV7XgjZ94GQ+TwZo+sBpb29PANCCOHleu7rKZrOWx3QSKOsDcfP7JtXrdVEsFrX6kAFsp5q91063C3FUr/oAWqpWq4ZyNktDBo9W192KV8Efu32JiKjvBAIBANbPjg2a1dXVtsf4/X7tmbFW3YdbW1sAjM8BvvDCCwCAO3fuOCqXPN7cfW6nvGaTk5MQQqBYLCIWiyEajTY8f+n3+xEIBLCysoJkMtlycEWvvf3229ozgnoffPABrl271vZ8ed6g3K8M/oiIiPrA+Pg4isUiCoVC09GjGxsbDdtk4OE0mJLHC90gDPnqVCAQwMLCAgA0jIzVu3LlSteCv1YDSaye6cvlclAUpeH5xUKhgFdffdX18vUDBn9ERNS3mj2AP6wCgQDy+TwKhQISiUTDfhnYWLUMdlpX+oE1bjh9+nTbY+QI3G6wqiM50OXll182HFsqlfDxxx9btu4Fg0GcOnWq6WCjQcbgj4iI+o4MSDqdC66fyCDO7jxwiqJocwCazc3NATiaYkWS6c7MzDgqVzKZBABkMhktDTdWIJFpZbPZlsc4La9dsrVOX0effPKJYR9wdK337t0zTBtTKpW0yadbtYg2ax01j3LuVwz+iIjIFebpRvQ/y4BAHwCZW6/kVCeqqiKTyUBRFEMXnmwpkoGhfsoQ+YWtb/WRQYzXU73IljBz8Cev36oVb3Z21jKQuHjxIhRFwdramnbe3bt3EYlEMD093ZBeq3q/fPkygKNn/MbGxuDz+TAxMaEFZXIKGHTJkyMAACAASURBVDk3npVgMIj19XWtZU1VVSQSCcRiMW3uvFwuZ5hCplKpYHd3VyuvZCc/83WZ/w8cPX+YTCZx+/ZtqKoKVVVx+/ZtJJNJTE5OanUQDocRjUYNLXsvvvhiR39wyOs/c+aM43M90eshJuBoXyKivuLWaF/oRntavayO0W8rFovaiNdkMtkw8rJcLmv75ZQacqoSOfpVjhKOxWLaNq+nepHT1MjpQoSwrisrVlOTyBGo8rxsNmuoK7v1LoRxepZIJGKYjiYWi4lIJNJ0ehQhGqdwSSQShus0HxOLxZpOJWMnP6traVZ/Ml9FUcT29rZhn5wOxuplnqbGKm8zOeLa6fQ1Xo329QnR2zVjfD4fNjc3taZrIiLy1p07dzA/P+/ZEmLy+Smv8ndifn4eALC5uenoPNkKef36dUfnqaraMAK114LBIPL5/NDm54Z4PI6xsTHH72+n99Mxvc5uXyIioi4Lh8O4f/++49UtvA789vf3sby8PLT5uaFUKqFUKiEcDntdFNv6OvgzL1HTip1nOqzS8/pZEK/yd1K3buS1vr7e0zy7aX19vacLePPedqbdfebGA+39pNf3o9vMzwkOKzmP39ramq1n2vrBzs4Onn766YYpUIYlPzccHBxgY2MD6XTa80Ddib4O/m7cuIFQKOTaXEBup+eUqqp9Mzz8OHWhqir29/eRSqXaBnK1Wg03btyAoigd5elFnbW7vgsXLmBhYaGjLyr9g8Xm1/r6eseL0vPefqxVXejvx2bvQyv7+/tYWlrS1p7d2dlpuPZW77GdV6uWof39/Ybjj3M/9oOJiQnL/w+j8fFxZDIZ3Lt3z+ui2DI9PW1r2pZBzc8NhUIBN2/eNEy6PRB6/ZQhHA74gMtL9bidnhPy4dN+0WldyIen250vlyuyesjZLi/qzM717e3tWS4FZIfVGqVCPF62Sb9EkxO8tx+zqgur+1H/XrR7L62W0NIPUNDnbX4PrcqjX1pLvzSWfu1VM/1D6ublvTq9H4Vwd3m3YefRA/o0pLi825BTVbXjVp1+s7KyYpgXqZl0Oo1AINBxE75XdWbn+qampnDy5EltOSYnmv2FKKc9cLpEk9cG5d62uh/170W7Lpvbt28DgDZ9BQBtqSoz/THNXLx4Ufu/nH4ikUhgY2NDmzZCr1Kp4LnnnrMs+3HuRyIaPQMT/MnndGR3i/6XY7NnfFRVRS6Xg8/nQzAYbJjF3HxerVZDoVBAMBiEqqpYWloyPLOkL0MwGDTMW2TOz+fzGb4QE4mE1g0l99spt0zH/FyM/rxCoaCVSV8v8ktZphOPx3vWNVSr1RCNRnHu3Lm2x8o6ldcpu9Ds1Jm8dv09IevOfJ+4bWZmpmENTjeec9N3V/LedufednI/NvPrX/8aABqe15Jr0ErlctlWen6/v+HYCxcuAAAePHjQcPyDBw+0/Vas7kciIku9bmtEh92+squmWq1q3Syy20P+bL4cRVFEJBLRukJkN4s8znye/ue9vT1RLBa1LhiZr+zOkV10+vmKFEUxzCUViUQMP5vL2KrcyWTSkK++S8dcTiEedxvpu4xkF1G1WrXcb5W3E63Ol92A+jmjrM5JJBLaMfV6XetubXa8/tpl3cvuuEgk0rI+3Lw+fR5yvjEh7M8n1ixtmLoMeW+7c283ux+tjm1Gzh8HWM9B14qdPOR+eW1m8vqapWV1P9rFbl/72O1LbvKq23dggj+9R48eab+Amx0nf9nrJ2us1+sNxzX72fyLXf98jv5Y+QUo91s9h9PqWszb5BevOR1zUGAnLTlhZrtr7VSr881BXKsy6K9VPoPVKo/jbHOi3fnyfkokEh2nbX7FYrGGe4/39vHv7Wb3Y7P8mnn06JHhuTvz5LrNOAn+ZD3pn00sFovaJLXN0jrO/cjgzz4Gf+QmTvLc4ngADZN/mrebf15aWsLGxobj85rlFwwGm46kFEJo+1tVp1XadsqtqirGxsagKIo28aWdtKRKpYKtrS1Eo1Fb12pXq/Ptvm/yerPZLC5evNjw3JXd63RSH3bZOb/TPKzOq9Vq+NnPfoZSqYR0Oq0908V7+/j3dif3aiv7+/u4ffs2NjY2AAD5fN6wDFknefh8PkN5I5EIbt26BeDocQL5bKHb1wI8nuT53XffdXTeKHrnnXcAAG+++abHJaFh8M4772BycrLnkzwPZMuf1fZ2P7t9ntP9zY5xq9zNtiWTSaEoitZaaicvu1qdb/c6Hj16ZOjqM7da2L1Ou9uc6PQ9PU7asuWzVZcq7+0jTu7tTu5VO2QLKNC6u9VuHUqytbVcLotqtdq2ZdRJPlZkyx9ffPHV+xe7fZscD1h/YXTSpdnuuHbnNVvzT34BNFuzsFna5m3m5xn1x7V7Xs+8Tf8F4uRa7Wp1vt36l+QzaIAxALRznU62OWHn/E7zcFJ3vLePf293cq9K+mftrLp49dO0NGP3XjKnmc1mRTabNTyreJxraYbdvvax25fcxKleHJCj7c6ePdv0mGQyaTj2uGR6mUxGm01fv1KA7PLZ2NjQ9lcqFSwtLTnKR3aHHx4eattkejMzM47SCoVCAB5PI9FLiUQCANquPODz+aCqKgKBAG7duoVisah14Q2KWCzmWlpyRGskEml6DO9t5/e23fvRbH9/3/B75t/+7d8ajpFlaNXt69Tk5CRisRhCoRB+/etfO/oMu3k/EtGQ6nW4CYctf7K1QD7sLEcI6luH9BO1ylYF+ZezoijaX83yQWr8oaXBfF6zyXfNeehfMm39KGR9HvrWFH3LRyKRsCy3nIhWURRtWzabNbSMWE1Mq3/g3zwKulwuG7rGzNdqbomxQ5+fVWuI1ehKqzyBoy5OeVy5XDa8t63qTOZrlW63r0+WFXA+2rfZxMKPHj3SBibI+4b3tjv3drPRvq3qRQ5GkS2e8rjt7W1D2WQrZLOWUTv3ojxGv1+OLtan2yotjvbtDbb8kZs42reF7e1t7Zd9JBLRAkF9mvqXVC6Xta5E+YUop7Ro9oUnX/qRjPr05JdzJBKx/CKR+2OxWEM3mvxlHovFLPPXp5NMJrXt5hGFVudZbTPnJ0dI6rupmn3xtdKszsx1AcBydQ9zuWXAADQ+82enzuxuc/P6hHgcHOi/hNsFf+3uuWQyadnFx3v7ePe2nfux2UuWT6b16NEjwzVY1Ue7PNodI1l1hzc71up+tIvBn30M/shNHO1LQ0d2G16/ft3jknRHPB7H2NjY0F7fsOH92Jwc7dvjr4OBND8/DwC9Hp1JQ8qj++n1gXzmjwZDOBzG/fv3Wy5UP6hKpRJKpRLC4bDXRSGbeD8SER1h8Edd4/f7kU6nsba25trghH5wcHCAjY0NpNPptuvBUv/g/UhEdITBHwF4vCZru5dT4+PjyGQyuHfvXhdKbZ+b11coFHDz5k1tEmYaHP1yP7qJ9+Nw0Y+0J++sr687nh1gkDD4IwCAEMLWqxN+v9/z56zcvL7r16/zi3aA9cP96KZhuB9VVe3oj8t+Sd8ttVoNN27cwEsvvaT9QRqPxy2PdeOP816p1WqIx+NaOXO5nOVxpVLJcD1Op5NqJpVKWdZPoVBAMBi0XOXowoULWFhYQK1Wc6UM/YbBHxEReWp3d3eg03eDqqoIh8O4evUqpqenUa/Xkc1msbq6ahkACiFQrVYBANVqtW8H69RqNRweHmJlZQVCCGSzWYRCIcvWzY8++sjw86VLl46df6lUwuLiYsP2XC6HVCqFTCaDTCaDDz/8EKlUStsfCASwvLyMcDg8lC2ADP6IiMgzqqoavnQHLX23pNNpBAIBTE1NAThqoZ6dnQUArK6uWraWyRbffm75PTw81K4JgHZNVpP5nzhxwtATc9yJ01VVxXvvvdewvVKpIBQKYXl5GX6/H36/H5FIBIuLi4bngaempnDy5Emk0+ljlaMfMfgjIqKOqKqKXC6nddOlUilDN5lVl6R5WyKR0Lrc5PZaraZ1yQGPu+2WlpZwcHBw7PSBo6lxmnWp9lqtVkM0GsW5c+cs9ycSCYRCoabdpWbt3pdarYZcLqfVb6FQgM/nQzAY1FYZ0h+7vr6u7d/Z2XF0bfrAT5YNaFyJplKpIBgMIh6PuzYiP51O44033mjY/uDBAwDAs88+q2175plnADS2Ps7MzCAajQ5d9y+DPyIi6sjCwgI+/fRTrQuyUCgYuslkt6ReuVw2/LyysqL9X7b4TExMaM9h7e/v49q1a6jX6wCA559/XgsAO02/3zx8+BAA8Nxzz1nuv379urbcn52R6u3el3A4jFAopNWvoigol8soFAp46623tHRqtRrC4TBOnjwJIQR+9KMf4fz58x2Plq9UKtpSiwsLC4Z9Ms3V1VW88sorCAaDxwq4dnZ28K1vfcuyVfT+/fsAjMtDyuPMz/7J90S+R0Oju5NINwKcr/BBRETd08kKH3JJQf2KInKVkWw2q21Di1VNnBwjxOOVXfQrAXWafqe6sSKDXD3Hitwul0cEYFhRxnyem++LXLrQfEy7JSytmFffMa/mJK+xWCxq9ZFMJh3nI8TjlYT0ZbZzP1htl8tLWpXXDV6t8MGWPyIicmxrawuA8XmzF154AcDRiiHdEAgEAFg/LzbIVldX2x4j56kE0LIb0s33RR5v7kq3U16zyclJCCFQLBYRi8UQjUYbnsX0+/0IBAJYWVlBMplsaIWz64MPPsC1a9c6OtdMzp05bPccgz8iInJsY2OjYZv8ouz0S5taGx8fR7FYbOjG1XPzfZHHC5em/QKOAnjZ5Ws1Cle6cuVKR/dRoVDAq6++2vKYVgNJIpGI4zwHEYM/IiJyTH6BWrVAdfsLdFS+oK0EAgHk83kUCgXt+Tm9brwv+kE2bjh9+nTbY+QIXKeCwSBOnTrVdDAQYF1HcqDLyy+/7DjPQcTgj4iIHJubmwNwNJWHJFuiZmZmupKnDELcmP+tn8ggzu58coqiaHMAmrn5viSTSQBAJpPR0nBjBRKZVjabbXlMJ/dRq1ZK+X/ZMqivo08++cSwz8w8OnnQMfgjIiLHLl68CEVRsLa2prWg3L17F5FIBNPT09pxsvVGBm76aTzkCg76lhhzYCGnN1FVFZlMBoqiGLrtOk2/n6Z6kS1h5uBP1qtVK97s7KxlQGLnfdGnJ/PU5y33X758GcDRM35jY2Pw+XyYmJjQgjI5BUyr0b/BYBDr6+tay5qqqkgkEojFYtqcf7lczjCFTKVSwe7uruE+spufHZOTk0gmk7h9+zZUVYWqqrh9+zaSyaRhBLAsCwCcOXPmWHn2nV4PMQFH+xIR9ZVORvsK8XhUJf4wSjKbzYp6vW44plwua6NU8/m8EEIIRVFENpvVRqTKUbyxWEzbJtMsFova+clk0rX0Y7FYR6NWuzE6s1qtCgBib29P2wbdyFg0GZ0qxNG1WqXX6n2xSrdZXuVyWRt9G4lERLlc1vbFYjERiUQsyyDl8/mGUb766zQfE4vFRLFYtEzLTn5WmtWfzFdRFLG9vW15rhwprR897SavRvv6hOjtpEc+nw+bm5ta0zQREXnrzp07mJ+f76s58OTzWf1UJgCYn58HAGxubrqarmyRdLrutKqq2oAOrwSDQeTz+aHMLx6PY2xsrGvrgXfrfmrjdXb7EhEReSwcDuP+/fuOV7fwOvDb39/H8vLyUOZXKpVQKpUQDod7kl8vMfgjIqK+Yl6KbBTIefzW1taO/Uxbr+zs7ODpp59uWMJtGPI7ODjAxsYG0um05wF2NzD4IyKivjIxMWH5/2E3Pj6OTCaDe/fueV0UW6anp21N2zKI+RUKBdy8edNyebhh8KTXBSAiItLrt+f8esnv93ft+TKyb9jfA7b8EREREY0QBn9EREREI4TBHxEREdEIYfBHRERENEI8GfAxPz+P999/34usiYjIRC5hdeXKFY9L0v8ePnwIgHVF7tja2vJk0Yuer/CxvLyMX/7yl73MkojI4L//+7/xH//xH7hw4YLXRSGiEbewsGBYr7oHXu958EdE5LV+XM6MiKhHuLwbERER0Shh8EdEREQ0Qhj8EREREY0QBn9EREREI4TBHxEREdEIYfBHRERENEIY/BERERGNEAZ/RERERCOEwR8RERHRCGHwR0RERDRCGPwRERERjRAGf0REREQjhMEfERER0Qhh8EdEREQ0Qhj8EREREY0QBn9EREREI4TBHxEREdEIYfBHRERENEIY/BERERGNEAZ/RERERCOEwR8RERHRCGHwR0RERDRCGPwRERERjRAGf0REREQjhMEfERER0Qhh8EdEREQ0Qhj8EREREY0QBn9EREREI4TBHxEREdEIYfBHRERENEIY/BERERGNEAZ/RERERCOEwR8RERHRCHnS6wIQEXXbhQsXUCwW8cwzzwAAfvOb38Dv9+PrX/+6dsyjR4/wL//yL5ibm/OqmEREPcHgj4iG3s7ODoQQ+J//+R/DdlVVDT//13/9Vw9LRUTkDXb7EtHQ++lPf4onn2z9t67P58Ps7GyPSkRE5B0Gf0Q09H7wgx/giy++aLrf5/PhG9/4Br72ta/1sFRERN5g8EdEQ+/UqVM4c+YM/uiPrH/lPfHEE/i7v/u7HpeKiMgbDP6IaCRcvXoVPp/Pct/vf/97/OAHP+hxiYiIvMHgj4hGwszMjOX2J554AmfPnsWJEyd6XCIiIm8w+COikfDnf/7nOHfuHJ544gnDdiEEfvjDH3pUKiKi3mPwR0Qj44c//CGEEIZtTzzxBL7//e97VCIiot5j8EdEI+N73/sennrqKe3nJ598EhcvXoTf7/ewVEREvcXgj4hGxp/+6Z/iu9/9rjbn3xdffIGFhQWPS0VE1FsM/ohopMzPz2tz/v3Jn/wJvvvd73pcIiKi3mLwR0Qj5dKlS/jKV74CAPjbv/1b/PEf/7HHJSIi6q2G9Y5+97vfIZ/Pt5wNn4hokJ06dQoff/wxvvrVr2Jra8vr4hARdcVXv/pVvPLKKw3bfcI09O3999/nyDciIiKiIWCe4QDA6w0tf7/5zW+aHUxEROSJ+fl5AMDm5qbHJel/Pp8Pm5ubmJub87oo5KE7d+5onxszPvNHRERENEIY/BERERGNEAZ/RERERCOEwR8RERHRCGHwR0RERDRCGPwRERERjRAGf0RENFLi8Tji8bjXxehLtVoN6+vrXhdj5K2vr0NV1a6lz+CPiIioh1RVhc/n87oYDWq1Gm7cuIGXXnoJPp8PPp+vaZAs9+tf/apWqyEej2vlzOVylseVSiXD9SwtLbmSfyqVsqyfQqGAYDCIYDCIQqFg2HfhwgUsLCygVqu5UgYzBn9ERDRSVlZWsLKy4ln+u7u7nuXdjKqqCIfDuHr1Kqanp1Gv15HNZrG6umoZAAohUK1WAQDVarVvF4ao1Wo4PDzEysoKhBDIZrMIhUKWrZsfffSR4edLly4dO/9SqYTFxcWG7blcDqlUCplMBplMBh9++CFSqZS2PxAIYHl5GeFwuCstgAz+iIiIekRVVcOXfL9Ip9MIBAKYmpoCAPj9fszOzgIAVldXLVvLxsfHDf/2o8PDQ+2aAGjXFI1GG449ceIEhBDaS1GUY+Wtqiree++9hu2VSgWhUAjLy8vw+/3w+/2IRCJYXFxEqVTSjpuamsLJkyeRTqePVQ4rDP6IiGhk1Go15HI5BINBy58LhQJ8Ph+CwSAqlYp2jOyiAx534y0tLeHg4EBL26oL1LwtkUhoXXz67V4+h1ir1RCNRnHu3DnL/YlEAqFQqGl3qZmqqsjlctr1pVIpQ/elnTrXH7u+vq7t39nZcXRt+sBPlg0AYrGYYXulUkEwGEQ8Hsf+/r6jPJpJp9N44403GrY/ePAAAPDss89q25555hkAja2PMzMziEaj7nf/CpPNzU1hsZmIiMgzc3NzYm5u7tjpKIoiAGjfc/qf9/b2hBBClMtlAUBEIhEhhND264+p1+siEokIAOLRo0dCCCGq1aohbX1a+m3mn4UQIhaLiVgsduzrk+lvbm7aPj6fzwsAolwuW6YlywdAFItFy/16iqKIZDIphDiqE0VRhKIool6va/vb1bn+3Gw2K4QQYnt727IMdpXLZe065HtmrgP5UhRFVKvVjvKRZZXXZn6/5X1jJvM1lxmAyOfzjsvQIp57jcEfERH1PbeCPyEav4ytgjE7xxSLRQFAJBKJY6flJqfBnwyImqUlxFGwK4M2feBkPk8GaPrAaW9vTwDQgjh5Xrt6ymazlsd0EiTrg3DzeybV63VRLBa1+pABrFPVatVwrt3332p7vV5vWt52WgV/7PYlIiLqQCAQAGD9/NggWV1dbXuM3+/Xnj1r1Q25tbUFwPgc4AsvvAAAuHPnjqNyyePNXed2yms2OTkJIQSKxSJisRii0WjDs5d+vx+BQAArKytIJpMNI3Dt+uCDD3Dt2rWOzjXz+/0A3L/HGPwRERFRW+Pj4ygWiygUCk1HoW5sbDRskwGM02BKHi90gzDkq1OBQAALCwsAYDkKV7py5UpHwV+hUMCrr77a8phWA0kikYjjPDvB4I+IiOgYevWF3Q8CgQDy+TwKhQISiUTDfhnYWLUMdlpP+kE1bjh9+nTbY+QIXKeCwSBOnTrVdPAPYF1HcqDLyy+/7DjPTjD4IyIi6oAMStyYD85LMoizO5+coijaHIBmc3NzAI6mWJFkujMzM47KlUwmAQCZTEZLw40VSGRa2Wy25TFOywu0bqWU/5ctg/o6+uSTTwz7zMyjk4+LwR8REY0M85Qj+p9lUKAPgswtWHK6E1VVkclkoCiKoRtPthbJwFA/bYhcMULf8iMDGS+nepEtYebgT167VSve7OysZUBy8eJFKIqCtbU17by7d+8iEolgenq6Ib1WdX758mUAR8/4jY2NwefzYWJiQgvK5BQw+rnxzILBINbX17WWNVVVkUgkEIvFtDn/crmcYQqZSqWC3d1drbySnfzsmJycRDKZxO3bt6GqKlRVxe3bt5FMJjE5OWk4Vpb7zJkzx8rTjMEfERGNjImJCcP/9T+PjY0Z/jUfDxwNXggGgxgbG8Pk5CQymYxh/z/90z9BURQ8//zzKBQKmJqa0lrKbt68CQDa6iI/+9nPtOfPvPTNb34TwOPWJwBaoAUc1YHV8mQrKysNz6/JgSGKohjO++lPf6odY7fOx8fHUS6XtSAzEomgXC5rAVK9XkckEmkZNF+7dg3RaFTrik2n0/jOd75jWOHlK1/5Cs6fP68tZ/e///u/ls/l2cnPrmvXruHSpUsYGxvDwsICZmZmLAeJyPdEvkdu8QnTk5N37tzB/Px83y7VQkREo2d+fh4AsLm56Un+MogZhO9Gn8+Hzc1NrQvWDtkCef36dUd5qaqqDejwSjAYRD6fH8r84vE4xsbGHL8vQMt47nW2/BEREY24cDiM+/fvO17dwuvAb39/H8vLy0OZX6lUQqlUQjgcdj3trgV/Xj6/YJd5iRmgu+XWp22Vdz/oZbnaPbjbr3XklvX19a4s2N1MP38m+blrXS43HnL3Uq/vdbeZnxMcRrK7dm1t7djPtPXKzs4Onn766YYl3IYhv4ODA2xsbCCdTnclwHYl+FNV1fJ5gH5348YNhEKhjidyBDq/djfydqMcZscpl6qq2N/fRyqVavvlWqvVcOPGjZbzHbldR73Wrj4uXLiAhYWFjr5M9NMI2Hn1SqsyrK+vo1AouBIEjNLnTv9Z6VX9uu0493o/MD8nOKzGx8eRyWRw7949r4tiy/T0tK1pWwYxv0KhgJs3bxomy3aVg+VAmpJr4g0iHHOZneNc+3HzdqscZp2WS65N2e58uUyQXPewG2XpB3bqY29vz7DmpV0wLZUkt5nzsVoeqdv065vqr6tYLGprfB5nzUwhRudzZ/VZ6UX9dkOn97rk5vJuww4Ol3ej4dTV5d1UVW1YImVU9Mu190s5VlZWDCOomkmn0wgEAj1rqveKnfqYmprCyZMntWWTnJDTFLRy8eJFx+kel/4vVX13RSAQ0K6z2eoAdvTL/d6Lclh9Vrpdv91ynHudiFzmIFK0pG/ZkK9qtSqy2axQFEUIIRp+ln8tRyIRUS6XhRCPWyj026RqtSoSiYQAIBRFEdvb27bLp1ev17V8FEURjx49Mvy1bS6nJPNOJpPaX92trj2fz2t/4UYiERGLxSzT1p8j8zBfvz7tZtusymG37trVSSdanS/rz+o9tFuWZtfU7D5TFKXhnmr2nrbLw+36EMJ6IXTZatiM+Xpa5VMul3v+mWx1zfJ68/m8ZVmkUf/ctfqsOKnfdmXq5efG6l63iy1/9oEtfyRat/y50u1r/kWkKIphm/7nYrEohDjqApC/dGWXRrlc1rZJ1WpVKIqidXHJXx4yHScURRGRSETrdpC/fK3KKSUSCe0XYL1e137h27n2vb09USwWRSQSsUxbf5z+WvW/HPVdPJKsp1blsFt37eqkE63Ol18sVsGLnbK0uiZz3QthfU+1e0/dvOfa1Ye+jPov63bBn9N8ev2ZbFWWer1uSJOfO+t7vdVnxUn9titTLz83Vve6XQz+7GPwR0J4EPxZbbNzjNU2q2eWADj+YpS/SB89eqRtk78kW5XT/Feq+a/dVtdgfrbFTp3IFoBkMmkrj1bHtKs7u3XiVKvzzV8Ykt2ytLsmu3XV6j11655rlr+ZvM5EItFR+nbz6eVn8rhl4eeu+WelWb6t9vfL5+Y49zqDP/sY/JEQrYM/VyZ5tpr80rzNzjFW24LBYNOReU7KuLS0hI2NjYZz2pVTnpfNZnHx4sWGIdd2r8tOXk6Os3NMu7qzWydOtTq/2T67ZWl3TXbqqt176tY91yz/To85bj69/Ewetyz83HX2OWq2v58+N53e6/Pz8/jFL37h+koHw2hrawvf/OY3G5YKo9FSqVTw8OFDq89a/0/yLH+ZCNF8sWQ7NjY2Osr/xz/+MRRFQSgUwtjY2EDNxmHrxgAACcRJREFUtdWu7jqtk26wWxY37od276lb99ywOm79yIEIrRYq5+euc1b1y88NERmY2wL7rdtX/qzvIumEVV52yymE0J4hgqnLwu51OckLMD5rc9z6bVZ3dsvpVKvznebp9H5w8n60e0+Pe8+1y9/pMcfNp5efyVZlkc+C6QcD8HNnv5zt9rWq33743HR6r7Pb1z6A3b7U5aleui2ZTAIAMpmM9hdtJ7Pdy3Sczlzu8/mgqioCgQBu3bqFYrGIaDTqKA0nZPnOnj177LTa1V2ndXIciUQCABqmobBbFjfuh3bvqVv3nFOtWsL6yXHqp1ar4e2334aiKJienm56HD93zT8rrTSr33773AzKvU40tBxEik3pR8olEgnDSLlqtWo5Kan5GDvb9C+rEXCtyFFm+ukL5F/I+MNf/Fb5A0cPLMtzyuWy4a/dVteuZ5W2PFc/5YKiKA0PQ8u/suVf1HJUpiy3VTns1J2dOnFK/+C61WSuzUYw2i1Lq2uyus/05bH7nrp1z9mpD/21H2e0r9X91Wp/Nz+Tza652STE/NxZ3+vNPitO67ddmXr5ueFo394AW/5I9GC0b7FY1H4xNPsFoH8JIWxvE+LoF4Yc+WY155hd5XJZ+4Uugz05NYFVuWWZ9HNXmb8gWl271dxi5mvb3t7WvkAikYjlnF7lclk7Rv7S1Jfbqhx2665dnTjR6v2WZB1Zre5htyzNrsnJfdbqPbVTb27VhxCPgwon8/w5zadXn8lWeSQSCcv3nZ8763vd6rPSSf06fc86fU/sXLfVvW4Xgz/7AAZ/1IPRvkROyG6g69eve1yS/hCPxzE2Nsb6oAbD9lk5zr0+Pz8PANjc3HS7WEPH5/Nhc3MTc3NzXheFPNQinuv/0b40fMLhMO7fv4/9/X2vi+K5UqmEUqmEcDjsdVGoDw3TZ4X3OlH/YPBHPef3+5FOp7G2ttbTwSb95uDgABsbG0in0w1zphEBw/NZ4b0+2Hox4G3Ura+v93Q97oEO/nw+n60Xda5bdTw+Po5MJoN79+51odTd42Z9FAoF3Lx5E+Pj410uNQ2yQf2s6A3Dva6qale/T7qdfqdqtRpu3LiBl156Sfv9Fo/HLY8dpO/fWq2GeDyulTOXy1keVyqVDNeztLTUlfwuXLiAhYUF1Gq1jtJ3aqCDP2ExmajVizrXzTr2+/0D9yyTm/Vx/fr1gf4ypN4ZxM+K3jDc67u7uwOdfidUVUU4HMbVq1cxPT2Ner2ObDaL1dVVywBQCIFqtQoAqFarffv9W6vVcHh4iJWVFQghkM1mEQqFLFs3P/roI8PPly5d6kp+gUAAy8vLCIfDPWkBHOjgj4iIqNtUVUUqlRrY9DuVTqcRCAQwNTUF4OiPkNnZWQDA6uqqZWuZDPL7Odg/PDzUrgmAdk1Wc4meOHHC8Ie9oihdy29qagonT55EOp12nIdTDP6IiGhoqaqKXC6ndbelUilD15pVF6V5WyKR0Javk9trtRoKhQKCwSAAIJVKad2CBwcHx04fOBod3ayLtdtqtRqi0SjOnTtnuT+RSCAUCjXtLjVr9z7UajXkcjmtPguFAnw+H4LBICqVSkPZ1tfXtf07OzuOrk0fiMmyAY2Tj1cqFQSDQcTj8WMNurKbHwDMzMwgGo12vfuXwR8REQ2thYUFfPrpp1qXZKFQMHStyW5KvXK5bPh5ZWVF+79sAZqYmEAwGEShUMD+/j6uXbuGer0OAHj++ee1ALDT9L328OFDAMBzzz1nuf/69euIxWIIhUK2BiO1ex/C4TBCoZBWn4qioFwuo1Ao4K233tLSqdVqCIfDOHnyJIQQ+NGPfoTz5893PCCqUqloq+ksLCwY9sk0V1dX8corryAYDB47KGuVH/C4vmX9d42DSQGJiIg80ckkz3LlFP2k0nKi6Ww2q21Di4nRnRwjxOOJv52uRW11TKfgwiTPcrLuZukLcbQSjJwIXb+ms/k8N9+HbDZreYyTFZEkueKMfFlNXF6v10WxWNTqI5lMOs7HaX7N9jk10Gv7EhERdWJrawuA8fmzF154AcDRBLjdEAgEAFg/PzZIVldX2x4jpyIC0LKr0s33QR5v7jq3U16zyclJCCFQLBYRi8UQjUYbnr30+/0IBAJYWVlBMpnUuuc7YTc/oPv3D1f4ICKivtfJCh8yMDB/n5m3Wx3XyTFup98pN1b4aFUen89n2F4qlfDiiy9CURRkMhmMjY0NRD3pHRwc4Pnnn2+ZtqqqDdfWjfzcukau8EFERCNHjsy0apGKRCJdzbvb6feTQCCAfD6PQqGgPc+m1433QT+oxg2nT59ue4zf73ftfbWTXzcx+CMioqEkW74ODw+1bXKAwczMTFfylEFJJ/PB9RMZxNmdc05RFG0OQDM334dkMgkAyGQyWhpurEAi08pmsy2Pceu+aZef1UhgNzH4IyKioXTx4kUoioK1tTWt1enu3buIRCKYnp7WjpOtOTJw00/rIVd00LdemQMNOd2JqqrIZDJQFMUwH1yn6Xs51YtsmTIHf7IerVrxZmdnLYMWO++DPj2Zpz5vuf/y5csAjp7xGxsbg8/nw8TEhBaUySlgWo3+DQaDWF9f16aQUVUViUQCsVhMm4Mvl8sZppCpVCrY3d013Ddu5qfPBwDOnDnTND1XOBgdQkRE5IlORvsKIUS1WhXJZFIbYZnNZkW9XjccUy6XtVGr+XxeCCGEoigim81qI1TlKN5YLKZtk2kWi0Xt/GQy6Vr6sViso1GscGG0b7VaFQDE3t6eIV3zy4qiKJbptXofrNJtlle5XNZG30YiEVEul7V9sVhMRCIRyzJI+Xy+YdSt/jrNx8RiMVEsFi3Tcis/SY6C1o+M7lSr0b4c8EFERH2vkwEf3datwQfH5caADwBaC6TTpQVVVdVGrXolGAwin88PXH7xeBxjY2OuLOfIAR9ERETkSDgcxv379x2vbuF14Le/v4/l5eWBy69UKqFUKiEcDrtQqtYY/BERETlkXppsGMl5/NbW1jpeQaPXdnZ28PTTTzcsqdbv+R0cHGBjYwPpdLonwTODPyIiIocmJiYs/z9sxsfHkclkcO/ePa+LYsv09HRPp1FxK79CoYCbN28aJsLupid7kgsREdEQ6bfn/LrJ7/e78gwaNdfr+mXLHxEREdEIYfBHRERENEIY/BERERGNEAZ/RERERCOEwR8RERHRCGlY4eP999/H97//fa/KQ0REREQusVrho2Gql+9+97v413/9V3zxxRe9KRURERERue6rX/2q5faGlj8iIiIiGlpc25eIiIholDD4IyIiIhohDP6IiIiIRsiTAP6f14UgIiIiop74xf8HrMJf9NsXWu0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "from keras.utils.vis_utils import plot_model\n",
    "plot_model(model, to_file='Elmo_Model_plot.png', show_shapes=True, show_layer_names=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([2, 2, 2, 2, 2, 1, 1, 2, 2, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_te[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pred2label(pred):\n",
    "    out = []\n",
    "    for pred_i in pred:\n",
    "        out_i = []\n",
    "        for p in pred_i:\n",
    "            p_i = np.argmax(p, axis=-1)\n",
    "            out_i.append(idx2tag[p_i].replace(\"PAD\",\"text\"))\n",
    "        out.append(out_i)\n",
    "    return out\n",
    "pred_labels = pred2label(y_pred)\n",
    "\n",
    "def pred2labelV2(pred):\n",
    "    out = []\n",
    "    for pred_i in pred:\n",
    "        out_i = []\n",
    "        for p in pred_i:\n",
    "            out_i.append(idx2tag[p].replace(\"PAD\",\"text\"))\n",
    "        out.append(out_i)\n",
    "    return out\n",
    "test_labels = pred2labelV2(y_te)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "F1-score: 98.1%\n"
     ]
    }
   ],
   "source": [
    "from seqeval.metrics import precision_score, recall_score, f1_score, classification_report\n",
    "print(\"F1-score: {:.1%}\".format(f1_score(test_labels, pred_labels)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "           precision    recall  f1-score   support\n",
      "\n",
      "     text       0.98      0.98      0.98      4419\n",
      "   object       0.99      0.97      0.98      2068\n",
      "\n",
      "micro avg       0.99      0.98      0.98      6487\n",
      "macro avg       0.99      0.98      0.98      6487\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(test_labels, pred_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "      object       1.00      0.98      0.99      3588\n",
      "        text       1.00      1.00      1.00     91927\n",
      "\n",
      "    accuracy                           1.00     95515\n",
      "   macro avg       1.00      0.99      0.99     95515\n",
      "weighted avg       1.00      1.00      1.00     95515\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from  sklearn_crfsuite.metrics import flat_classification_report  \n",
    "report = flat_classification_report(y_pred=pred_labels, y_true=test_labels)\n",
    "print(report)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Word           ||True ||Pred\n",
      "KORRAS.        : object text\n",
      "KOORMUSTEST-KORRAS: object text\n",
      "24-tunni       : object text\n",
      "130/82         : object text\n",
      "130/95         : object text\n",
      "min.Pikkus     : object text\n",
      "170            : object text\n",
      "rr125/80       : text  object\n",
      "RR-monitooringul: object text\n",
      "114/73         : object text\n",
      "145/80mmHg     : text  object\n",
      "52Pikkus       : object text\n",
      "166            : object text\n",
      "rR             : text  object\n",
      "136/84         : text  object\n",
      "140            : object text\n",
      "157/94         : object text\n",
      "regRR140-90mmhg: object text\n",
      "170/100        : object text\n",
      "140/94--&gt;   : object text\n",
      "140/90         : object text\n",
      "142/78         : text  object\n",
      "160/100mmHg    : text  object\n",
      "170cm          : object text\n",
      "13-101035-1    : object text\n",
      "RR-le          : object text\n",
      "156/86         : object text\n",
      "id=\"0\"         : object text\n",
      "RRmonitooringul: object text\n",
      "110            : object text\n",
      "03.06.2010     : object text\n",
      "120x1          : object text\n",
      "120x1          : object text\n",
      "AVR            : text  object\n",
      "130/80         : text  object\n",
      "Rr130/80       : text  object\n",
      "134/75mmHg     : text  object\n",
      "120kg          : object text\n",
      "RR-monitooring:: object text\n",
      "135/93         : object text\n",
      "*10.06.2014    : object text\n",
      "RR-monitooringul: object text\n",
      "130/80         : text  object\n",
      "120x           : object text\n",
      "120,1          : object text\n",
      "145;           : object text\n",
      "120,2kg        : object text\n"
     ]
    }
   ],
   "source": [
    "j = 0\n",
    "print(\"{:15}||{:5}||{}\".format(\"Word\", \"True\", \"Pred\"))\n",
    "\n",
    "for i in range(len(y_pred)):\n",
    "    p = np.argmax(y_pred[i], axis=-1)\n",
    "#     print(30 * \"=\")\n",
    "    for w, t, pred in zip(X_te[i], y_te[i], p):\n",
    "        if w != \"\": #PAD\n",
    "            if idx2tag[t] != idx2tag[pred]:\n",
    "                print(\"{:15}: {:5} {}\".format(w, idx2tag[t], idx2tag[pred]))\n",
    "                j = j+1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "47"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "j"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Kasutaja\\Anaconda3\\envs\\myEnv\\lib\\site-packages\\sklearn\\utils\\deprecation.py:144: FutureWarning: The sklearn.metrics.scorer module is  deprecated in version 0.22 and will be removed in version 0.24. The corresponding classes / functions should instead be imported from sklearn.metrics. Anything that cannot be imported from sklearn.metrics is now part of the private API.\n",
      "  warnings.warn(message, FutureWarning)\n",
      "C:\\Users\\Kasutaja\\Anaconda3\\envs\\myEnv\\lib\\site-packages\\sklearn\\utils\\deprecation.py:144: FutureWarning: The sklearn.feature_selection.base module is  deprecated in version 0.22 and will be removed in version 0.24. The corresponding classes / functions should instead be imported from sklearn.feature_selection. Anything that cannot be imported from sklearn.feature_selection is now part of the private API.\n",
      "  warnings.warn(message, FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "from eli5.lime import TextExplainer\n",
    "from eli5.lime.samplers import MaskingTextSampler\n",
    "# https://eli5.readthedocs.io/en/latest/autodocs/lime.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "class NERExplainerGenerator(object):\n",
    "    \n",
    "    def __init__(self, model, tag2idx, max_len):\n",
    "        self.model = model\n",
    "        self.tag2idx = tag2idx\n",
    "        self.idx2tag = idx2tag\n",
    "        self.max_len = max_len\n",
    "        \n",
    "    def _preprocess(self, texts):\n",
    "\n",
    "        X = [[w[0] for w in s] for s in sentences]\n",
    "        new_X = []\n",
    "        for seq in X:\n",
    "            new_seq = []\n",
    "            for i in range(max_len):\n",
    "                try:\n",
    "                    new_seq.append(seq[i])\n",
    "                except:\n",
    "                    new_seq.append(\"PAD\")\n",
    "            new_X.append(new_seq)\n",
    "        X = new_X\n",
    "        X = batch_to_ids(X)\n",
    "        X = elmo_model(X)\n",
    "\n",
    "\n",
    "        return X['elmo_representations'][1].detach().numpy()\n",
    "    \n",
    "    def get_predict_function(self, word_index):\n",
    "        def predict_func(texts):\n",
    "            X = self._preprocess(texts)\n",
    "            p = self.model.predict(X)\n",
    "            return p[:,word_index,:]\n",
    "        return predict_func\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'26.05.09 patsient hospitaliseeritud erütrotsüütide ülekandeks, 2 doosi RR 120/85 temperatuur 37.7. Võetud vereanalüüsid. Enesetunne rahuldav.'"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels = [[s[1] for s in sent] for sent in sentences]\n",
    "sentences = [\" \".join([s[0] for s in sent]) for sent in sentences]\n",
    "sentences[0]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Põhjendus: (text) 6A (text) dünaamikas (text) Kirjeldus: (text) Võrreldes (text) 23.10 (text) tehtud (text) ü/v-ga (text) dünaamika (text) puudub. (text) Pikkus-165, (object) kaal-107,6. (text) v/s-10,5 (text) tühjalt. (text)\n"
     ]
    }
   ],
   "source": [
    "Sentneceindex = 21\n",
    "label = labels[20:22]\n",
    "text = sentences[20:22]\n",
    "# print(label)\n",
    "# print(text)\n",
    "\n",
    "text = \" \".join(str(x) for x in text)\n",
    "label = [' '.join(x) for x in label]\n",
    "label = \" \".join(str(x) for x in label)\n",
    "print(\" \".join([f\"{t} ({l})\" for t, l in zip(text.split(), label.split())]))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0: Põhjendus:\n",
      "1: 6A\n",
      "2: dünaamikas\n",
      "3: Kirjeldus:\n",
      "4: Võrreldes\n",
      "5: 23.10\n",
      "6: tehtud\n",
      "7: ü/v-ga\n",
      "8: dünaamika\n",
      "9: puudub.\n",
      "10: Pikkus-165,\n",
      "11: kaal-107,6.\n",
      "12: v/s-10,5\n",
      "13: tühjalt.\n"
     ]
    }
   ],
   "source": [
    "for i, w in enumerate(text.split()):\n",
    "    print(f\"{i}: {w}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "explainer_generator = NERExplainerGenerator(model, tag2idx, max_len)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_index = 10\n",
    "predict_func = explainer_generator.get_predict_function(word_index=word_index)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(('Põhjendus: 6A UNK Kirjeldus: Võrreldes UNK.10 UNK ü/v-ga dünaamika puudub. Pikkus-UNK, kaal-107,UNK. v/s-10,5 tühjalt.', 'Põhjendus: UNK dünaamikas Kirjeldus: UNK UNK.10 UNK ü/UNK-ga UNK puudub. Pikkus-165, kaal-UNK,UNK. UNK/s-UNK,5 tühjalt.', 'Põhjendus: UNK dünaamikas Kirjeldus: UNK 23.10 UNK UNK/UNK-ga dünaamika puudub. Pikkus-UNK, UNK-107,UNK. v/s-UNK,UNK tühjalt.', 'Põhjendus: 6A dünaamikas Kirjeldus: Võrreldes 23.10 tehtud ü/v-UNK dünaamika puudub. Pikkus-165, kaal-107,6. v/s-10,5 tühjalt.', 'UNK: UNK dünaamikas Kirjeldus: Võrreldes UNK.10 UNK ü/v-ga dünaamika UNK. UNK-UNK, UNK-107,6. v/s-10,5 tühjalt.', 'Põhjendus: UNK dünaamikas Kirjeldus: Võrreldes UNK.10 tehtud UNK/v-UNK dünaamika UNK. UNK-165, UNK-107,UNK. UNK/UNK-UNK,5 UNK.', 'UNK: UNK dünaamikas UNK: Võrreldes 23.UNK UNK UNK/UNK-UNK UNK UNK. UNK-165, UNK-107,UNK. UNK/s-10,UNK UNK.', 'Põhjendus: 6A dünaamikas Kirjeldus: Võrreldes UNK.10 tehtud ü/v-ga dünaamika puudub. Pikkus-UNK, kaal-107,6. v/s-10,UNK tühjalt.', 'Põhjendus: UNK dünaamikas UNK: UNK UNK.10 tehtud UNK/UNK-ga dünaamika puudub. UNK-165, UNK-107,6. v/s-10,UNK UNK.', 'Põhjendus: 6A UNK Kirjeldus: Võrreldes UNK.10 UNK ü/v-ga dünaamika UNK. Pikkus-UNK, kaal-107,6. v/s-10,5 tühjalt.'), array([0.88465169, 0.75180937, 0.75180937, 0.97801925, 0.80757281,\n",
      "       0.69156403, 0.55167723, 0.93250476, 0.75180937, 0.88465169]))\n"
     ]
    }
   ],
   "source": [
    "sampler = MaskingTextSampler(\n",
    "    replacement=\"UNK\",\n",
    "    max_replace=0.7,\n",
    "    token_pattern=None,\n",
    "    bow=False\n",
    ")\n",
    "print(sampler.sample_near(text, n_samples=10))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "ename": "RuntimeError",
     "evalue": "[enforce fail at ..\\c10\\core\\CPUAllocator.cpp:72] data. DefaultCPUAllocator: not enough memory: you tried to allocate 25482067968 bytes. Buy new RAM!\n",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-34-83cb6e892130>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      6\u001b[0m )\n\u001b[0;32m      7\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 8\u001b[1;33m \u001b[0mte\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtext\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpredict_func\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      9\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     10\u001b[0m te.explain_prediction(\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\myEnv\\lib\\site-packages\\eli5\\lime\\lime.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, doc, predict_proba)\u001b[0m\n\u001b[0;32m    254\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msimilarity_\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0msims\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    255\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mX_\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mX\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 256\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0my_proba_\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpredict_proba\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msamples\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    257\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mclf_\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mclone\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mclf\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    258\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-27-84e79b232965>\u001b[0m in \u001b[0;36mpredict_func\u001b[1;34m(texts)\u001b[0m\n\u001b[0;32m     28\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mget_predict_function\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mword_index\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     29\u001b[0m         \u001b[1;32mdef\u001b[0m \u001b[0mpredict_func\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtexts\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 30\u001b[1;33m             \u001b[0mX\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_preprocess\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtexts\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     31\u001b[0m             \u001b[0mp\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     32\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0mp\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mword_index\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-27-84e79b232965>\u001b[0m in \u001b[0;36m_preprocess\u001b[1;34m(self, texts)\u001b[0m\n\u001b[0;32m     21\u001b[0m         \u001b[0mX\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnew_X\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     22\u001b[0m         \u001b[0mX\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mbatch_to_ids\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 23\u001b[1;33m         \u001b[0mX\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0melmo_model\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     24\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     25\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\myEnv\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m    539\u001b[0m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    540\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 541\u001b[1;33m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    542\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    543\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\myEnv\\lib\\site-packages\\allennlp\\modules\\elmo.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, inputs, word_inputs)\u001b[0m\n\u001b[0;32m    169\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    170\u001b[0m         \u001b[1;31m# run the biLM\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 171\u001b[1;33m         \u001b[0mbilm_output\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_elmo_lstm\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mreshaped_inputs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mreshaped_word_inputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    172\u001b[0m         \u001b[0mlayer_activations\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mbilm_output\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'activations'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    173\u001b[0m         \u001b[0mmask_with_bos_eos\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mbilm_output\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'mask'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\myEnv\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m    539\u001b[0m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    540\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 541\u001b[1;33m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    542\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    543\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\myEnv\\lib\\site-packages\\allennlp\\modules\\elmo.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, inputs, word_inputs)\u001b[0m\n\u001b[0;32m    605\u001b[0m                 \u001b[0mtype_representation\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtoken_embedding\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'token_embedding'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    606\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 607\u001b[1;33m             \u001b[0mtoken_embedding\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_token_embedder\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    608\u001b[0m             \u001b[0mmask\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtoken_embedding\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'mask'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    609\u001b[0m             \u001b[0mtype_representation\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtoken_embedding\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'token_embedding'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\myEnv\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m    539\u001b[0m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    540\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 541\u001b[1;33m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    542\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    543\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\myEnv\\lib\\site-packages\\allennlp\\modules\\elmo.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, inputs)\u001b[0m\n\u001b[0;32m    374\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_convolutions\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    375\u001b[0m             \u001b[0mconv\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'char_conv_{}'\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 376\u001b[1;33m             \u001b[0mconvolved\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mconv\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcharacter_embedding\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    377\u001b[0m             \u001b[1;31m# (batch_size * sequence_length, n_filters for this width)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    378\u001b[0m             \u001b[0mconvolved\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0m_\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmax\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mconvolved\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdim\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\myEnv\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m    539\u001b[0m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    540\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 541\u001b[1;33m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    542\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    543\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\myEnv\\lib\\site-packages\\torch\\nn\\modules\\conv.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, input)\u001b[0m\n\u001b[0;32m    200\u001b[0m                             _single(0), self.dilation, self.groups)\n\u001b[0;32m    201\u001b[0m         return F.conv1d(input, self.weight, self.bias, self.stride,\n\u001b[1;32m--> 202\u001b[1;33m                         self.padding, self.dilation, self.groups)\n\u001b[0m\u001b[0;32m    203\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    204\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mRuntimeError\u001b[0m: [enforce fail at ..\\c10\\core\\CPUAllocator.cpp:72] data. DefaultCPUAllocator: not enough memory: you tried to allocate 25482067968 bytes. Buy new RAM!\n"
     ]
    }
   ],
   "source": [
    "te = TextExplainer(\n",
    "    sampler=sampler,\n",
    "    position_dependent=True,\n",
    "    random_state=42,\n",
    "#     char_based =True #True if explanation should be char-based, False if it should be token-based. Default is False.\n",
    ")\n",
    "\n",
    "te.fit(text, predict_func)\n",
    "\n",
    "te.explain_prediction(\n",
    "    target_names=list(explainer_generator.idx2tag.values())\n",
    "   \n",
    ")\n",
    "# https://github.com/TeamHG-Memex/eli5/blob/017c738f8dcf3e31346de49a390835ffafad3f1b/eli5/explain.py#L83\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sentences = [['minu', 'nimi' ,'on', 'nesma', 'hästi', 'Eemaldatud', '700', 'ml', 'plasmat','RR',' 112/60',' mmHg'], ['Pt.', 'tehtud' ,'plasmaferees', 'talus', 'hästi']]\n",
    "# character_ids = batch_to_ids(sentences)\n",
    "# embeddings = elmo(character_ids)\n",
    "# scipy.spatial.distance.cosine(embeddings['elmo_representations'][1][1][4].detach().numpy(),embeddings['elmo_representations'][1][1][1].detach().numpy())"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}