nmt_tutorial.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "nmt_tutorial.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "C8GTvgr7pArM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import unicodedata\n",
        "import re\n",
        "import math\n",
        "import psutil\n",
        "import time\n",
        "import datetime\n",
        "from io import open\n",
        "import random\n",
        "from random import shuffle\n",
        "import argparse\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "import torch\n",
        "from torch.autograd import Variable\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "from torch import optim\n",
        "import torch.cuda\n",
        "\n",
        "\"\"\"this line clears sys to allow for argparse to work as gradient clipper\"\"\"\n",
        "import sys; sys.argv=['']; del sys"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xdBOnTnVrecq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "use_cuda = torch.cuda.is_available()\n",
        "print(use_cuda)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ACDMcwbu4ZcQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\"\"\"This function converts a Unicode string to plain ASCII \n",
        "from https://stackoverflow.com/a/518232/2809427\"\"\"\n",
        "def uniToAscii(sentence):\n",
        "    return ''.join(\n",
        "        c for c in unicodedata.normalize('NFD', sentence)\n",
        "        if unicodedata.category(c) != 'Mn'\n",
        "    )\n",
        "\n",
        "\n",
        "\"\"\"Lowercase, trim, and remove non-letter characters (from pytorch)\"\"\"\n",
        "def normalizeString(s):\n",
        "    s = re.sub(r\" ##AT##-##AT## \", r\" \", s)\n",
        "    s = uniToAscii(s.lower().strip())\n",
        "    s = re.sub(r\"([.!?])\", r\" \\1\", s)\n",
        "    s = re.sub(r\"[^a-zA-Z.!?]+\", r\" \", s)\n",
        "    return s   \n",
        "\n",
        "  \n",
        "\"\"\"Denote patterns that sentences must start with to be kept in dataset. \n",
        "Can be changed if desired (from pytorch)\"\"\"\n",
        "eng_prefixes = (\n",
        "    \"i am \", \"i m \",\n",
        "    \"he is\", \"he s \",\n",
        "    \"she is\", \"she s\",\n",
        "    \"you are\", \"you re \",\n",
        "    \"we are\", \"we re \",\n",
        "    \"they are\", \"they re \"\n",
        ")\n",
        "\n",
        "\n",
        "\n",
        "\"\"\"Filters each input-output pair, keeping sentences that are less than max_length \n",
        "if start_filter is true, also filters out sentences that don't start with eng_prefixes\"\"\"\n",
        "def filterPair(p, max_length, start_filter):\n",
        "    filtered = len(p[0].split(' ')) < max_length and \\\n",
        "        len(p[1].split(' ')) < max_length \n",
        "    if start_filter:\n",
        "        return filtered and p[1].startswith(eng_prefixes)\n",
        "    else:\n",
        "        return filtered\n",
        "\n",
        "\"\"\"Filters all of the input-output language pairs in the dataset using filterPair \n",
        "for each pair (from pytorch)\"\"\"\n",
        "def filterPairs(pairs, max_length, start_filter):\n",
        "    return [pair for pair in pairs if filterPair(pair, max_length, start_filter)]\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SeMWu8LUtE5s",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\"\"\"start of sentence tag\"\"\"\n",
        "SOS_token = 0\n",
        "\n",
        "\"\"\"end of sentence tag\"\"\"\n",
        "EOS_token = 1\n",
        "\n",
        "\"\"\"unknown word tag (this is used to handle words that are not in our Vocabulary)\"\"\"\n",
        "UNK_token = 2\n",
        "\n",
        "\n",
        "\"\"\"Lang class, used to store the vocabulary of each language\"\"\"\n",
        "class Lang:\n",
        "    def __init__(self, language):\n",
        "        self.language_name = language\n",
        "        self.word_to_index = {\"SOS\":SOS_token, \"EOS\":EOS_token, \"<UNK>\":UNK_token}\n",
        "        self.word_to_count = {}\n",
        "        self.index_to_word = {SOS_token: \"SOS\", EOS_token: \"EOS\", UNK_token: \"<UNK>\"}\n",
        "        self.vocab_size = 3\n",
        "        self.cutoff_point = -1\n",
        "\n",
        "\n",
        "    def countSentence(self, sentence):\n",
        "        for word in sentence.split(' '):\n",
        "            self.countWords(word)\n",
        "\n",
        "    \"\"\"counts the number of times each word appears in the dataset\"\"\"\n",
        "    def countWords(self, word):\n",
        "        if word not in self.word_to_count:\n",
        "            self.word_to_count[word] = 1\n",
        "        else:\n",
        "            self.word_to_count[word] += 1\n",
        "\n",
        "    \"\"\"if the number of unique words in the dataset is larger than the\n",
        "    specified max_vocab_size, creates a cutoff point that is used to\n",
        "    leave infrequent words out of the vocabulary\"\"\"\n",
        "    def createCutoff(self, max_vocab_size):\n",
        "        word_freqs = list(self.word_to_count.values())\n",
        "        word_freqs.sort(reverse=True)\n",
        "        if len(word_freqs) > max_vocab_size:\n",
        "            self.cutoff_point = word_freqs[max_vocab_size]\n",
        "\n",
        "    \"\"\"assigns each unique word in a sentence a unique index\"\"\"\n",
        "    def addSentence(self, sentence):\n",
        "        new_sentence = ''\n",
        "        for word in sentence.split(' '):\n",
        "            unk_word = self.addWord(word)\n",
        "            if not new_sentence:\n",
        "                new_sentence =unk_word\n",
        "            else:\n",
        "                new_sentence = new_sentence + ' ' + unk_word\n",
        "        return new_sentence\n",
        "\n",
        "    \"\"\"assigns a word a unique index if not already in vocabulary\n",
        "    and it appeaars often enough in the dataset\n",
        "    (self.word_to_count is larger than self.cutoff_point)\"\"\"\n",
        "    def addWord(self, word):\n",
        "        if self.word_to_count[word] > self.cutoff_point:\n",
        "            if word not in self.word_to_index:\n",
        "                self.word_to_index[word] = self.vocab_size\n",
        "                self.index_to_word[self.vocab_size] = word\n",
        "                self.vocab_size += 1\n",
        "            return word\n",
        "        else:\n",
        "            return self.index_to_word[2]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fifvzYxufA72",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "'''prepares both the input and output Lang classes from the passed dataset'''\n",
        "\n",
        "def prepareLangs(lang1, lang2, file_path, reverse=False):\n",
        "    print(\"Reading lines...\")\n",
        "\n",
        "    if len(file_path) == 2:\n",
        "        lang1_lines = open(file_path[0], encoding='utf-8').\\\n",
        "            read().strip().split('\\n')\n",
        "\n",
        "        lang2_lines = open(file_path[1], encoding='utf-8').\\\n",
        "            read().strip().split('\\n')\n",
        "\n",
        "        if len(lang1_lines) != len(lang2_lines):\n",
        "            print(\"Input and output text sizes do not align\")\n",
        "            print(\"Number of lang1 lines: %s \" %len(lang1_lines))\n",
        "            print(\"Number of lang2 lines: %s \" %len(lang2_lines))\n",
        "            quit()\n",
        "\n",
        "        pairs = []\n",
        "\n",
        "        for line in range(len(lang1_lines)):\n",
        "            pairs.append([normalizeString(lang1_lines[line]),\n",
        "                          normalizeString(lang2_lines[line])])            \n",
        "\n",
        "\n",
        "    elif len(file_path) == 1:\n",
        "        lines = open(file_path[0], encoding='utf-8').\\\n",
        "    \tread().strip().split('\\n')\n",
        "        pairs = [[normalizeString(s) for s in l.split('\\t')] for l in lines]\n",
        "\n",
        "    if reverse:\n",
        "        pairs = [list(reversed(p)) for p in pairs]\n",
        "        input_lang = Lang(lang2)\n",
        "        output_lang = Lang(lang1)\n",
        "    else:\n",
        "        input_lang = Lang(lang1)\n",
        "        output_lang = Lang(lang2)\n",
        "\n",
        "    return input_lang, output_lang, pairs"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "08g-wgM6s8CU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\"\"\"completely prepares both input and output languages \n",
        "and returns cleaned and trimmed train and test pairs\"\"\"\n",
        "\n",
        "def prepareData(lang1, lang2, file_path, max_vocab_size=50000, \n",
        "                reverse=False, trim=0, start_filter=False, perc_train_set=0.9, \n",
        "                print_to=None):\n",
        "    \n",
        "    input_lang, output_lang, pairs = prepareLangs(lang1, lang2, \n",
        "                                                  file_path, reverse)\n",
        "    \n",
        "    print(\"Read %s sentence pairs\" % len(pairs))\n",
        "    \n",
        "    if print_to:\n",
        "        with open(print_to,'a') as f:\n",
        "            f.write(\"Read %s sentence pairs \\n\" % len(pairs))\n",
        "    \n",
        "    if trim != 0:\n",
        "        pairs = filterPairs(pairs, trim, start_filter)\n",
        "        print(\"Trimmed to %s sentence pairs\" % len(pairs))\n",
        "        if print_to:\n",
        "            with open(print_to,'a') as f:\n",
        "                f.write(\"Read %s sentence pairs \\n\" % len(pairs))\n",
        "\n",
        "    print(\"Counting words...\")\n",
        "    for pair in pairs:\n",
        "        input_lang.countSentence(pair[0])\n",
        "        output_lang.countSentence(pair[1])\n",
        "\n",
        "\n",
        "    input_lang.createCutoff(max_vocab_size)\n",
        "    output_lang.createCutoff(max_vocab_size)\n",
        "\n",
        "    pairs = [(input_lang.addSentence(pair[0]),output_lang.addSentence(pair[1])) \n",
        "             for pair in pairs]\n",
        "\n",
        "    shuffle(pairs)\n",
        "    \n",
        "    train_pairs = pairs[:math.ceil(perc_train_set*len(pairs))]\n",
        "    test_pairs = pairs[math.ceil(perc_train_set*len(pairs)):]\n",
        "\n",
        "    print(\"Train pairs: %s\" % (len(train_pairs)))\n",
        "    print(\"Test pairs: %s\" % (len(test_pairs)))\n",
        "    print(\"Counted Words -> Trimmed Vocabulary Sizes (w/ EOS and SOS tags):\")\n",
        "    print(\"%s, %s -> %s\" % (input_lang.language_name, len(input_lang.word_to_count),\n",
        "                            input_lang.vocab_size,))\n",
        "    print(\"%s, %s -> %s\" % (output_lang.language_name, len(output_lang.word_to_count), \n",
        "                            output_lang.vocab_size))\n",
        "    print()\n",
        "\n",
        "    if print_to:\n",
        "        with open(print_to,'a') as f:\n",
        "            f.write(\"Train pairs: %s\" % (len(train_pairs)))\n",
        "            f.write(\"Test pairs: %s\" % (len(test_pairs)))\n",
        "            f.write(\"Counted Words -> Trimmed Vocabulary Sizes (w/ EOS and SOS tags):\")\n",
        "            f.write(\"%s, %s -> %s\" % (input_lang.language_name, \n",
        "                                      len(input_lang.word_to_count),\n",
        "                                      input_lang.vocab_size,))\n",
        "            f.write(\"%s, %s -> %s \\n\" % (output_lang.language_name, len(output_lang.word_to_count), \n",
        "                            output_lang.vocab_size))\n",
        "        \n",
        "    return input_lang, output_lang, train_pairs, test_pairs"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O4Hrm_ortNM2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\"\"\"converts a sentence to one hot encoding vectors - pytorch allows us to just\n",
        "use the number corresponding to the unique index for that word,\n",
        "rather than a complete one hot encoding vector for each word\"\"\"\n",
        "def indexesFromSentence(lang, sentence):\n",
        "    indexes = []\n",
        "    for word in sentence.split(' '):\n",
        "        try:\n",
        "            indexes.append(lang.word_to_index[word])\n",
        "        except:\n",
        "            indexes.append(lang.word_to_index[\"<UNK>\"])\n",
        "    return indexes\n",
        "\n",
        "\n",
        "def tensorFromSentence(lang, sentence):\n",
        "    indexes = indexesFromSentence(lang, sentence)\n",
        "    indexes.append(EOS_token)\n",
        "    result = torch.LongTensor(indexes).view(-1)\n",
        "    if use_cuda:\n",
        "        return result.cuda()\n",
        "    else:\n",
        "        return result\n",
        "      \n",
        "\"\"\"converts a pair of sentence (input and target) to a pair of tensors\"\"\"\n",
        "def tensorsFromPair(input_lang, output_lang, pair):\n",
        "    input_variable = tensorFromSentence(input_lang, pair[0])\n",
        "    target_variable = tensorFromSentence(output_lang, pair[1])\n",
        "    return (input_variable, target_variable)\n",
        "  \n",
        "\n",
        "\"\"\"converts from tensor of one hot encoding vector indices to sentence\"\"\"\n",
        "def sentenceFromTensor(lang, tensor):\n",
        "    raw = tensor.data\n",
        "    words = []\n",
        "    for num in raw:\n",
        "        words.append(lang.index_to_word[num.item()])\n",
        "    return ' '.join(words)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nONw2n6X2jHM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\"\"\"seperates data into batches of size batch_size\"\"\"\n",
        "def batchify(data, input_lang, output_lang, batch_size, shuffle_data=True):\n",
        "    if shuffle_data == True:\n",
        "        shuffle(data)\n",
        "    number_of_batches = len(data) // batch_size\n",
        "    batches = list(range(number_of_batches))\n",
        "    longest_elements = list(range(number_of_batches))\n",
        "    \n",
        "    for batch_number in range(number_of_batches):\n",
        "        longest_input = 0\n",
        "        longest_target = 0\n",
        "        input_variables = list(range(batch_size))\n",
        "        target_variables = list(range(batch_size))\n",
        "        index = 0      \n",
        "        for pair in range((batch_number*batch_size),((batch_number+1)*batch_size)):\n",
        "            input_variables[index], target_variables[index] = tensorsFromPair(input_lang, output_lang, data[pair])\n",
        "            if len(input_variables[index]) >= longest_input:\n",
        "                longest_input = len(input_variables[index])\n",
        "            if len(target_variables[index]) >= longest_target:\n",
        "                longest_target = len(target_variables[index])\n",
        "            index += 1\n",
        "        batches[batch_number] = (input_variables, target_variables)\n",
        "        longest_elements[batch_number] = (longest_input, longest_target)\n",
        "    return batches , longest_elements, number_of_batches\n",
        "\n",
        "\n",
        "\"\"\"pads batches to allow for sentences of variable lengths to be computed in parallel\"\"\"\n",
        "def pad_batch(batch):\n",
        "    padded_inputs = torch.nn.utils.rnn.pad_sequence(batch[0],padding_value=EOS_token)\n",
        "    padded_targets = torch.nn.utils.rnn.pad_sequence(batch[1],padding_value=EOS_token)\n",
        "    return (padded_inputs, padded_targets)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c6a1ErP7tGTN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class EncoderRNN(nn.Module):\n",
        "\tdef __init__(self,input_size,hidden_size,layers=1,dropout=0.1,\n",
        "               bidirectional=True):\n",
        "\t\tsuper(EncoderRNN, self).__init__()\n",
        "\n",
        "\t\tif bidirectional:\n",
        "\t\t\tself.directions = 2\n",
        "\t\telse:\n",
        "\t\t\tself.directions = 1\n",
        "\t\tself.input_size = input_size\n",
        "\t\tself.hidden_size = hidden_size\n",
        "\t\tself.num_layers = layers\n",
        "\t\tself.dropout = dropout\n",
        "\t\tself.embedder = nn.Embedding(input_size,hidden_size)\n",
        "\t\tself.dropout = nn.Dropout(dropout)\n",
        "\t\tself.lstm = nn.LSTM(input_size=hidden_size,hidden_size=hidden_size,\n",
        "                        num_layers=layers,dropout=dropout,\n",
        "                        bidirectional=bidirectional,batch_first=False)\n",
        "\t\tself.fc = nn.Linear(hidden_size*self.directions, hidden_size)\n",
        "\n",
        "\tdef forward(self, input_data, h_hidden, c_hidden):\n",
        "\t\tembedded_data = self.embedder(input_data)\n",
        "\t\tembedded_data = self.dropout(embedded_data)\n",
        "\t\thiddens, outputs = self.lstm(embedded_data, (h_hidden, c_hidden))\n",
        "\n",
        "\t\treturn hiddens, outputs\n",
        "\n",
        "\t\"\"\"creates initial hidden states for encoder corresponding to batch size\"\"\"\n",
        "\tdef create_init_hiddens(self, batch_size):\n",
        "\t\th_hidden = Variable(torch.zeros(self.num_layers*self.directions, \n",
        "                                    batch_size, self.hidden_size))\n",
        "\t\tc_hidden = Variable(torch.zeros(self.num_layers*self.directions, \n",
        "                                    batch_size, self.hidden_size))\n",
        "\t\tif torch.cuda.is_available():\n",
        "\t\t\treturn h_hidden.cuda(), c_hidden.cuda()\n",
        "\t\telse:\n",
        "\t\t\treturn h_hidden, c_hidden"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DCgWDqL8txLp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class DecoderAttn(nn.Module):\n",
        "\tdef __init__(self, hidden_size, output_size, layers=1, dropout=0.1, bidirectional=True):\n",
        "\t\tsuper(DecoderAttn, self).__init__()\n",
        "\n",
        "\t\tif bidirectional:\n",
        "\t\t\tself.directions = 2\n",
        "\t\telse:\n",
        "\t\t\tself.directions = 1\n",
        "\t\tself.output_size = output_size\n",
        "\t\tself.hidden_size = hidden_size\n",
        "\t\tself.num_layers = layers\n",
        "\t\tself.dropout = dropout\n",
        "\t\tself.embedder = nn.Embedding(output_size,hidden_size)\n",
        "\t\tself.dropout = nn.Dropout(dropout)\n",
        "\t\tself.score_learner = nn.Linear(hidden_size*self.directions, \n",
        "                                   hidden_size*self.directions)\n",
        "\t\tself.lstm = nn.LSTM(input_size=hidden_size,hidden_size=hidden_size,\n",
        "                        num_layers=layers,dropout=dropout,\n",
        "                        bidirectional=bidirectional,batch_first=False)\n",
        "\t\tself.context_combiner = nn.Linear((hidden_size*self.directions)\n",
        "                                      +(hidden_size*self.directions), hidden_size)\n",
        "\t\tself.tanh = nn.Tanh()\n",
        "\t\tself.output = nn.Linear(hidden_size, output_size)\n",
        "\t\tself.soft = nn.Softmax(dim=1)\n",
        "\t\tself.log_soft = nn.LogSoftmax(dim=1)\n",
        "\n",
        "\n",
        "\tdef forward(self, input_data, h_hidden, c_hidden, encoder_hiddens):\n",
        "\n",
        "\t\tembedded_data = self.embedder(input_data)\n",
        "\t\tembedded_data = self.dropout(embedded_data)\t\n",
        "\t\tbatch_size = embedded_data.shape[1]\n",
        "\t\thiddens, outputs = self.lstm(embedded_data, (h_hidden, c_hidden))\t\n",
        "\t\ttop_hidden = outputs[0].view(self.num_layers,self.directions,\n",
        "                                 hiddens.shape[1],\n",
        "                                 self.hidden_size)[self.num_layers-1]\n",
        "\t\ttop_hidden = top_hidden.permute(1,2,0).contiguous().view(batch_size,-1, 1)\n",
        "\n",
        "\t\tprep_scores = self.score_learner(encoder_hiddens.permute(1,0,2))\n",
        "\t\tscores = torch.bmm(prep_scores, top_hidden)\n",
        "\t\tattn_scores = self.soft(scores)\n",
        "\t\tcon_mat = torch.bmm(encoder_hiddens.permute(1,2,0),attn_scores)\n",
        "\t\th_tilde = self.tanh(self.context_combiner(torch.cat((con_mat,\n",
        "                                                         top_hidden),dim=1)\n",
        "                                              .view(batch_size,-1)))\n",
        "\t\tpred = self.output(h_tilde)\n",
        "\t\tpred = self.log_soft(pred)\n",
        "\n",
        "\t\t\n",
        "\t\treturn pred, outputs"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "344LVWD1VXFF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "'''Performs training on a single batch of training data. Computing the loss \n",
        "according to the passed loss_criterion and back-propagating on this loss.'''\n",
        "\n",
        "def train_batch(input_batch, target_batch, encoder, decoder, \n",
        "                encoder_optimizer, decoder_optimizer, loss_criterion):\n",
        "\tencoder_optimizer.zero_grad()\n",
        "\tdecoder_optimizer.zero_grad()\n",
        "\tloss = 0\n",
        "\tenc_h_hidden, enc_c_hidden = encoder.create_init_hiddens(input_batch.shape[1])\n",
        "\n",
        "\tenc_hiddens, enc_outputs = encoder(input_batch, enc_h_hidden, enc_c_hidden)\n",
        "\n",
        "\tdecoder_input = Variable(torch.LongTensor(1,input_batch.shape[1]).\n",
        "                           fill_(output_lang.word_to_index.get(\"SOS\")).cuda()) if use_cuda \\\n",
        "\t\t\t\t\telse Variable(torch.LongTensor(1,input_batch.shape[1]).\n",
        "                        fill_(output_lang.word_to_index.get(\"SOS\")))\n",
        "\n",
        "\tdec_h_hidden = enc_outputs[0]\n",
        "\tdec_c_hidden = enc_outputs[1]\n",
        "\t\n",
        "\tfor i in range(target_batch.shape[0]):\n",
        "\t\tpred, dec_outputs = decoder(decoder_input, dec_h_hidden, \n",
        "                                dec_c_hidden, enc_hiddens)\n",
        "\n",
        "\t\tdecoder_input = target_batch[i].view(1,-1)\n",
        "\t\tdec_h_hidden = dec_outputs[0]\n",
        "\t\tdec_c_hidden = dec_outputs[1]\n",
        "\t\t\n",
        "\t\tloss += loss_criterion(pred,target_batch[i])\n",
        "\n",
        "\n",
        "\tloss.backward()\n",
        "\n",
        "\ttorch.nn.utils.clip_grad_norm_(encoder.parameters(),args.clip)\n",
        "\ttorch.nn.utils.clip_grad_norm_(decoder.parameters(),args.clip)\n",
        "\n",
        "\tencoder_optimizer.step()\n",
        "\tdecoder_optimizer.step()\n",
        "\n",
        "\treturn loss.item() / target_batch.shape[0]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_AkStAnhVYs-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "'''Performs a complete epoch of training through all of the training_batches'''\n",
        "\n",
        "def train(train_batches, encoder, decoder, encoder_optimizer, decoder_optimizer, loss_criterion):\n",
        "\n",
        "\tround_loss = 0\n",
        "\ti = 1\n",
        "\tfor batch in train_batches:\n",
        "\t\ti += 1\n",
        "\t\t(input_batch, target_batch) = pad_batch(batch)\n",
        "\t\tbatch_loss = train_batch(input_batch, target_batch, encoder, decoder, encoder_optimizer, decoder_optimizer, loss_criterion)\n",
        "\t\tround_loss += batch_loss\n",
        "\n",
        "\treturn round_loss / len(train_batches)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Xdu2FER-Vcnk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "'''Evaluates the loss on a single batch of test data. Computing the loss \n",
        "according to the passed loss_criterion. Does not perform back-prop'''\n",
        "\n",
        "def test_batch(input_batch, target_batch, encoder, decoder, loss_criterion):\n",
        "\t\n",
        "\tloss = 0\n",
        "\n",
        "\t#create initial hidde state for encoder\n",
        "\tenc_h_hidden, enc_c_hidden = encoder.create_init_hiddens(input_batch.shape[1])\n",
        "\n",
        "\tenc_hiddens, enc_outputs = encoder(input_batch, enc_h_hidden, enc_c_hidden)\n",
        "\n",
        "\tdecoder_input = Variable(torch.LongTensor(1,input_batch.shape[1]).\n",
        "                           fill_(output_lang.word_to_index.get(\"SOS\")).cuda()) if use_cuda \\\n",
        "\t\t\t\t\telse Variable(torch.LongTensor(1,input_batch.shape[1]).\n",
        "                        fill_(output_lang.word_to_index.get(\"SOS\")))\n",
        "\tdec_h_hidden = enc_outputs[0]\n",
        "\tdec_c_hidden = enc_outputs[1]\n",
        "\t\n",
        "\tfor i in range(target_batch.shape[0]):\n",
        "\t\tpred, dec_outputs = decoder(decoder_input, dec_h_hidden, dec_c_hidden, enc_hiddens)\n",
        "\n",
        "\t\ttopv, topi = pred.topk(1,dim=1)\n",
        "\t\tni = topi.view(1,-1)\n",
        "\t\t\n",
        "\t\tdecoder_input = ni\n",
        "\t\tdec_h_hidden = dec_outputs[0]\n",
        "\t\tdec_c_hidden = dec_outputs[1]\n",
        "\n",
        "\t\tloss += loss_criterion(pred,target_batch[i])\n",
        "\t\t\n",
        "\treturn loss.item() / target_batch.shape[0]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "L9bKcK3hVd-F",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "'''Computes the loss value over all of the test_batches'''\n",
        "\n",
        "def test(test_batches, encoder, decoder, loss_criterion):\n",
        "\n",
        "\twith torch.no_grad():\n",
        "\t\ttest_loss = 0\n",
        "\n",
        "\t\tfor batch in test_batches:\n",
        "\t\t\t(input_batch, target_batch) = pad_batch(batch)\n",
        "\t\t\tbatch_loss = test_batch(input_batch, target_batch, encoder, decoder, loss_criterion)\n",
        "\t\t\ttest_loss += batch_loss\n",
        "\n",
        "\treturn test_loss / len(test_batches)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2tBxl0KlVjnE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "'''Returns the predicted translation of a given input sentence. Predicted\n",
        "translation is trimmed to length of cutoff_length argument'''\n",
        "\n",
        "def evaluate(encoder, decoder, sentence, cutoff_length):\n",
        "\twith torch.no_grad():\n",
        "\t\tinput_variable = tensorFromSentence(input_lang, sentence)\n",
        "\t\tinput_variable = input_variable.view(-1,1)\n",
        "\t\tenc_h_hidden, enc_c_hidden = encoder.create_init_hiddens(1)\n",
        "\n",
        "\t\tenc_hiddens, enc_outputs = encoder(input_variable, enc_h_hidden, enc_c_hidden)\n",
        "\n",
        "\t\tdecoder_input = Variable(torch.LongTensor(1,1).fill_(output_lang.word_to_index.get(\"SOS\")).cuda()) if use_cuda \\\n",
        "\t\t\t\t\t\telse Variable(torch.LongTensor(1,1).fill_(output_lang.word_to_index.get(\"SOS\")))\n",
        "\t\tdec_h_hidden = enc_outputs[0]\n",
        "\t\tdec_c_hidden = enc_outputs[1]\n",
        "\n",
        "\t\tdecoded_words = []\n",
        "\n",
        "\t\tfor di in range(cutoff_length):\n",
        "\t\t\tpred, dec_outputs = decoder(decoder_input, dec_h_hidden, dec_c_hidden, enc_hiddens)\n",
        "\n",
        "\t\t\ttopv, topi = pred.topk(1,dim=1)\n",
        "\t\t\tni = topi.item()\n",
        "\t\t\tif ni == output_lang.word_to_index.get(\"EOS\"):\n",
        "\t\t\t\tdecoded_words.append('<EOS>')\n",
        "\t\t\t\tbreak\n",
        "\t\t\telse:\n",
        "\t\t\t\tdecoded_words.append(output_lang.index_to_word[ni])\n",
        "\n",
        "\t\t\tdecoder_input = Variable(torch.LongTensor(1,1).fill_(ni).cuda()) if use_cuda \\\n",
        "\t\t\t\t\t\t\telse Variable(torch.LongTensor(1,1).fill_(ni))\n",
        "\t\t\tdec_h_hidden = dec_outputs[0]\n",
        "\t\t\tdec_c_hidden = dec_outputs[1]\n",
        "\n",
        "\t\toutput_sentence = ' '.join(decoded_words)\n",
        "    \n",
        "\t\treturn output_sentence"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q_BVKYuEVk-S",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "'''Evaluates prediction translations for a specified number (n) of sentences\n",
        "chosen randomly from a list of passed sentence pairs. Returns three sentences\n",
        "in the format:\n",
        "                  > input sentence\n",
        "                  = correct translation\n",
        "                  < predicted translation'''\n",
        "\n",
        "def evaluate_randomly(encoder, decoder, pairs, n=2, trim=100):\n",
        "\tfor i in range(n):\n",
        "\t\tpair = random.choice(pairs)\n",
        "\t\tprint('>', pair[0])\n",
        "\t\tprint('=', pair[1])\n",
        "\t\toutput_sentence = evaluate(encoder, decoder, pair[0],cutoff_length=trim)\n",
        "\t\tprint('<', output_sentence)\n",
        "\t\tprint('')    \n",
        "\t\tif create_txt:\n",
        "\t\t\tf = open(print_to, 'a')\n",
        "\t\t\tf.write(\"\\n \\\n",
        "\t\t\t\t> %s \\n \\\n",
        "\t\t\t\t= %s \\n \\\n",
        "\t\t\t\t< %s \\n\" % (pair[0], pair[1], output_sentence))\n",
        "\t\t\tf.close()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "n_N2hD47WTf_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "'''Used to plot the progress of training. Plots the loss value vs. time'''\n",
        "def showPlot(times, losses, fig_name):\n",
        "    x_axis_label = 'Minutes'\n",
        "    colors = ('red','blue')\n",
        "    if max(times) >= 120:\n",
        "    \ttimes = [mins/60 for mins in times]\n",
        "    \tx_axis_label = 'Hours'\n",
        "    i = 0\n",
        "    for key, losses in losses.items():\n",
        "    \tif len(losses) > 0:\n",
        "    \t\tplt.plot(times, losses, label=key, color=colors[i])\n",
        "    \t\ti += 1\n",
        "    plt.legend(loc='upper left')\n",
        "    plt.xlabel(x_axis_label)\n",
        "    plt.ylabel('Loss')\n",
        "    plt.title('Training Results')\n",
        "    plt.savefig(fig_name+'.png')\n",
        "    plt.close('all')\n",
        "    \n",
        "'''prints the current memory consumption'''\n",
        "def mem():\n",
        "\tif use_cuda:\n",
        "\t\tmem = torch.cuda.memory_allocated()/1e7\n",
        "\telse:\n",
        "\t\tmem = psutil.cpu_percent()\n",
        "\tprint('Current mem usage:')\n",
        "\tprint(mem)\n",
        "\treturn \"Current mem usage: %s \\n\" % (mem)\n",
        "\n",
        "'''converts a time measurement in seconds to hours'''\n",
        "def asHours(s):\n",
        "\tm = math.floor(s / 60)\n",
        "\th = math.floor(m / 60)\n",
        "\ts -= m * 60\n",
        "\tm -= h * 60\n",
        "\treturn '%dh %dm %ds' % (h, m, s)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "701CcLqgWYrE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "'''The master function that trains the model. Evlautes progress on the train set\n",
        "(if present) and also records the progress of training in both a txt file and\n",
        "a png graph. Also can save the weights of both the Encoder and Decoder \n",
        "for future use.'''\n",
        "\n",
        "def train_and_test(epochs, test_eval_every, plot_every, learning_rate, \n",
        "                   lr_schedule, train_pairs, test_pairs, input_lang, \n",
        "                   output_lang, batch_size, test_batch_size, encoder, decoder, \n",
        "                   loss_criterion, trim, save_weights):\n",
        "\t\n",
        "\ttimes = []\n",
        "\tlosses = {'train set':[], 'test set': []}\n",
        "\n",
        "\ttest_batches, longest_seq, n_o_b = batchify(test_pairs, input_lang, \n",
        "                                              output_lang, test_batch_size, \n",
        "                                              shuffle_data=False)\n",
        "\n",
        "\tstart = time.time()\n",
        "\tfor i in range(1,epochs+1):\n",
        "    \n",
        "\t\t'''adjust the learning rate according to the learning rate schedule\n",
        "\t\tspecified in lr_schedule'''\n",
        "\t\tif i in lr_schedule.keys():\n",
        "\t\t\tlearning_rate /= lr_schedule.get(i)\n",
        "\n",
        "\n",
        "\t\tencoder.train()\n",
        "\t\tdecoder.train()\n",
        "\n",
        "\t\tencoder_optimizer = optim.SGD(encoder.parameters(), lr=learning_rate)\n",
        "\t\tdecoder_optimizer = optim.SGD(decoder.parameters(), lr=learning_rate)\n",
        "\n",
        "\t\tbatches, longest_seq, n_o_b = batchify(train_pairs, input_lang, \n",
        "                                           output_lang, batch_size, \n",
        "                                           shuffle_data=True)\n",
        "\t\ttrain_loss = train(batches, encoder, decoder, encoder_optimizer, \n",
        "                       decoder_optimizer, loss_criterion)\n",
        "\t\t\n",
        "\t\tnow = time.time()\n",
        "\t\tprint(\"Iter: %s \\nLearning Rate: %s \\nTime: %s \\nTrain Loss: %s \\n\" \n",
        "          % (i, learning_rate, asHours(now-start), train_loss))\n",
        "\n",
        "\t\tif create_txt:\n",
        "\t\t\twith open(print_to, 'a') as f:\n",
        "\t\t\t\tf.write(\"Iter: %s \\nLeaning Rate: %s \\nTime: %s \\nTrain Loss: %s \\n\" \\\n",
        "\t\t\t\t\t% (i, learning_rate, asHours(now-start), train_loss))\n",
        "\n",
        "\t\tif i % test_eval_every == 0:\n",
        "\t\t\tif test_pairs:\n",
        "\t\t\t\ttest_loss = test(test_batches, encoder, decoder, criterion)\n",
        "\t\t\t\tprint(\"Test set loss: %s\" % (test_loss))\n",
        "\t\t\t\tif create_txt:\n",
        "\t\t\t\t\twith open(print_to, 'a') as f:\n",
        "\t\t\t\t\t\tf.write(\"Test Loss: %s \\n\" % (test_loss))\n",
        "\t\t\t\tevaluate_randomly(encoder, decoder, test_pairs, trim)\n",
        "\t\t\telse:\n",
        "\t\t\t\tevaluate_randomly(encoder, decoder, train_pairs, trim)\n",
        "\n",
        "\t\tif i % plot_every == 0:\n",
        "\t\t\ttimes.append((time.time()-start)/60)\n",
        "\t\t\tlosses['train set'].append(train_loss)\n",
        "\t\t\tif test_pairs:\n",
        "\t\t\t\tlosses['test set'].append(test_loss)\n",
        "\t\t\tshowPlot(times, losses, output_file_name)\n",
        "\t\t\tif save_weights:\n",
        "\t\t\t\ttorch.save(encoder.state_dict(), output_file_name+'_enc_weights.pt')\n",
        "\t\t\t\ttorch.save(decoder.state_dict(), output_file_name+'_dec_weights.pt')\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lcNvrlwkuKmG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from google.colab import files\n",
        "uploaded = files.upload()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QxNonFx4upl8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ls"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2yqYvI7Xt-Ot",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\"\"\"PROVIDE INFORMATION ON THE DATASET AND DATASET CLEANING\"\"\"\n",
        "\n",
        "input_lang_name = 'fre'\n",
        "output_lang_name = 'en'\n",
        "\n",
        "\"\"\"name of your dataset\"\"\"\n",
        "dataset = 'orig'\n",
        "\n",
        "\"\"\"file path of dataset in the form of a tuple. If translated sentences are\n",
        "stored in two files, this tuple will have two elements\"\"\"\n",
        "raw_data_file_path = ('eng-fra.txt',)\n",
        "\n",
        "\"\"\"True if you want to reverse the order of the sentence pairs. For example, \n",
        "in our dataset the sentence pairs list the English sentence first followed by\n",
        "the French translation. But we want to translate from French to English,\n",
        "so we set reverse as True.\"\"\"\n",
        "reverse=True\n",
        "\n",
        "\"\"\"Remove sentences from dataset that are longer than trim (in either language)\"\"\"\n",
        "trim = 10\n",
        "\n",
        "\"\"\"max number of words in the vocabulary for both languages\"\"\"\n",
        "max_vocab_size= 20000\n",
        "\n",
        "\"\"\"if true removes sentences from the dataset that don't start with eng_prefixes.\n",
        "Typically will want to use False, but implemented to compare results with Pytorch\n",
        "tutorial. Can also change the eng_prefixes to prefixes of other languages or\n",
        "other English prefixes. Just be sure that the prefixes apply to the OUTPUT\n",
        "language (i.e. the language that the model is translating to NOT from)\"\"\"\n",
        "start_filter = True\n",
        "\n",
        "\"\"\"denotes what percentage of the data to use as training data. the remaining \n",
        "percentage becomes test data. Typically want to use 0.8-0.9. 1.0 used here to \n",
        "compare with PyTorch results where no test set was utilized\"\"\"\n",
        "perc_train_set = 1.0"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7aTcns-8HNT-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\"\"\"OUTPUT OPTIONS\"\"\"\n",
        "\n",
        "\"\"\"denotes how often to evaluate a loss on the test set and print\n",
        "sample predictions on the test set.\n",
        "if no test set, simply prints sample predictions on the train set.\"\"\"\n",
        "test_eval_every = 1\n",
        "\n",
        "\"\"\"denotes how often to plot the loss values of train and test (if applicable)\"\"\"\n",
        "plot_every = 1\n",
        "\n",
        "\"\"\"if true creates a txt file of the output\"\"\"\n",
        "create_txt = True\n",
        "\n",
        "\"\"\"if true saves the encoder and decoder weights to seperate .pt files for later use\"\"\"\n",
        "save_weights = False"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MC4-4tIkWuqp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\"\"\"HYPERPARAMETERS: FEEL FREE TO PLAY WITH THESE TO TRY TO ACHIEVE BETTER RESULTS\"\"\"\n",
        "\n",
        "\"\"\"signifies whether the Encoder and Decoder should be bidirectional LSTMs or not\"\"\"\n",
        "bidirectional = True\n",
        "if bidirectional:\n",
        "\tdirections = 2\n",
        "else:\n",
        "\tdirections = 1\n",
        "\n",
        "\"\"\"number of layers in both the Encoder and Decoder\"\"\"\n",
        "layers = 2\n",
        "\n",
        "\"\"\"Hidden size of the Encoder and Decoder\"\"\"\n",
        "hidden_size = 440\n",
        "\n",
        "\"\"\"Dropout value for Encoder and Decoder\"\"\"\n",
        "dropout = 0.2\n",
        "\n",
        "\"\"\"Training set batch size\"\"\"\n",
        "batch_size = 32\n",
        "\n",
        "\"\"\"Test set batch size\"\"\"\n",
        "test_batch_size = 32\n",
        "\n",
        "\"\"\"number of epochs (full passes through the training data)\"\"\"\n",
        "epochs = 100\n",
        "\n",
        "\"\"\"Initial learning rate\"\"\"\n",
        "learning_rate= 1\n",
        "\n",
        "\"\"\"Learning rate schedule. Signifies by what factor to divide the learning rate\n",
        "at a certain epoch. For example {5:10} would divide the learning rate by 10\n",
        "before the 5th epoch and {5:10, 10:100} would divide the learning rate by 10\n",
        "before the 5th epoch and then again by 100 before the 10th epoch\"\"\"\n",
        "lr_schedule = {}\n",
        "\n",
        "\"\"\"loss criterion, see https://pytorch.org/docs/stable/nn.html for other options\"\"\"\n",
        "criterion = nn.NLLLoss()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KSB24rtUW1I9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\"\"\"******************************************************************\n",
        "********************NO NEED TO ALTER ANYTHING BELOW******************\n",
        "******************************************************************\"\"\"\n",
        "\n",
        "use_cuda = torch.cuda.is_available()\n",
        "\n",
        "\n",
        "\"\"\"for plotting of the loss\"\"\"\n",
        "plt.switch_backend('agg')\n",
        "\n",
        "output_file_name = \"testdata.%s_trim.%s_vocab.%s_directions.%s_layers.%s_hidden.%s_dropout.%s_learningrate.%s_batch.%s_epochs.%s\" % (dataset,trim,max_vocab_size,directions,layers,hidden_size,dropout,learning_rate,batch_size,epochs)\n",
        "\n",
        "if create_txt:\n",
        "\tprint_to = output_file_name+'.txt'\n",
        "\twith open(print_to, 'w+') as f:\n",
        "\t\tf.write(\"Starting Training \\n\")\n",
        "else:\n",
        "\tprint_to = None\n",
        "\n",
        "input_lang, output_lang, train_pairs, test_pairs = prepareData(\n",
        "    input_lang_name, output_lang_name, raw_data_file_path, \n",
        "    max_vocab_size=max_vocab_size, reverse=reverse, trim=trim, \n",
        "    start_filter=start_filter, perc_train_set=perc_train_set, print_to=print_to)\n",
        "print('Train Pairs #')\n",
        "print(len(train_pairs))\n",
        "\n",
        "\n",
        "\"\"\"for gradient clipping from \n",
        "https://github.com/pytorch/examples/blob/master/word_language_model/main.py\"\"\"\n",
        "parser = argparse.ArgumentParser(description='PyTorch Wikitext-2 RNN/LSTM Language Model')\n",
        "parser.add_argument('--clip', type=float, default=0.25,\n",
        "                    help='gradient clipping')\n",
        "args = parser.parse_args()\n",
        "\n",
        "mem()\n",
        "\n",
        "if create_txt:\n",
        "\twith open(print_to, 'a') as f:\n",
        "\t\tf.write(\"\\nRandom Train Pair: %s \\n\\nRandom Test Pair: %s \\n\\n\" \n",
        "            % (random.choice(train_pairs),random.choice(test_pairs) \n",
        "               if test_pairs else \"None\"))\n",
        "\t\tf.write(mem())\n",
        "\n",
        "\n",
        "\"\"\"create the Encoder\"\"\"\n",
        "encoder = EncoderRNN(input_lang.vocab_size, hidden_size, layers=layers, \n",
        "                     dropout=dropout, bidirectional=bidirectional)\n",
        "\n",
        "\"\"\"create the Decoder\"\"\"\n",
        "decoder = DecoderAttn(hidden_size, output_lang.vocab_size, layers=layers, \n",
        "                      dropout=dropout, bidirectional=bidirectional)\n",
        "\n",
        "print('Encoder and Decoder Created')\n",
        "mem()\n",
        "\n",
        "if use_cuda:\n",
        "\tprint('Cuda being used')\n",
        "\tencoder = encoder.cuda()\n",
        "\tdecoder = decoder.cuda()\n",
        "\n",
        "print('Number of epochs: '+str(epochs))\n",
        "\n",
        "if create_txt:\n",
        "\twith open(print_to, 'a') as f:\n",
        "\t\tf.write('Encoder and Decoder Created\\n')\n",
        "\t\tf.write(mem())\n",
        "\t\tf.write(\"Number of epochs %s \\n\" % (epochs))\n",
        "\n",
        "train_and_test(epochs, test_eval_every, plot_every, learning_rate, lr_schedule, \n",
        "               train_pairs, test_pairs, input_lang, output_lang, batch_size, \n",
        "               test_batch_size, encoder, decoder, criterion, trim, save_weights)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ghhKL3IEwIhW",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ls"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GZd1JuDpZVrH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from google.colab import files\n",
        "files.download('testdata.orig_trim.10_vocab.20000_directions.2_layers.2_hidden.440_dropout.0.2_learningrate.1_batch.32_epochs.100.txt') "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Kv8oh5K6wYWD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "outside_sent = \"ils ont tenu une réunion au café local en bas de la rue.\"\n",
        "outside_sent = normalizeString(outside_sent)\n",
        "evaluate(encoder, decoder, outside_sent, cutoff_length=10)"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}