Transformer.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch import nn\n",
    "import torchaudio\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from IPython.display import Audio\n",
    "from librosa import display\n",
    "import re\n",
    "import os\n",
    "import math\n",
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = \"cuda\" if torch.cuda.is_available() else \"cpu\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TextDataset(torch.utils.data.Dataset):\n",
    "\n",
    "    def __init__(self, text_path):\n",
    "\n",
    "        self.text_path = text_path\n",
    "\n",
    "        self.phrases = self._get_phrases()\n",
    "        self.words = self._get_words(self.phrases)\n",
    "\n",
    "        self.vocabulary = self._create_vocabulary(self.words)\n",
    "\n",
    "        self.max_length, self.real_length = self._get_text_length()\n",
    "\n",
    "    \n",
    "    def _get_phrases(self):\n",
    "\n",
    "        phrases = []\n",
    "\n",
    "        with open(self.text_path, 'r') as f:\n",
    "\n",
    "            for i in f:\n",
    "\n",
    "                phrases.append(i)\n",
    "\n",
    "            f.close()\n",
    "\n",
    "        #phrases = [i.replace('\\n', '') for i in phrases] # Not really necessary here\n",
    "        phrases = [i.replace('\"', ' \" ').replace(\"'ve\", \" 've\").replace(\"'d\", \" 'd\").replace(\"'t\", \" 't\").replace(\"'re\", \" 're\").replace(\"'s\", \" 's\") for i in phrases]\n",
    "        phrases = [i.replace(',', ' ,').replace(\".\", \" .\").replace(\"!\", \" !\").replace(\"?\", \" ?\").replace(\"(\", \" ( \").replace(\")\", \" ) \") for i in phrases]\n",
    "        phrases = [i.replace(':', ' :').replace(\";\", \" ;\").replace(\"-\", \" - \") for i in phrases]\n",
    "        \n",
    "        return phrases\n",
    "    \n",
    "    def _get_words(self, phrases):\n",
    "        words = ' '.join(phrases)\n",
    "        words = words.split(' ')\n",
    "\n",
    "        return words\n",
    "    \n",
    "    def _get_letters(self, words):\n",
    "        letters = ' '.join(words)\n",
    "        letters = ''.join(letters.split())\n",
    "        letters = [i for i in letters]\n",
    "\n",
    "        return letters\n",
    "    \n",
    "    def _create_vocabulary(self, words):\n",
    "        idx2word = [\"<pad>\", \" \", \",\", \"'\", \".\", \":\", \";\", \"!\", \"?\", \"(\", \")\", 're', 'll', 've', 'd', 'm', 't', 's', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '\"', \"-\"]\n",
    "\n",
    "        for word in words:\n",
    "            word = re.sub(\"[\\s]\", '', word)\n",
    "            if word not in idx2word:\n",
    "                idx2word.append(word)\n",
    "\n",
    "        idx2word.append(\"<EOS>\")\n",
    "        idx2word.append(\"<SOS>\")\n",
    "\n",
    "        return idx2word\n",
    "    \n",
    "    def _get_text_length(self):\n",
    "\n",
    "        maximum_length = 0\n",
    "\n",
    "        real_length = []\n",
    "        \n",
    "        for sentence in self.phrases:\n",
    "            word_length = 0\n",
    "\n",
    "            for character in sentence:\n",
    "                word_length += len(character) # Including spaces between words\n",
    "\n",
    "            sentence_length = word_length\n",
    "            real_length.append(sentence_length)\n",
    "\n",
    "            if sentence_length > maximum_length:\n",
    "                maximum_length = sentence_length\n",
    "        \n",
    "        maximum_length += 2 # In order to include an extra space and \"<EOS>\"\n",
    "\n",
    "        return maximum_length, real_length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "FILE_PATH = 'C:/Users/giova/OneDrive/Área de Trabalho/Faster than the Flame.txt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "FILE_PATH = 'C:/Users/giova/OneDrive/Área de Trabalho/Texte.txt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = TextDataset(FILE_PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['<pad>', ' ', ',', \"'\", '.', ':', ';', '!', '?', '(', ')', 're', 'll', 've', 'd', 'm', 't', 's', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '\"', '-', 'Faster', 'faster', 'than', 'the', 'flame', 'Fists', 'up', 'in', 'air', 'tonight', 'Leave', 'sane', 'unleash', 'wild', 'This', 'is', 'our', 'time', 'this', 'fate', 'Pyres', 'will', 'inflame', 'night', 'Restless', 'world', 'alight', 'war', 'last', 'crusade', 'Incendere', '', 'cendere', 'Inflammatum', 'flammatum', 'Illuminatum', 'And', 'we', 'all', 'rise', 'against', 'damned', 'Stand', 'for', 'heaven', 'Hold', 'pastor', 'together', 'by', 'chain', 'at', \"'re\", 'going', 'When', 'set', 'on', 'fire', 'Rolling', 'Send', 'us', 'to', 'Be', 'prepared', 'sacrifice', 'raid', 'Embers', 'left', 'those', 'alive', 'Madness', 'raised', 'ignite', 'let', 'reign', 'Flame', 'burning', 'heavens', 'name', 'from', 'Lord', 'came', 'Armageddon', 'proclaim', 'now', 'shall', 'lights', 'sky', 'and', 'no', 'sin', 'can', 'deny', 'We', 'burn', '<EOS>', '<SOS>']\n"
     ]
    }
   ],
   "source": [
    "print(dataset.vocabulary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Faster , faster , faster than the flame\\n', 'Faster , faster , faster than the flame\\n', 'Fists up in the air tonight\\n', 'Leave the sane , unleash the wild\\n', 'This is our time , this is our fate\\n', 'Pyres will inflame the night\\n', 'Restless is the world alight\\n', 'This is our war , the last crusade\\n', 'Incendere  ( cendere ) \\n', 'Inflammatum  ( flammatum ) \\n', 'Illuminatum\\n', 'And we all rise against the damned\\n', 'Stand up faster for heaven , faster than the flame\\n', 'Hold the pastor together , pastor by the chain\\n', \"And at night we 're going wild\\n\", 'When we set the world on fire\\n', 'Rolling faster , faster , faster than the flame\\n', 'Send us all to war tonight\\n', 'Be prepared for sacrifice\\n', 'This is our time , this is our raid\\n', 'Embers left to those alive\\n', 'Madness raised , we all ignite\\n', 'This is our night , let fire reign\\n', 'Incendere  ( cendere ) \\n', 'Inflammatum  ( flammatum ) \\n', 'Illuminatum\\n', 'And we all rise against the damned\\n', 'Stand up faster for heaven , faster than the flame\\n', 'Hold the pastor together , pastor by the chain\\n', \"And at night we 're going wild\\n\", 'When we set the world on fire\\n', 'Rolling faster , faster , faster than the flame\\n', 'Flame , flame , burning wild in heavens name\\n', 'Flame , flame , from the Lord we came\\n', 'Flame , flame , Armageddon we proclaim\\n', 'Flame , flame , fire now shall reign\\n', 'Stand up faster for heaven , faster than the flame\\n', 'Hold the pastor together , pastor by the chain\\n', 'When we fire lights the sky and no sin we can deny\\n', 'We all burn faster , faster , faster than the flame']\n"
     ]
    }
   ],
   "source": [
    "print(dataset.phrases[0:100])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "126\n"
     ]
    }
   ],
   "source": [
    "print(len(dataset.vocabulary))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "53\n"
     ]
    }
   ],
   "source": [
    "print(dataset.max_length)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "class HeadAttention(nn.Module):\n",
    "\n",
    "    def __init__(self, d_model, d_queries, d_values, dropout, in_decoder=False):\n",
    "\n",
    "        super(HeadAttention, self).__init__()\n",
    "\n",
    "        self.d_model = d_model\n",
    "        self.d_queries = d_queries\n",
    "        self.d_values = d_values\n",
    "        self.d_keys = d_values # size of key vectors, same as of the query vectors to allow dot-products for similarity\n",
    "\n",
    "        self.in_decoder = in_decoder\n",
    "\n",
    "        self.create_queries = nn.Linear(d_model, d_queries, bias=True)\n",
    "        self.create_values = nn.Linear(d_model, d_values, bias=True)\n",
    "        self.create_keys = nn.Linear(d_model, d_values, bias=True)\n",
    "\n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "\n",
    "        self.relu = nn.ReLU()\n",
    "        self.softmax = nn.Softmax(dim=-1)\n",
    "\n",
    "    def forward(self, input, real_lengths):\n",
    "\n",
    "        batch_size = input.size(0) # (Batch, Sequences, d_model)\n",
    "\n",
    "        queries = self.relu(self.create_queries(input)) # (Batch, Sequences, d_queries)\n",
    "        keys = self.relu(self.create_keys(input)) # (Batch, Sequences, d_keys)\n",
    "        values = self.relu(self.create_values(input)) # (Batch, Sequences, d_values)\n",
    "\n",
    "        similarity_matrix = []\n",
    "\n",
    "        for batch in range(batch_size):\n",
    "\n",
    "            dot_product = torch.matmul(queries[batch], keys[batch].T)\n",
    "            similarity_matrix.append(dot_product.unsqueeze(0))\n",
    "\n",
    "        del dot_product\n",
    "\n",
    "        similarity_matrix = torch.cat(similarity_matrix, 0) # (Batch, Sequences, Sequences)\n",
    "\n",
    "        similarity_matrix = similarity_matrix/(math.sqrt(self.d_keys))\n",
    "\n",
    "        # Applying mask of -inf to ignore padded keys ---> Actually using -1e6 to avoid NaNs\n",
    "\n",
    "        if self.in_decoder:\n",
    "\n",
    "            # In the decoder, masks are shifted left to right:\n",
    "            # <Start-Of-Sentence> <PAD> <PAD> <PAD> ----> <Start-Of-Sentence> <prediction1> <prediction2> ... <End-Of-Sentence>\n",
    "            # <Start-of-Sentence> [prediction1] [prediction2] [prediction3] ... <End-of-Sentence>\n",
    "\n",
    "            mask = torch.zeros_like(similarity_matrix, device=device)\n",
    "\n",
    "            for batch in range(similarity_matrix.size(0)):\n",
    "\n",
    "                #mask[batch, :real_lengths[batch]] = 1\n",
    "                mask[batch, real_lengths+1:] = 1 # For Batch = 1\n",
    "\n",
    "                mask = mask.bool()\n",
    "\n",
    "            similarity_matrix = similarity_matrix.masked_fill(mask, -1e6)\n",
    "\n",
    "        else:\n",
    "\n",
    "            mask = torch.zeros_like(similarity_matrix, device=device)\n",
    "\n",
    "            for batch in range(similarity_matrix.size(0)):\n",
    "\n",
    "                #mask[batch, real_lengths[batch]:] = 1\n",
    "                mask[batch, real_lengths:] = 1 # For Batch = 1\n",
    "\n",
    "                mask = mask.bool()\n",
    "\n",
    "            similarity_matrix = similarity_matrix.masked_fill(mask, -1e6) # (Batch, Sequence, Sequence)\n",
    "\n",
    "        del mask\n",
    "\n",
    "        attention_weights = self.softmax(similarity_matrix) # (Batch, Sequences, Sequences)\n",
    "\n",
    "        attention_weights = self.dropout(attention_weights)\n",
    "\n",
    "        attention_output = torch.bmm(attention_weights, values) # (Batch, Sequences, d_values)\n",
    "\n",
    "        return attention_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "class PositionWiseFeedForward(nn.Module):\n",
    "\n",
    "    def __init__(self, d_model, d_inner, dropout):\n",
    "\n",
    "        super(PositionWiseFeedForward, self).__init__()\n",
    "\n",
    "        self.d_model = d_model\n",
    "        self.d_inner = d_inner\n",
    "\n",
    "        self.neuron1 = nn.Linear(d_model, d_inner)\n",
    "        self.Relu = nn.ReLU()\n",
    "        self.neuron2 = nn.Linear(d_inner, d_model)\n",
    "\n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "\n",
    "    def forward(self, attention_output_cat):\n",
    "\n",
    "        sequences = self.neuron1(attention_output_cat)\n",
    "        sequences = self.Relu(sequences)\n",
    "\n",
    "        sequences = self.neuron2(self.dropout(sequences))\n",
    "\n",
    "        output = sequences + attention_output_cat\n",
    "\n",
    "        return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Encoder(nn.Module):\n",
    "\n",
    "    def __init__(self, d_model, n_heads, d_queries, d_values, d_inner, dropout, masks=False, text_length=None):\n",
    "\n",
    "        super(Encoder, self).__init__()\n",
    "\n",
    "        self.n_heads = n_heads\n",
    "\n",
    "        self.attention_heads = nn.ModuleList([HeadAttention(d_model, d_queries, d_values, dropout, masks) for i in range(n_heads)])\n",
    "\n",
    "        self.neuron = nn.Linear(self.n_heads*d_values, d_model)\n",
    "\n",
    "        self.position_wise_neuron = PositionWiseFeedForward(d_model, d_inner, dropout)\n",
    "\n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "\n",
    "        # Adding scaling parameter for residual blocks: https://aclanthology.org/2020.emnlp-main.463.pdf\n",
    "\n",
    "        self.layer_normA = nn.LayerNorm(d_model)\n",
    "        self.residual_scalingA = nn.Parameter(torch.ones((1, text_length, d_model)))\n",
    "        self.layer_normB = nn.LayerNorm(d_model)\n",
    "        self.residual_scalingB = nn.Parameter(torch.ones((1, text_length, d_model)))\n",
    "\n",
    "    def forward(self, encoder_input, real_input_length):\n",
    "\n",
    "        residual_block1 = encoder_input # (Batch, Sequence, d_model) ---> Vectors\n",
    "\n",
    "        vectors = self.dropout(encoder_input)\n",
    "\n",
    "        attention_output = []\n",
    "\n",
    "        for head in range(self.n_heads):\n",
    "\n",
    "            x = self.attention_heads[head](vectors, real_input_length)\n",
    "\n",
    "            attention_output.append(x)\n",
    "\n",
    "            del x\n",
    "        \n",
    "        attention_output = torch.cat(attention_output, -1) # (Batch, Sequences, d_values*n_heads)\n",
    "\n",
    "        attention_output = self.neuron(attention_output) # (Batch, Sequences, d_model)\n",
    "\n",
    "        del vectors\n",
    "\n",
    "        residual_block2 = attention_output\n",
    "\n",
    "        #attention_output = residual_block1 + attention_output\n",
    "        attention_output = residual_block1 + (attention_output * self.residual_scalingA)\n",
    "\n",
    "        attention_output = self.dropout(attention_output)\n",
    "\n",
    "        attention_output = self.layer_normA(attention_output)\n",
    "\n",
    "        encoded_sequence = self.position_wise_neuron(attention_output)\n",
    "\n",
    "        del attention_output\n",
    "\n",
    "        #encoded_sequence = encoded_sequence + residual_block2\n",
    "        encoded_sequence = (encoded_sequence * self.residual_scalingB) + residual_block2\n",
    "\n",
    "        encoder_output = self.dropout(encoded_sequence)\n",
    "\n",
    "        encoder_output = self.layer_normB(encoder_output)\n",
    "\n",
    "        #print(residual_block1.var())\n",
    "        #print(residual_block2.var())\n",
    "\n",
    "        del encoded_sequence, residual_block1, residual_block2\n",
    "\n",
    "        return encoder_output # (Batch, Sequences, d_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Decoder(nn.Module):\n",
    "\n",
    "    def __init__(self, d_model, n_heads, d_queries, d_values, d_inner, dropout, text_length=None):\n",
    "\n",
    "        super(Decoder, self).__init__()\n",
    "\n",
    "        self.n_heads = n_heads\n",
    "\n",
    "        self.attention_headsA = nn.ModuleList([HeadAttention(d_model, d_queries, d_values, dropout, in_decoder=True) for i in range(n_heads)])\n",
    "        self.attention_headsB = nn.ModuleList([HeadAttention(d_model, d_queries, d_values, dropout, in_decoder=False) for i in range(n_heads)])\n",
    "\n",
    "        self.neuronA = nn.Linear(self.n_heads*d_values, d_model)\n",
    "        self.neuronB = nn.Linear(self.n_heads*d_values, d_model)\n",
    "\n",
    "        self.position_wise_neuron = PositionWiseFeedForward(d_model, d_inner, dropout)\n",
    "\n",
    "        self.layer_normA = nn.LayerNorm(d_model)\n",
    "        self.residual_scalingA = nn.Parameter(torch.ones((1, text_length, d_model)))\n",
    "        self.layer_normB = nn.LayerNorm(d_model)\n",
    "        self.residual_scalingB = nn.Parameter(torch.ones((1, text_length, d_model)))\n",
    "        self.layer_normC = nn.LayerNorm(d_model)\n",
    "        self.residual_scalingC = nn.Parameter(torch.ones((1, text_length, d_model)))\n",
    "\n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "\n",
    "    def forward(self, encoder_output, target_sequences, real_target_length):\n",
    "\n",
    "        residual_block1 = target_sequences\n",
    "\n",
    "        vectors = self.dropout(target_sequences)\n",
    "\n",
    "        attention_output = []\n",
    "\n",
    "        for head in range(self.n_heads):\n",
    "\n",
    "            x = self.attention_headsA[head](vectors, real_target_length)\n",
    "\n",
    "            attention_output.append(x)\n",
    "\n",
    "            del x\n",
    "        \n",
    "        attention_output = torch.cat(attention_output, -1) # (Batch, Sequences, d_values*n_heads)\n",
    "\n",
    "        attention_output = self.neuronA(attention_output) # (Batch, Sequences, d_model)\n",
    "\n",
    "        del vectors\n",
    "\n",
    "        residual_block2 = attention_output\n",
    "\n",
    "        #attention_output = residual_block1 + attention_output\n",
    "        attention_output = residual_block1 + (attention_output * self.residual_scalingA)\n",
    "\n",
    "        attention_decoder = self.dropout(attention_output)\n",
    "\n",
    "        attention_decoder = self.layer_normA(attention_decoder)\n",
    "        \n",
    "        attention_output = []\n",
    "\n",
    "        for head in range(self.n_heads):\n",
    "\n",
    "            x = self.attention_headsB[head](encoder_output, real_target_length)\n",
    "\n",
    "            attention_output.append(x)\n",
    "\n",
    "            del x\n",
    "\n",
    "        attention_output = torch.cat(attention_output, -1) # (Batch, Sequences, d_values*n_heads)\n",
    "\n",
    "        attention_output = self.neuronB(attention_output) # (Batch, Sequences, d_model)\n",
    "\n",
    "        #attention_output = attention_output + residual_block2\n",
    "        attention_output = residual_block2 + (attention_output * self.residual_scalingB)\n",
    "\n",
    "        attention_encoder = self.dropout(attention_output)\n",
    "\n",
    "        attention_encoder = self.layer_normB(attention_encoder)\n",
    "\n",
    "        decoded_sequence = attention_encoder + attention_decoder\n",
    "\n",
    "        residual_block3 = decoded_sequence\n",
    "\n",
    "        decoded_sequence = self.position_wise_neuron(attention_output)\n",
    "\n",
    "        del attention_output, attention_encoder, attention_decoder\n",
    "\n",
    "        #decoded_sequence = decoded_sequence + residual_block3\n",
    "        decoded_sequence = (decoded_sequence * self.residual_scalingC) + residual_block3\n",
    "\n",
    "        decoder_output = self.dropout(decoded_sequence)\n",
    "\n",
    "        decoder_output = self.layer_normC(decoder_output)\n",
    "\n",
    "        #print(residual_block1.var())\n",
    "        #print(residual_block2.var())\n",
    "        #print(residual_block3.var())\n",
    "\n",
    "        del decoded_sequence, residual_block1, residual_block2, residual_block3\n",
    "\n",
    "        return decoder_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_positional_encoding(d_model, max_length=100):\n",
    "    \"\"\"\n",
    "    Computes positional encoding as defined in the paper.\n",
    "    :param d_model: size of vectors throughout the transformer model\n",
    "    :param max_length: maximum sequence length up to which positional encodings must be calculated\n",
    "    :return: positional encoding, a tensor of size (1, max_length, d_model)\n",
    "    \"\"\"\n",
    "    positional_encoding = torch.zeros((max_length, d_model))  # (max_length, d_model)\n",
    "    for i in range(max_length):\n",
    "        for j in range(d_model):\n",
    "            if j % 2 == 0:\n",
    "                positional_encoding[i, j] = math.sin(i / math.pow(10000, j / d_model))\n",
    "            else:\n",
    "                positional_encoding[i, j] = math.cos(i / math.pow(10000, (j - 1) / d_model))\n",
    "\n",
    "    positional_encoding = positional_encoding.unsqueeze(0)  # (1, max_length, d_model)\n",
    "\n",
    "    return positional_encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Broca(nn.Module):\n",
    "\n",
    "    \"\"\"\n",
    "    The generator, which will generate the words that she'll speak.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, text_length, vocab_size, positional_encoding, d_model=512, n_heads=8, d_queries=64, d_values=64, d_inner=2056, n_layers=6, dropout=0.1):\n",
    "\n",
    "        super(Broca, self).__init__()\n",
    "\n",
    "        self.vocab_size = vocab_size\n",
    "        self.positional_encoding = positional_encoding.to(device)\n",
    "        self.d_model = d_model\n",
    "        self.n_heads = n_heads\n",
    "        self.n_layers = n_layers\n",
    "\n",
    "        self.positional_encoding.requires_grad = False\n",
    "\n",
    "        self.embedding = nn.Embedding(vocab_size, self.d_model)\n",
    "\n",
    "        #self.image_encoder = ImageEncoder(d_model)\n",
    "\n",
    "        self.encoder = nn.ModuleList(\n",
    "            Encoder(\n",
    "                    d_model=d_model,\n",
    "                    n_heads=n_heads,\n",
    "                    d_queries=d_queries,\n",
    "                    d_values=d_values,\n",
    "                    d_inner=d_inner,\n",
    "                    dropout=dropout,\n",
    "                    text_length=text_length) for i in range(self.n_layers)\n",
    "        )\n",
    "\n",
    "        self.decoder = nn.ModuleList(\n",
    "            Decoder(\n",
    "                    d_model=d_model,\n",
    "                    n_heads=n_heads,\n",
    "                    d_queries=d_queries,\n",
    "                    d_values=d_values,\n",
    "                    d_inner=d_inner,\n",
    "                    dropout=dropout,\n",
    "                    text_length=text_length) for i in range(self.n_layers)\n",
    "        )\n",
    "\n",
    "        self.output_neuron = nn.Linear(self.d_model, vocab_size)\n",
    "\n",
    "        #self.softmax = nn.LogSoftmax(-1)\n",
    "        #self.softmax = nn.Softmax(-1) # Not really necessary.\n",
    "\n",
    "    def preprocess_dialogue(self, input_text, target_text=False):\n",
    "\n",
    "        if re.findall(',', input_text):\n",
    "\n",
    "            input_text = input_text.replace(',', ' ,')\n",
    "\n",
    "        if re.findall(';', input_text):\n",
    "\n",
    "            input_text = input_text.replace(';', ' ;')\n",
    "        \n",
    "        if re.findall(\"'\", input_text):\n",
    "\n",
    "            input_text = input_text.replace(\"'\", \" ' \")\n",
    "\n",
    "        if re.findall('.', input_text):\n",
    "\n",
    "            input_text = input_text.replace('.', ' .')\n",
    "\n",
    "        if re.findall(':', input_text):\n",
    "\n",
    "            input_text = input_text.replace(':', ' :')\n",
    "\n",
    "        if re.findall('!', input_text):\n",
    "\n",
    "            input_text = input_text.replace('!', ' !')\n",
    "\n",
    "        if re.findall('\\?', input_text):\n",
    "\n",
    "            input_text = input_text.replace('?', ' ?')\n",
    "\n",
    "        if re.findall('\\(', input_text):\n",
    "\n",
    "            input_text = input_text.replace('(', ' ( ')\n",
    "\n",
    "        if re.findall('\\)', input_text):\n",
    "\n",
    "            input_text = input_text.replace(')', ' ) ')\n",
    "\n",
    "        if re.findall('\"', input_text):\n",
    "\n",
    "            input_text = input_text.replace('\"', ' \" ')\n",
    "\n",
    "        if re.findall('-', input_text):\n",
    "\n",
    "            input_text = input_text.replace('-', ' - ')\n",
    "\n",
    "        input_text = input_text.split(' ')\n",
    "        #print(input_text)\n",
    "\n",
    "        for i in range(input_text.count('')):\n",
    "\n",
    "            input_text.remove('')\n",
    "\n",
    "        tokens = []\n",
    "\n",
    "        if target_text:\n",
    "\n",
    "            tokens.append(dataset.vocabulary.index(\"<SOS>\"))\n",
    "        \n",
    "        for word in input_text:\n",
    "\n",
    "            word = re.sub(\"[\\s]\", '', word) # \\n\n",
    "\n",
    "            value = dataset.vocabulary.index(word)\n",
    "            tokens.append(value)\n",
    "            tokens.append(dataset.vocabulary.index(' '))\n",
    "\n",
    "        tokens = np.array(tokens)\n",
    "        sentence_size = tokens.shape[0]\n",
    "\n",
    "        if sentence_size < dataset.max_length:\n",
    "\n",
    "            pad_size = dataset.max_length - sentence_size\n",
    "            tokens = np.pad(tokens, [(0, 1)], constant_values=dataset.vocabulary.index(\"<EOS>\"))\n",
    "            tokens = np.pad(tokens, [(0, pad_size-1)], constant_values=dataset.vocabulary.index(\"<pad>\"))\n",
    "\n",
    "        tokens = torch.from_numpy(tokens)\n",
    "        tokens = tokens.unsqueeze(0).to(device)\n",
    "\n",
    "        return tokens, sentence_size\n",
    "\n",
    "\n",
    "    def forward(self, input, target_sentence): # Target Sentence (with <SOS> token) provided before forward function\n",
    "\n",
    "        if type(input) == str: # Dialogue\n",
    "\n",
    "            input, real_input_length = self.preprocess_dialogue(input)\n",
    "            target_sentence, target_input_length = self.preprocess_dialogue(target_sentence, target_text=True)\n",
    "\n",
    "            encoder_vectors = self.embedding(input) * math.sqrt(self.d_model) + self.positional_encoding.to(device)\n",
    "\n",
    "            decoder_vectors = self.embedding(target_sentence) * math.sqrt(self.d_model) + self.positional_encoding.to(device) # Embed out = (Batch, d_model)\n",
    "\n",
    "        else: # Image reaction ---> Input is already the image encoded into vectors ---> For Alice.\n",
    "\n",
    "            encoder_vectors = input.view(input.size(0), 1, -1)\n",
    "\n",
    "            encoder_vectors = encoder_vectors * math.sqrt(self.d_model)\n",
    "            \n",
    "            decoder_vectors = encoder_vectors\n",
    "\n",
    "        for layer in range(self.n_layers):\n",
    "\n",
    "            encoder_vectors = self.encoder[layer](encoder_vectors, real_input_length)\n",
    "\n",
    "            decoder_vectors = self.decoder[layer](encoder_vectors, decoder_vectors, target_input_length)\n",
    "\n",
    "        output = self.output_neuron(decoder_vectors) # (Batch, sequence, vocab_size)\n",
    "\n",
    "        #output = self.softmax(output) # (Batch, sequence, vocab_size)\n",
    "\n",
    "        return output # Probability of words. Take the one with highest probability, add it to the target sentence and repeat."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "126 53\n"
     ]
    }
   ],
   "source": [
    "print(len(dataset.vocabulary), dataset.max_length)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1, 53, 64])\n"
     ]
    }
   ],
   "source": [
    "positional_encoding = get_positional_encoding(d_model=64, max_length=53)\n",
    "\n",
    "print(positional_encoding.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Broca(\n",
    "    text_length=53,\n",
    "    vocab_size=len(dataset.vocabulary),\n",
    "    positional_encoding=positional_encoding,\n",
    "    d_model=64,\n",
    "    n_heads=8,\n",
    "    d_queries=16,\n",
    "    d_values=16,\n",
    "    d_inner=128,\n",
    "    n_layers=2,\n",
    "    dropout=0.1\n",
    ").to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialization --> Not necessary if your model is small and simple\n",
    "\n",
    "for p in model.parameters():\n",
    "    # Glorot initialization needs at least two dimensions on the tensor\n",
    "    if p.dim() > 1:\n",
    "        nn.init.xavier_uniform_(p, gain=1.)\n",
    "\n",
    "nn.init.normal_(model.embedding.weight, 0, math.pow(512, -0.5))\n",
    "\n",
    "with torch.no_grad():\n",
    "    model.output_neuron.weight = model.embedding.weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_lr(step, d_model, warmup_steps):\n",
    "    \"\"\"\n",
    "    The LR schedule. This version below is twice the definition in the paper, as used in the official T2T repository.\n",
    "    :param step: training step number\n",
    "    :param d_model: size of vectors throughout the transformer model\n",
    "    :param warmup_steps: number of warmup steps where learning rate is increased linearly; twice the value in the paper, as in the official T2T repo\n",
    "    :return: updated learning rate\n",
    "    \"\"\"\n",
    "    lr = 2. * (d_model ** -0.5) * min((step ** -0.5), step * (warmup_steps** -1.5))\n",
    "\n",
    "    return lr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Alternative LR schedule for warmup, based on gaussian distribution\n",
    "# The LR begins near zero, ascends towards a peak(LR = 5~10) and then decreases towards zero.\n",
    "\n",
    "def get_lr_gaussian(step, mean_step=5000, std=5000):\n",
    "    '''\n",
    "    Alternative LR schedule for warmup phase, based on gaussian distribution\n",
    "    The LR begins near zero, ascends towards a peak(LR = 5~10) and then decreases towards zero.\n",
    "\n",
    "    :param step: the current step. The X coordinate in the graphic of a bell curve.\n",
    "    :param mean_step: the step at which the learning rate will reach its peak.\n",
    "    :param std: the standard deviation of the gaussian distribution. Here, it's just a hyperparameter.\n",
    "    '''\n",
    "\n",
    "    x = step\n",
    "    mean = mean_step\n",
    "\n",
    "    A = ((x - mean)**2) / (2*(std**2))\n",
    "\n",
    "    num = math.e ** - A\n",
    "    den = std * math.sqrt(math.pi*2)\n",
    "\n",
    "    y = (num/den) * (x/mean) * 1e5 # x/mean and 1e5 are scaling factors.\n",
    "\n",
    "    return y"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Warmup Phase isn't really necessary if your model is small and simple."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.5e-07\n",
      "0.0025\n",
      "0.0017677669529663688\n",
      "0.0015811388300841897\n"
     ]
    }
   ],
   "source": [
    "print(get_lr(1, 64, 10000))\n",
    "print(get_lr(10000, 64, 10000))\n",
    "print(get_lr(20000, 64, 10000))\n",
    "print(get_lr(25000, 64, 10000))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0009680764746536077\n",
      "1.1587662110459311\n",
      "7.978845608028654\n",
      "9.678828980765735\n",
      "0.35454787295504064\n",
      "0.00017840634176811586\n"
     ]
    }
   ],
   "source": [
    "print(get_lr_gaussian(1, 5000, 5000))\n",
    "print(get_lr_gaussian(1000, 5000, 5000))\n",
    "print(get_lr_gaussian(5000, 5000, 5000))\n",
    "print(get_lr_gaussian(10000, 5000, 5000))\n",
    "print(get_lr_gaussian(20000, 5000, 5000))\n",
    "print(get_lr_gaussian(30000, 5000, 5000))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [],
   "source": [
    "#optimizer = torch.optim.Adam(model.parameters(), lr=get_lr(1, 512, 2000), betas=(0.9, 0.98), eps=1e-9)\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=1e-3, betas=(0.9, 0.98), eps=1e-9)\n",
    "criterion = nn.CrossEntropyLoss(ignore_index=dataset.vocabulary.index(\"<pad>\"), label_smoothing=0.1)\n",
    "EPOCHS = 10000 # Using steps rather than epochs is more convenient here.\n",
    "schedule_sampling = 0.9999 # Decaying factor. 0.9 decays faster than 0.999"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [],
   "source": [
    "def schedule_sampler(sampling_rate, output, target_encoded):\n",
    "\n",
    "    '''\n",
    "    Used to avoid exposure bias, which is caused by the use of target sentence during training,\n",
    "    compromising the evaluation performance.\n",
    "    https://arxiv.org/pdf/1906.07651.pdf\n",
    "\n",
    "    For Transformer, perform 2 iterations per step, the first on target sentence, the second\n",
    "    on mixed sentences(target + generated on first iteration).\n",
    "    The sampling rate should begin low(only target sentences) and increase over time.\n",
    "\n",
    "    Less efficient than Reinforcement Learning(ain't that right, ChatGPT?)\n",
    "    '''\n",
    "\n",
    "    mixed_target = target_encoded.clone()\n",
    "\n",
    "    for i in range(len(target_encoded)):\n",
    "\n",
    "        if torch.rand((1,)) > sampling_rate:\n",
    "\n",
    "            one_hot = torch.zeros_like(target_encoded)\n",
    "            one_hot[i] = output[i].argmax(-1)\n",
    "\n",
    "            mixed_target[i] = one_hot[i]\n",
    "\n",
    "    mixed_target = [dataset.vocabulary[i] for i in target_encoded]\n",
    "    mixed_target = \" \".join(mixed_target)\n",
    "    mixed_target = mixed_target.replace(\"<SOS>\", \"\").replace(\"<EOS>\", \"\").replace(\"<pad>\", \"\")\n",
    "    # Removing double spaces\n",
    "    mixed_target = \"\".join(mixed_target)\n",
    "    mixed_target = \" \".join(mixed_target.split())\n",
    "\n",
    "    return mixed_target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [],
   "source": [
    "teste = dataset.phrases[1]\n",
    "target = dataset.phrases[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "out = model(teste, target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "target, target_length = model.preprocess_dialogue(target, target_text=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[125,  35,   1,  36,   1,  37,   1,  33,   1,  38,   1,  39,   1, 124,\n",
      "           0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
      "           0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
      "           0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0]],\n",
      "       device='cuda:0', dtype=torch.int32)\n"
     ]
    }
   ],
   "source": [
    "print(target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_sentence(input):\n",
    "\n",
    "    target_indices = [dataset.vocabulary.index(\"<SOS>\")]\n",
    "\n",
    "    target_sentence = \"\"\n",
    "\n",
    "    with torch.no_grad():\n",
    "\n",
    "        for i in range(100):\n",
    "\n",
    "            try:\n",
    "\n",
    "                output = model(input, target_sentence)\n",
    "\n",
    "            except RuntimeError: # Triggered when encoded target sentence > dataset max length\n",
    "\n",
    "                break\n",
    "            \n",
    "            output = output[0, :len(target_indices)]\n",
    "\n",
    "            next_token = output[-1].argmax(-1).item()\n",
    "\n",
    "            if next_token == dataset.vocabulary.index(\"<EOS>\"):\n",
    "                break\n",
    "\n",
    "            target_indices.append(next_token)\n",
    "\n",
    "            target_sentence = [dataset.vocabulary[target_indices[i]] for i in range(len(target_indices))]\n",
    "            target_sentence = \" \".join(target_sentence)\n",
    "            target_sentence = target_sentence.replace(\"<SOS>\", \"\").replace(\"<EOS>\", \"\").replace(\"<pad>\", \"\")\n",
    "            target_sentence = \"\".join(target_sentence)\n",
    "            target_sentence = \" \".join(target_sentence.split())\n",
    "\n",
    "            #target_sentence += dataset.vocabulary[next_token]\n",
    "\n",
    "    #target_sentence = target_sentence.replace(\"<SOS>\", \"\").replace(\"<EOS>\", \"\").replace(\"<pad>\", \"\")\n",
    "\n",
    "    return target_sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Faster , faster , faster than the flame\n",
      "\n",
      "Illuminatum Illuminatum prepared\n"
     ]
    }
   ],
   "source": [
    "generated_text = generate_sentence(teste)\n",
    "\n",
    "print(teste)\n",
    "print(generated_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "40\n"
     ]
    }
   ],
   "source": [
    "print(len(dataset.phrases))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "step = 0\n",
    "\n",
    "for epoch in range(1, EPOCHS+1):\n",
    "\n",
    "    epoch_loss = 0\n",
    "\n",
    "    batches = torch.randperm(len(dataset.phrases)-1)\n",
    "\n",
    "    for i, idx in enumerate(batches, start=1):\n",
    "\n",
    "        inputext = dataset.phrases[idx]\n",
    "        targetext = dataset.phrases[idx+1]\n",
    "\n",
    "        input, input_length = model.preprocess_dialogue(inputext)\n",
    "        target, target_length = model.preprocess_dialogue(targetext, target_text=True)\n",
    "\n",
    "        model.zero_grad()\n",
    "\n",
    "        possibilities = model(inputext, targetext) # (Batch, sequences, vocab_size)\n",
    "        possibilities_nopad = possibilities[0, :target_length] # (Sequences, vocab_size)\n",
    "        target = target.squeeze(0)\n",
    "\n",
    "        first_loss = 0.\n",
    "\n",
    "        for item in range(len(possibilities_nopad)):\n",
    "\n",
    "            first_loss += criterion(possibilities_nopad[item], target[item].long())\n",
    "\n",
    "        epoch_loss += first_loss.item()\n",
    "\n",
    "        first_loss.backward()\n",
    "\n",
    "        optimizer.step()\n",
    "\n",
    "        model.zero_grad()\n",
    "\n",
    "        sampling_rate = 1 - (schedule_sampling ** step)\n",
    "\n",
    "        new_targetext = schedule_sampler(sampling_rate, possibilities.squeeze(0), target)\n",
    "        new_target, _ = model.preprocess_dialogue(new_targetext, target_text=True)\n",
    "        new_target = new_target.squeeze(0)\n",
    "\n",
    "\n",
    "        possibilities = model(inputext, new_targetext)\n",
    "        possibilities_nopad = possibilities[0, :target_length]\n",
    "\n",
    "        second_loss = 0.\n",
    "\n",
    "        for item in range(len(possibilities_nopad)):\n",
    "\n",
    "            second_loss += criterion(possibilities_nopad[item], target[item].long())\n",
    "\n",
    "        epoch_loss += second_loss.item()\n",
    "\n",
    "        second_loss.backward()\n",
    "\n",
    "        optimizer.step()\n",
    "\n",
    "        step += 1\n",
    "\n",
    "        grads = torch.mean(model.encoder[0].attention_heads[0].create_queries.weight.grad)\n",
    "        grads2 = torch.mean(model.output_neuron.weight.grad)\n",
    "\n",
    "        if step % 100 == 0:\n",
    "\n",
    "            print(f\"{epoch}/{EPOCHS}\\tCurrent Step: {step}\")\n",
    "            print(f\"Current Sampling Rate: {sampling_rate}\")\n",
    "            print(f\"Last First Iter Loss: {first_loss.item()}\\tSecond Iter Loss: {second_loss.item()}\")\n",
    "            print(f\"Total Epoch Loss: {epoch_loss/(len(batches))}\")\n",
    "            print(f\"First Layer Gradients Average: {grads}\")\n",
    "            print(f\"Final Layer Gradients Average: {grads2}\")\n",
    "\n",
    "            decoded_out = [dataset.vocabulary[possibilities_nopad[i].argmax(-1)] for i in range(len(possibilities_nopad))]\n",
    "            decoded_out = \" \".join(decoded_out)\n",
    "            decoded_out = decoded_out.replace(\"<SOS>\", \"\").replace(\"<EOS>\", \"\").replace(\"<pad>\", \"\")\n",
    "            decoded_out = \"\".join(decoded_out)\n",
    "            decoded_out = \" \".join(decoded_out.split())\n",
    "\n",
    "            print(f\"Input Text: {inputext}\")\n",
    "            print(f\"Output: {decoded_out}\")\n",
    "            print(f\"Target Text: {targetext}\")\n",
    "\n",
    "            generated_text = generate_sentence(inputext)\n",
    "\n",
    "            print(f\"Evaluation Output: {generated_text}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1/10000\tCurrent Step: 0\n",
      "Current Sampling Rate: 0.8325699054762894\n",
      "Last First Iter Loss: 9.208990097045898\tSecond Iter Loss: 9.04998779296875\n",
      "Total Epoch Loss: 0.0\n",
      "First Layer Gradients Average: -2.696774856758566e-07\n",
      "Final Layer Gradients Average: -4.1392117511307447e-10\n",
      "Input Text: Hold the pastor together , pastor by the chain\n",
      "\n",
      "Output: Restless is the world alight\n",
      "Target Text: And at night we 're going wild\n",
      "\n",
      "Evaluation Output: Illuminatum at Hold lights ' re going no sin at can deny no at going at we at deny at at no no no we we we\n"
     ]
    }
   ],
   "source": [
    "print(f\"{epoch}/{EPOCHS}\\tCurrent Step: {step}\")\n",
    "print(f\"Current Sampling Rate: {sampling_rate}\")\n",
    "print(f\"Last First Iter Loss: {first_loss.item()}\\tSecond Iter Loss: {second_loss.item()}\")\n",
    "print(f\"Total Epoch Loss: {epoch_loss/(len(batches))}\")\n",
    "print(f\"First Layer Gradients Average: {grads}\")\n",
    "print(f\"Final Layer Gradients Average: {grads2}\")\n",
    "\n",
    "decoded_out = [dataset.vocabulary[possibilities_nopad[i].argmax(-1)] for i in range(len(possibilities_nopad))]\n",
    "decoded_out = \" \".join(decoded_out)\n",
    "decoded_out = decoded_out.replace(\"<SOS>\", \"\").replace(\"<EOS>\", \"\").replace(\"<pad>\", \"\")\n",
    "decoded_out = \"\".join(decoded_out)\n",
    "decoded_out = \" \".join(decoded_out.split())\n",
    "\n",
    "print(f\"Input Text: {inputext}\")\n",
    "print(f\"Output: {decoded_out}\")\n",
    "print(f\"Target Text: {targetext}\")\n",
    "\n",
    "generated_text = generate_sentence(inputext)\n",
    "\n",
    "print(f\"Evaluation Output: {generated_text}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.4"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "2469a70536e4d2335a2ea8907942d0699c37342a371ac185bdb5b0aa6f073890"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}