HW_4.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "HW_4.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/sunnyshen321/DS100/blob/master/HW_4.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uf6Cy9z28XVd"
      },
      "source": [
        "# SETUP"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4dpfjfA18Wsn",
        "outputId": "c7dc5972-473a-4e04-ab5c-49e20946e2e3"
      },
      "source": [
        "!pip install transformers"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting transformers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/98/87/ef312eef26f5cecd8b17ae9654cdd8d1fae1eb6dbd87257d6d73c128a4d0/transformers-4.3.2-py3-none-any.whl (1.8MB)\n",
            "\u001b[K     |████████████████████████████████| 1.8MB 11.0MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers) (3.0.12)\n",
            "Collecting tokenizers<0.11,>=0.10.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/fd/5b/44baae602e0a30bcc53fbdbc60bd940c15e143d252d658dfdefce736ece5/tokenizers-0.10.1-cp36-cp36m-manylinux2010_x86_64.whl (3.2MB)\n",
            "\u001b[K     |████████████████████████████████| 3.2MB 36.5MB/s \n",
            "\u001b[?25hRequirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from transformers) (20.9)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (2019.12.20)\n",
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n",
            "\u001b[K     |████████████████████████████████| 890kB 40.4MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers) (4.41.1)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (1.19.5)\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.6/dist-packages (from transformers) (3.4.0)\n",
            "Requirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers) (0.8)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2020.12.5)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (1.0.0)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.6/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.7.4.3)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.6/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.4.0)\n",
            "Building wheels for collected packages: sacremoses\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.43-cp36-none-any.whl size=893261 sha256=222bb961f253180b15eb64db4ada168cd643a7b0754b4eaf4235af7d1283e1ba\n",
            "  Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n",
            "Successfully built sacremoses\n",
            "Installing collected packages: tokenizers, sacremoses, transformers\n",
            "Successfully installed sacremoses-0.0.43 tokenizers-0.10.1 transformers-4.3.2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5G2ykxvmY_AD"
      },
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import transformers\n",
        "import sys, argparse\n",
        "import numpy as np\n",
        "import re\n",
        "import random\n",
        "from tqdm import tqdm\n",
        "from collections import Counter\n",
        "from transformers import BertTokenizer, BertModel\n",
        "from collections import Counter\n",
        "\n",
        "#Sets random seeds for reproducibility\n",
        "seed=0\n",
        "np.random.seed(seed)\n",
        "torch.manual_seed(seed)\n",
        "torch.cuda.manual_seed(seed)\n",
        "torch.backends.cudnn.deterministic = True"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jbWTDJwiCU_j",
        "outputId": "fcda82a5-7d89-4bb0-fd60-56b4034b161e"
      },
      "source": [
        "print(torch.__version__)\n",
        "print(transformers.__version__)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "1.7.0+cu101\n",
            "4.3.2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2G1DETVICaQG"
      },
      "source": [
        "# **IMPORTANT**: GPU is not enabled by default\n",
        "\n",
        "You must switch runtime environments if your output of the next block of code has an error saying \"ValueError: Expected a cuda device, but got: cpu\"\n",
        "\n",
        "Go to Runtime > Change runtime type > Hardware accelerator > GPU"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sGE0IvlpCesG",
        "outputId": "427b1fd0-7796-490a-de4e-fa6e69aac110"
      },
      "source": [
        "# if this cell prints \"Running on cpu\", you must switch runtime environments\n",
        "# go to Runtime > Change runtime type > Hardware accelerator > GPU\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "print(\"Running on {}\".format(device))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Running on cuda\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YLL42K0dGG4K"
      },
      "source": [
        "# BERT Classification Example (Nothing for you to Implement here)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4VAaLsahGRHA"
      },
      "source": [
        "Before you implement anything, here's an example of a classification model using BERT and the [Transformers](https://huggingface.co/transformers/) python library from Huggingface. This model is trained using the data you annotated in Homework 1 to do the same topic classification you did in Homework 3. Note that it gets a higher accuracy score then either the CNN or logistic regression model that we tried in Homework 3. BERT tops out here at an accuracy of around 0.637 on the dev data.\n",
        "\n",
        "Running the cells below will train this BERT-based classifier - this takes a while, so feel free to stop it running."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6WkRnOd-ZDiL",
        "outputId": "e4776a8c-05ec-4b37-c803-d20a706503df"
      },
      "source": [
        "!wget https://raw.githubusercontent.com/dbamman/nlp21/main/HW3/acl.train\n",
        "!wget https://raw.githubusercontent.com/dbamman/nlp21/main/HW3/acl.dev"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2021-02-17 07:07:19--  https://raw.githubusercontent.com/dbamman/nlp21/main/HW3/acl.train\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.111.133, 185.199.109.133, 185.199.110.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.111.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 1027009 (1003K) [text/plain]\n",
            "Saving to: ‘acl.train’\n",
            "\n",
            "acl.train           100%[===================>]   1003K  --.-KB/s    in 0.03s   \n",
            "\n",
            "2021-02-17 07:07:19 (34.4 MB/s) - ‘acl.train’ saved [1027009/1027009]\n",
            "\n",
            "--2021-02-17 07:07:19--  https://raw.githubusercontent.com/dbamman/nlp21/main/HW3/acl.dev\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.108.133, 185.199.109.133, 185.199.110.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.108.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 359826 (351K) [text/plain]\n",
            "Saving to: ‘acl.dev’\n",
            "\n",
            "acl.dev             100%[===================>] 351.39K  --.-KB/s    in 0.02s   \n",
            "\n",
            "2021-02-17 07:07:19 (20.3 MB/s) - ‘acl.dev’ saved [359826/359826]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i2Am5XHILvUt"
      },
      "source": [
        "trainingFile = \"acl.train\"\n",
        "devFile = \"acl.dev\"\n",
        "\n",
        "labels = {'APPLICATIONS': 11,\n",
        " 'CSSCA': 23,\n",
        " 'DIALOGUE': 12,\n",
        " 'DISCOURSE': 13,\n",
        " 'ETHICS': 8,\n",
        " 'GENERATION': 9,\n",
        " 'GREEN': 15,\n",
        " 'GROUNDING': 18,\n",
        " 'IE': 6,\n",
        " 'INTERPRET': 10,\n",
        " 'IR': 22,\n",
        " 'LEXSEM': 7,\n",
        " 'LING': 24,\n",
        " 'MLCLASS': 1,\n",
        " 'MLLM': 16,\n",
        " 'MT': 4,\n",
        " 'MULTILING': 3,\n",
        " 'OTHER': 25,\n",
        " 'PHON': 5,\n",
        " 'QA': 17,\n",
        " 'RESOURCES': 14,\n",
        " 'SA': 21,\n",
        " 'SENTSEM': 0,\n",
        " 'SPEECH': 19,\n",
        " 'SUMM': 2,\n",
        " 'SYNTAX': 20}\n",
        "\n",
        "def read_acl_data(filename, labels):\n",
        " \n",
        "    data = []\n",
        "    data_labels = []\n",
        "    file = open(filename)\n",
        "    for line in file:\n",
        "        cols = line.split(\"\\t\")\n",
        "        idd = cols[0]\n",
        "        label = cols[1]\n",
        "        title = cols[2]\n",
        "        abstract = cols[3]\n",
        "\n",
        "        data.append(\"%s %s\" % (title, abstract))\n",
        "        data_labels.append(labels[label])\n",
        "        \n",
        "    file.close()\n",
        "    return data, data_labels"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wYwiVO9dIg1H"
      },
      "source": [
        "class BERTClassifier(nn.Module):\n",
        "\n",
        "   def __init__(self, params):\n",
        "      super().__init__()\n",
        "        \n",
        "      self.tokenizer = BertTokenizer.from_pretrained('bert-base-cased', do_lower_case=False, do_basic_tokenize=False)\n",
        "      self.bert = BertModel.from_pretrained(\"bert-base-cased\")\n",
        "        \n",
        "      self.num_labels = params[\"label_length\"]\n",
        "\n",
        "      self.fc = nn.Linear(768, self.num_labels)\n",
        "\n",
        "   def get_batches(self, all_x, all_y, batch_size=32, max_toks=256):\n",
        "            \n",
        "      \"\"\" Get batches for input x, y data, with data tokenized according to the BERT tokenizer \n",
        "      (and limited to a maximum number of WordPiece tokens \"\"\"\n",
        "\n",
        "      batches_x=[]\n",
        "      batches_y=[]\n",
        "        \n",
        "      for i in range(0, len(all_x), batch_size):\n",
        "\n",
        "            current_batch=[]\n",
        "\n",
        "            x=all_x[i:i+batch_size]\n",
        "\n",
        "            batch_x = self.tokenizer(x, padding=True, truncation=True, return_tensors=\"pt\", max_length=max_toks)\n",
        "            batch_y=all_y[i:i+batch_size]\n",
        "\n",
        "            batches_x.append(batch_x.to(device))\n",
        "            batches_y.append(torch.LongTensor(batch_y).to(device))\n",
        "            \n",
        "      return batches_x, batches_y\n",
        "  \n",
        "\n",
        "   def forward(self, batch_x): \n",
        "    \n",
        "      bert_output = self.bert(input_ids=batch_x[\"input_ids\"],\n",
        "                         attention_mask=batch_x[\"attention_mask\"],\n",
        "                         token_type_ids=batch_x[\"token_type_ids\"],\n",
        "                         output_hidden_states=True)\n",
        "\n",
        "      # We're going to represent an entire document just by its [CLS] embedding (at position 0)\n",
        "      # And use the *last* layer output (layer -1)\n",
        "      # as a result of this choice, this embedding will be optimized for this purpose during the training process.\n",
        "      \n",
        "      bert_hidden_states = bert_output['hidden_states']\n",
        "\n",
        "      out = bert_hidden_states[-1][:,0,:]\n",
        "\n",
        "      out = self.fc(out)\n",
        "\n",
        "      return out.squeeze()\n",
        "\n",
        "   def evaluate(self, batch_x, batch_y):\n",
        "      \n",
        "      self.eval()\n",
        "      corr = 0.\n",
        "      total = 0.\n",
        "\n",
        "      with torch.no_grad():\n",
        "\n",
        "         for x, y in zip(batch_x, batch_y):\n",
        "            y_preds = self.forward(x)\n",
        "            for idx, y_pred in enumerate(y_preds):\n",
        "              prediction=torch.argmax(y_pred)\n",
        "              if prediction == y[idx]:\n",
        "                corr += 1.\n",
        "              total+=1                          \n",
        "      return corr/total\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MptJ2EPYRP67"
      },
      "source": [
        "train_x, train_y = read_acl_data(trainingFile, labels)\n",
        "dev_x, dev_y = read_acl_data(devFile, labels)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6k9j11-IRAvk",
        "outputId": "bffc77e2-58e3-4404-bfba-41fb02bf7d29"
      },
      "source": [
        "bert_model = BERTClassifier(params={\"label_length\": len(labels)})\n",
        "bert_model.to(device)\n",
        "\n",
        "batch_x, batch_y = bert_model.get_batches(train_x, train_y)\n",
        "dev_batch_x, dev_batch_y = bert_model.get_batches(dev_x, dev_y)\n",
        "\n",
        "optimizer = torch.optim.Adam(bert_model.parameters(), lr=1e-5)\n",
        "cross_entropy=nn.CrossEntropyLoss()\n",
        "\n",
        "num_epochs=30\n",
        "best_dev_acc = 0.\n",
        "\n",
        "for epoch in range(num_epochs):\n",
        "    bert_model.train()\n",
        "\n",
        "    # Train\n",
        "    for x, y in zip(batch_x, batch_y):\n",
        "      y_pred = bert_model.forward(x)\n",
        "      loss = cross_entropy(y_pred.view(-1, bert_model.num_labels), y.view(-1))\n",
        "      optimizer.zero_grad()\n",
        "      loss.backward()\n",
        "      optimizer.step()\n",
        "    \n",
        "    # Evaluate\n",
        "    dev_accuracy=bert_model.evaluate(dev_batch_x, dev_batch_y)\n",
        "    if epoch % 1 == 0:\n",
        "        print(\"Epoch %s, dev accuracy: %.3f\" % (epoch, dev_accuracy))\n",
        "        if dev_accuracy > best_dev_acc:\n",
        "          torch.save(bert_model.state_dict(), 'best-model-parameters.pt')\n",
        "          best_dev_acc = dev_accuracy\n",
        "\n",
        "bert_model.load_state_dict(torch.load('best-model-parameters.pt'))\n",
        "print(\"\\nBest Performing Model achieves dev accuracy of : %.3f\" % (best_dev_acc))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 0, dev accuracy: 0.126\n",
            "Epoch 1, dev accuracy: 0.240\n",
            "Epoch 2, dev accuracy: 0.411\n",
            "Epoch 3, dev accuracy: 0.491\n",
            "Epoch 4, dev accuracy: 0.529\n",
            "Epoch 5, dev accuracy: 0.554\n",
            "Epoch 6, dev accuracy: 0.571\n",
            "Epoch 7, dev accuracy: 0.597\n",
            "Epoch 8, dev accuracy: 0.574\n",
            "Epoch 9, dev accuracy: 0.591\n",
            "Epoch 10, dev accuracy: 0.614\n",
            "Epoch 11, dev accuracy: 0.583\n",
            "Epoch 12, dev accuracy: 0.603\n",
            "Epoch 13, dev accuracy: 0.606\n",
            "Epoch 14, dev accuracy: 0.611\n",
            "Epoch 15, dev accuracy: 0.600\n",
            "Epoch 16, dev accuracy: 0.609\n",
            "Epoch 17, dev accuracy: 0.620\n",
            "Epoch 18, dev accuracy: 0.594\n",
            "Epoch 19, dev accuracy: 0.603\n",
            "Epoch 20, dev accuracy: 0.611\n",
            "Epoch 21, dev accuracy: 0.614\n",
            "Epoch 22, dev accuracy: 0.629\n",
            "Epoch 23, dev accuracy: 0.617\n",
            "Epoch 24, dev accuracy: 0.620\n",
            "Epoch 25, dev accuracy: 0.620\n",
            "Epoch 26, dev accuracy: 0.614\n",
            "Epoch 27, dev accuracy: 0.617\n",
            "Epoch 28, dev accuracy: 0.617\n",
            "Epoch 29, dev accuracy: 0.606\n",
            "\n",
            "Best Performing Model achieves dev accuracy of : 0.629\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OdxZq0H1Ntfq"
      },
      "source": [
        "# OBJECT NUMBER Probe"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ggxpw3mRSBHK"
      },
      "source": [
        "Now let's move on to \"Probing\" the representations in BERT's layers. We'll explore this using a simple task called \"Object Number\", which tries to predict whether the direct object of the main verb in a sentence is singular (NN, label=1) or plural (NNS, label=0).  For more on this probe, see Conneau et al. (2018), [What you can cram into a single $&!#* vector: Probing sentence embeddings for linguistic properties](https://arxiv.org/pdf/1805.01070.pdf)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GZ5LFzIUt2J1",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "45443bce-4a2a-42b7-be6a-67065dc31087"
      },
      "source": [
        "# download the data\n",
        "!wget https://raw.githubusercontent.com/facebookresearch/SentEval/master/data/probing/obj_number.txt"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2021-02-17 08:52:36--  https://raw.githubusercontent.com/facebookresearch/SentEval/master/data/probing/obj_number.txt\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.109.133, 185.199.108.133, 185.199.110.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.109.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 8947485 (8.5M) [text/plain]\n",
            "Saving to: ‘obj_number.txt’\n",
            "\n",
            "obj_number.txt      100%[===================>]   8.53M  --.-KB/s    in 0.1s    \n",
            "\n",
            "2021-02-17 08:52:37 (68.0 MB/s) - ‘obj_number.txt’ saved [8947485/8947485]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CPhEr0cfSD65"
      },
      "source": [
        "def read_probe_data(filename):\n",
        "    labels={\"NNS\":0, \"NN\":1}\n",
        "\n",
        "    train=[]\n",
        "    dev=[]\n",
        "\n",
        "    with open(filename) as file:\n",
        "      for line in file:\n",
        "          cols = line.split(\"\\t\")\n",
        "          split = cols[0]\n",
        "          label = cols[1]\n",
        "          text = cols[2].rstrip()\n",
        "\n",
        "          if split == \"tr\":\n",
        "            train.append((text, labels[label]))\n",
        "          elif split == \"va\":\n",
        "            dev.append((text, labels[label]))\n",
        "\n",
        "    np.random.shuffle(train)\n",
        "    np.random.shuffle(dev)\n",
        "\n",
        "    train_x = []\n",
        "    train_y = []\n",
        "    \n",
        "    dev_x = []\n",
        "    dev_y = []\n",
        "\n",
        "    for text, label in train[:2000]:\n",
        "      train_x.append(text)\n",
        "      train_y.append(label)\n",
        "    \n",
        "    for text, label in dev[:2000]:\n",
        "      dev_x.append(text)\n",
        "      dev_y.append(label)\n",
        "\n",
        "    return train_x, train_y, dev_x, dev_y"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KISSQoorqrC6",
        "outputId": "f68d42b0-6f1b-4c7a-8dff-86f5aa6e330f"
      },
      "source": [
        "#Sets random seeds for reproducibility\n",
        "seed=0\n",
        "np.random.seed(seed)\n",
        "torch.manual_seed(seed)\n",
        "torch.cuda.manual_seed(seed)\n",
        "torch.backends.cudnn.deterministic = True\n",
        "\n",
        "# Load the data for the probe\n",
        "probe_train_x, probe_train_y, probe_dev_x, probe_dev_y=read_probe_data(\"obj_number.txt\")\n",
        "\n",
        "print(\"Label(y)\\tSentence(x)\")\n",
        "print(\"----------------------------------------------------------------------------------------\")\n",
        "for i in range(5):\n",
        "  print(\"%s\\t\\t%s\" % (probe_train_y[i], probe_train_x[i]))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Label(y)\tSentence(x)\n",
            "----------------------------------------------------------------------------------------\n",
            "1\t\tI untied my apron , pulled it off , and tossed it onto the counter .\n",
            "0\t\tSally handled her chopsticks like an expert as she popped a piece of fried Calamari into her mouth .\n",
            "0\t\tI continue my chores with a spring in my step .\n",
            "1\t\tShe had found a large central staircase , and moved up it unerringly .\n",
            "1\t\tBecause those lungs would require oxygen every day until I died .\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EaMDQDaMZZHv"
      },
      "source": [
        "## Baseline\n",
        "\n",
        "Since we are working with a new dataset, let's implement a simple baseline to give us some context for how well we can expect our probe to perform. A good baseline to try first is to always predict the most common label in the training data:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bz1VBUAlZS-v",
        "outputId": "d617ed11-60a6-4d97-bc9e-9c8b587211eb"
      },
      "source": [
        "counts=Counter()\n",
        "for l in probe_train_y:\n",
        "  counts[l]+=1\n",
        "\n",
        "most_common=counts.most_common(1)[0][0]\n",
        "\n",
        "cor=tot=0.\n",
        "for l in probe_dev_y:\n",
        "  if l == most_common:\n",
        "    cor+=1\n",
        "  tot+=1\n",
        "\n",
        "print(\"Baseline accuracy: %.3f\" % (cor/tot))\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Baseline accuracy: 0.504\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9JWgCyicZsaW"
      },
      "source": [
        "## Implementing the Probe"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Oi_a5RuTaLkj"
      },
      "source": [
        "Your job in this homework is to implement a probe for object number: for each of the 12 layers in BERT, train a classifier to predict whether the direct object of the main verb in an input sentence is a singular or plural noun. We provide a copy of the BERT classifier below; you will need to make some minor changes to it.  Keep in mind that a probe is designed to uncover what BERT has learned about linguistic structure only given its pretraining as a language model; your model cannot update the BERT parameters.\n",
        "\n",
        "You may find it useful to refer to Huggingface's documentation on BERT here: https://huggingface.co/transformers/model_doc/bert.html."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B4KTiinxQSju"
      },
      "source": [
        "class BERTLayerClassifier(nn.Module):\n",
        "\n",
        "   def __init__(self, layer_id, num_labels):\n",
        "      #* Do *not* change this function *\n",
        "      super().__init__()\n",
        "        \n",
        "      self.tokenizer = BertTokenizer.from_pretrained('bert-base-cased', do_lower_case=False, do_basic_tokenize=False)\n",
        "      self.bert = BertModel.from_pretrained(\"bert-base-cased\")\n",
        "      \n",
        "      self.layer_id = layer_id\n",
        "      self.num_labels = num_labels\n",
        "\n",
        "      self.fc = nn.Linear(768, self.num_labels)\n",
        "\n",
        "      #####\n",
        "      # Since we're probing what BERT has learned from its language modeling objective,\n",
        "      # we need to *not* update the BERT parameters during training.\n",
        "      #\n",
        "      # Do *not* change this\n",
        "      #\n",
        "      for param in self.bert.parameters():\n",
        "        param.requires_grad = False\n",
        "      #\n",
        "      ####\n",
        "      self.train()\n",
        "\n",
        "   def get_batches(self, all_x, all_y, batch_size=32, max_toks=256):\n",
        "            \n",
        "      \"\"\" Get batches for input x, y data, with data tokenized according to the BERT tokenizer \n",
        "      (and limited to a maximum number of WordPiece tokens \n",
        "      * Do *not* change this function *\n",
        "      \"\"\"\n",
        "\n",
        "      batches_x=[]\n",
        "      batches_y=[]\n",
        "        \n",
        "      for i in range(0, len(all_x), batch_size):\n",
        "\n",
        "            current_batch=[]\n",
        "\n",
        "            x=all_x[i:i+batch_size]\n",
        "\n",
        "            batch_x = self.tokenizer(x, padding=True, truncation=True, return_tensors=\"pt\", max_length=max_toks)\n",
        "            batch_y=all_y[i:i+batch_size]\n",
        "\n",
        "            batches_x.append(batch_x.to(device))\n",
        "            batches_y.append(torch.LongTensor(batch_y).to(device))\n",
        "            \n",
        "      return batches_x, batches_y\n",
        "  \n",
        "\n",
        "   def forward(self, batch_x): \n",
        "      bert_output = self.bert(input_ids=batch_x[\"input_ids\"],\n",
        "                         attention_mask=batch_x[\"attention_mask\"],\n",
        "                         token_type_ids=batch_x[\"token_type_ids\"],\n",
        "                         output_hidden_states=True)\n",
        "      \n",
        "      bert_hidden_states = bert_output['hidden_states']\n",
        "\n",
        "      \"\"\" Insert your code here \"\"\"\n",
        "      #bert_layer_output = .... \n",
        "      \"\"\" Insert your code here \"\"\"\n",
        "\n",
        "      out = self.fc(bert_layer_output)\n",
        "\n",
        "      return out.squeeze()\n",
        "\n",
        "   def evaluate(self, batch_x, batch_y):\n",
        "      #* Do *not* change this function *\n",
        "      \n",
        "      self.eval()\n",
        "      corr = 0.\n",
        "      total = 0.\n",
        "\n",
        "      with torch.no_grad():\n",
        "\n",
        "         for x, y in zip(batch_x, batch_y):\n",
        "            y_preds = self.forward(x)\n",
        "            for idx, y_pred in enumerate(y_preds):\n",
        "              prediction=torch.argmax(y_pred)\n",
        "              if prediction == y[idx]:\n",
        "                corr += 1.\n",
        "              total+=1\n",
        "      self.train()                    \n",
        "      return corr/total"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zSaLh4dgogez"
      },
      "source": [
        "Aside from any change you make to the BERTLayerClassifier module, the rest of your code should be implemented in the `runProbes` function below.\n",
        "\n",
        "Within the `runProbes` function, you'll need to:\n",
        "\n",
        "* Create one classifier for each of BERT's 12 layers.\n",
        "\n",
        "* Train each classifier for ***5*** epochs using the training data and labels for the Object Number probe that have been loaded above.\n",
        "\n",
        "* Evaluate the classifier using the dev data and labels for the Object Number probe.\n",
        "\n",
        "* Return your results on the dev data in the dictionary format specified in `runProbes'.\n",
        "\n",
        "The output of `runProbes` should be a dictionary of 12 accuracies, one for each layer, for layers 0 to 11.\n",
        "\n",
        "When you're finished, you should be able to give an answer to the following question: What layer in BERT is most encoding information on syntactic objecthood?\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "74lnYh6rPgrt"
      },
      "source": [
        "**Tips**:\n",
        "\n",
        "* Set your learning rate to 0.01.  When updating the BERT parameters for a task, the learning rate should be set to a small number (e.g., 1e-5, as in the ACL classification example above); but when you're *not* updating BERT (as is the case here), you should use a larger learning rate (such as 0.01, 0.001, etc.):\n",
        "\n",
        "    `optimizer = torch.optim.Adam(<your_model>.parameters(), lr=0.01)`\n",
        "\n",
        "* You should be training *separate* classifiers for each layer; make sure the updates you make to the parameters for a classifier on one layer while training do not affect the parameters for another layer.\n",
        "\n",
        "* BERT learns representations for each token in it's input. Under the hood, the `BertTokenizer` object used in `BERTLayerClassifier` adds special tokens to the beginning and end of each input, such that the sentence `Sally handled her chopsticks` is transformed into `[CLS] Sally handled her chopsticks [SEP]`. When using BERT to make sentence-level classifications, the [CLS] token is often treated as a representation of the entire sentence. Your classifier for layer ***L*** should only use the \"[CLS]\" token representation from layer ***L*** for making a prediction.  \n",
        "\n",
        "* Remember to tell pytorch to use the gpu for your models with the `.to(device)` function. Without using the gpu your code will run much more slowly!  \n",
        "\n",
        "* You'll want to use the functions `get_batches` and `evaluate` that have been implemented in `BERTLayerClassifier`.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4txDCPgCQ42b"
      },
      "source": [
        "def runProbes():\n",
        "  \"\"\"\n",
        "  Your function should return the `dev_accuracy_per_layer' dictionary.\n",
        "  This dictionary should be filled with 12 entries.\n",
        "  The keys of this dictionary should be the numbers 0 through 11, each of which\n",
        "  represents the ID of one of the 12 BERT layers.\n",
        "  For each BERT layer, update this dictionary (initial values are set to 0) \n",
        "  with your classification accuracy on the dev data for the Object Number task\n",
        "  using the corresponding BERT layer.\n",
        "  \"\"\"\n",
        "  dev_accuracy_per_layer = {0:0., 1:0., 2:0., 3:0., 4:0., 5:0.,\n",
        "                            6:0., 7:0., 8:0., 9:0., 10:0., 11:0.,}\n",
        "\n",
        "  \"\"\" Insert your code here \"\"\"\n",
        "\n",
        "  return dev_accuracy_per_layer"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "06ToiDbmhEcg"
      },
      "source": [
        "# Execute this cell to run your probes and save your results.\n",
        "dev_accuracy_per_layer = runProbes()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "suQOkLBOGcjw"
      },
      "source": [
        "# Export your results"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gk0lr4u2jbNo"
      },
      "source": [
        "# Print out your accuracy for each layer here\n",
        "\n",
        "for layer_id in dev_accuracy_per_layer:\n",
        "  print(f\"Accuracy for layer {layer_id}: {dev_accuracy_per_layer[layer_id]}\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lbitbsf5lLXn"
      },
      "source": [
        "# Run this cell to store your accuracy for each layer to a file called 'dev_accuracies.txt'\n",
        "# Your file should have 12 lines.\n",
        "# When you submit your homework, upload this saved file to Gradescope along with your notebook.\n",
        "\n",
        "def save_accuracies(dev_accuracy_per_layer, output_file):\n",
        "  with open(output_file, 'w') as f:\n",
        "    for layer_id in dev_accuracy_per_layer:\n",
        "      f.write(f\"{layer_id}\\t{dev_accuracy_per_layer[layer_id]}\\n\")\n",
        "\n",
        "save_accuracies(dev_accuracy_per_layer, output_file='devAccuracies.txt')"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}