notebooks/BERT4DAT_eval.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "-djxLyuc-Opk"
   },
   "source": [
    "# Prepare\n",
    "Install required libraries and import"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Epk5taxa99eI",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "!pip install pytorch-pretrained-bert"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Fr6bTWdl-XzF"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from typing import *\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "5rdgM39FGZpM"
   },
   "outputs": [],
   "source": [
    "from fastai import *\n",
    "from fastai.text import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "bpZkAwZl0DaA"
   },
   "outputs": [],
   "source": [
    "from pytorch_pretrained_bert.modeling import BertConfig, BertForSequenceClassification\n",
    "from pytorch_pretrained_bert import BertTokenizer, BertAdam\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "L57NdoEnLQa2"
   },
   "outputs": [],
   "source": [
    "from datetime import datetime"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "azCEB4CuQk9A"
   },
   "source": [
    "Check, if and what kind of GPU is used"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Wtzha3q7QjjU"
   },
   "outputs": [],
   "source": [
    "cuda_available = torch.cuda.is_available()\n",
    "print(cuda_available)\n",
    "if cuda_available:\n",
    "    curr_device = torch.cuda.current_device()\n",
    "    print(torch.cuda.get_device_name(curr_device))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "mOKXVgJgGtYV"
   },
   "source": [
    "Create a config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "i0lgLyC6Gsnf"
   },
   "outputs": [],
   "source": [
    "class Config(dict):\n",
    "    def __init__(self, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        for k, v in kwargs.items():\n",
    "            setattr(self, k, v)\n",
    "    \n",
    "    def set(self, key, val):\n",
    "        self[key] = val\n",
    "        setattr(self, key, val)\n",
    "\n",
    "config = Config(\n",
    "    bert_model_name=\"bert-base-uncased\", # default: \"bert-base-uncased\", alt: \"bert-large-uncased\"\n",
    "    max_lr=2e-5, # default: 2e-5\n",
    "    moms=(0.8, 0.7), # default: (0.95, 0.85) or (0.8, 0.7)\n",
    "    epochs=10, # default: 5, 6, 10 or 20\n",
    "    use_fp16=False, # default: False\n",
    "    bs=2, # default: 2, 4 or 8\n",
    "    max_seq_len=512, # default: 128\n",
    "    train_size=0.9, #default: 0.9\n",
    "    use_bertAdam=True, #default: True\n",
    "    loss_func=nn.CrossEntropyLoss(), #default: None or nn.CrossEntropyLoss()\n",
    "    threshold=0.9, #default: 0.9\n",
    "    seed=904727489, #default: 31337, 424242 (reproducibility) or None\n",
    ")\n",
    "\n",
    "config_eval = Config(\n",
    "    num_repeats=1,\n",
    "    num_folds=10,\n",
    "    log_to_file = True,\n",
    "    log_file='./log/log_shrunk.txt',\n",
    ")\n",
    "shrunk = False\n",
    "load_from_gdrive = False\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "seeds = []\n",
    "if config.seed is None:\n",
    "    for i in range(config_eval.num_repeats):\n",
    "        seeds.append(random.randint(0, 2**31))\n",
    "else:\n",
    "    for i in range(config_eval.num_repeats):\n",
    "        seeds.append(config.seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# easily control what data is loaded and where to log to\n",
    "if not shrunk:\n",
    "    config_eval.log_file = './log/log_beg.txt'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "rvckhVpveeA5"
   },
   "source": [
    "Set up where the data comes from"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "mYcCTWgMediC"
   },
   "outputs": [],
   "source": [
    "data_folder = './data/'\n",
    "\n",
    "if shrunk:\n",
    "    data_filenames = ['1_classCorpus_Shrunk.tsv']\n",
    "else:\n",
    "    data_filenames = ['1_classCorpus_BegOnly_512.tsv']\n",
    "\n",
    "# One file shortened: ['1_classCorpus_Shrunk.tsv']\n",
    "# One file unshortened: ['1_classCorpus_BegOnly_512.tsv']\n",
    "# Second (smaller) dataset:\n",
    "# One file shortened: ['2_classCorpus_Shrunk.tsv']\n",
    "# One file unshortened: ['2_classCorpus_BegOnly_512.tsv']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "OmGISBrhW-VJ"
   },
   "outputs": [],
   "source": [
    "if load_from_gdrive:\n",
    "    from google.colab import drive\n",
    "    # Connect to drive to load the corpus from there\n",
    "    data_folder = data_folder.replace('.', '/content/drive/My Drive')\n",
    "    config_eval.log_file = config_eval.log_file.replace('.', '/content/drive/My Drive/data', 1)\n",
    "    drive.mount('/content/drive', force_remount=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Ptp6NhIC_FQb"
   },
   "source": [
    "# Data\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "SVU_viFX-ezy"
   },
   "source": [
    "To import the dataset, first we have to connect to our Google drive. For this, we have to authenticating the access and mount the drive"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ILcAtepvK3T4"
   },
   "outputs": [],
   "source": [
    "# Logging\n",
    "def get_info():\n",
    "    model_config = 'model: {}, max_lr: {}, epochs: {}, bs: {}, msl: {}, train_size: {}, BERT-Adam: {}, FP16: {}, Loss: {}, Threshold: {}, Data: {}'.format(config.bert_model_name, config.max_lr, config.epochs, config.bs, config.max_seq_len, config.train_size, config.use_bertAdam, config.use_fp16, config.loss_func, config.threshold, data_filenames)\n",
    "    return model_config\n",
    "    \n",
    "def logResult(precisions, recalls, accuracy, confusionMatrix):\n",
    "    if config_eval.log_to_file:\n",
    "        with open(config_eval.log_file, 'a') as log:\n",
    "            datetime.now().strftime('%Y-%m-%d %H:%M:%S')\n",
    "            avg_precision = np.average(precisions)\n",
    "            avg_recall = np.average(recalls)\n",
    "            log_txt = 'Precision: Average {0:.2%} -> {1}\\n Recall: Average {2:.2%} -> {3}\\n'.format(avg_precision, precisions, avg_recall, recalls)\n",
    "            log_txt += 'Accuracy: {}\\n'.format(accuracy)\n",
    "            log_txt += '{}'.format(confusionMatrix)\n",
    "            log.write(\"{}\\n\".format(log_txt))\n",
    "\n",
    "def logLine(line):\n",
    "    if config_eval.log_to_file:\n",
    "        with open(config_eval.log_file, 'a') as log:\n",
    "            log.write(line + '\\n')\n",
    "    \n",
    "logLine(datetime.now().strftime('%Y-%m-%d %H:%M:%S'))\n",
    "logLine(get_info())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "JmN-0wJEBAEH"
   },
   "source": [
    "Create proper tokenizer for our data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "6anB63ppBAtB"
   },
   "outputs": [],
   "source": [
    "\n",
    "class FastAiBertTokenizer(BaseTokenizer):\n",
    "    \"\"\"Wrapper around BertTokenizer to be compatible with fast.ai\"\"\"\n",
    "    def __init__(self, tokenizer: BertTokenizer, max_seq_len: int=512, **kwargs):\n",
    "        self._pretrained_tokenizer = tokenizer\n",
    "        self.max_seq_len = max_seq_len\n",
    "\n",
    "    def __call__(self, *args, **kwargs):\n",
    "        return self\n",
    "\n",
    "    def tokenizer(self, t:str) -> List[str]:\n",
    "        \"\"\"Limits the maximum sequence length. Prepend with [CLS] and append [SEP]\"\"\"\n",
    "        return [\"[CLS]\"] + self._pretrained_tokenizer.tokenize(t)[:self.max_seq_len - 2] + [\"[SEP]\"]\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "1G8rFbEEJWyu"
   },
   "source": [
    "Now, we can create our own databunch using the tokenizer above. Notice we're passing the include_bos=False and include_eos=False options. This is to prevent fastai from adding its own SOS/EOS tokens that will interfere with BERT's SOS/EOS tokens.\n",
    "\n",
    "We can pass our own list of Preprocessors to the databunch."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "TNRRj6jIJrp2"
   },
   "outputs": [],
   "source": [
    "class BertTokenizeProcessor(TokenizeProcessor):\n",
    "    \"\"\"Special Tokenizer, where we remove sos/eos tokens since we add that ourselves in the tokenizer.\"\"\"\n",
    "    def __init__(self, tokenizer):\n",
    "        super().__init__(tokenizer=tokenizer, include_bos=False, include_eos=False)\n",
    "\n",
    "class BertNumericalizeProcessor(NumericalizeProcessor):\n",
    "    \"\"\"Use a custom vocabulary to match the original BERT model.\"\"\"\n",
    "    def __init__(self, *args, **kwargs):\n",
    "        super().__init__(*args, vocab=Vocab(list(bert_tok.vocab.keys())), **kwargs)\n",
    "\n",
    "def get_bert_processor(tokenizer:Tokenizer=None, vocab:Vocab=None):\n",
    "    return [BertTokenizeProcessor(tokenizer=tokenizer),\n",
    "            NumericalizeProcessor(vocab=vocab)]\n",
    "\n",
    "class BertDataBunch(TextDataBunch):\n",
    "    @classmethod\n",
    "    def from_df(cls, path:PathOrStr, train_df:DataFrame, valid_df:DataFrame, test_df:Optional[DataFrame]=None,\n",
    "              tokenizer:Tokenizer=None, vocab:Vocab=None, classes:Collection[str]=None, text_cols:IntsOrStrs=1,\n",
    "              label_cols:IntsOrStrs=0, **kwargs) -> DataBunch:\n",
    "        \"Create a `TextDataBunch` from DataFrames.\"\n",
    "        p_kwargs, kwargs = split_kwargs_by_func(kwargs, get_bert_processor)\n",
    "        # use our custom processors while taking tokenizer and vocab as kwargs\n",
    "        processor = get_bert_processor(tokenizer=tokenizer, vocab=vocab, **p_kwargs)\n",
    "        if classes is None and is_listy(label_cols) and len(label_cols) > 1: classes = label_cols\n",
    "        src = ItemLists(path, TextList.from_df(train_df, path, cols=text_cols, processor=processor),\n",
    "                      TextList.from_df(valid_df, path, cols=text_cols, processor=processor))\n",
    "        src = src.label_for_lm() if cls==TextLMDataBunch else src.label_from_df(cols=label_cols, classes=classes)\n",
    "        if test_df is not None: src.add_test(TextList.from_df(test_df, path, cols=text_cols))\n",
    "        return src.databunch(**kwargs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "IwxQFKykzpQq"
   },
   "source": [
    "Load/get the different data sets for training and evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "oeaTvNRTypP0"
   },
   "outputs": [],
   "source": [
    "def load_data(filename):\n",
    "    fpath = data_folder + filename\n",
    "    df = pd.read_csv(fpath, sep='\\t', usecols=['label', 'text'])\n",
    "    df = df.dropna()\n",
    "    return df\n",
    "\n",
    "def load_all_data(filenames):\n",
    "    df = load_data(filenames[0])\n",
    "    for i in range(1, len(filenames)):\n",
    "        df = df.append(load_data(filenames[i]))\n",
    "    return df\n",
    "\n",
    "# load the datasets from files\n",
    "df = load_all_data(data_filenames)\n",
    "\n",
    "print(df.shape)\n",
    "print(df['label'].value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_label_indices(df):\n",
    "    #prepare labels\n",
    "    labels = df['label'].unique()\n",
    "    labels = np.delete(labels, np.where(labels == 'unrelated'))\n",
    "    labels.sort() \n",
    "  \n",
    "    #create dict\n",
    "    labelDict = dict()\n",
    "    for i in range (0, len(labels)):\n",
    "        labelDict[labels[i]] = i\n",
    "    labelDict['unrelated'] = len(labels)\n",
    "    return labelDict\n",
    "\n",
    "label_indices = create_label_indices(df)\n",
    "print(label_indices)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "SClv488eQC8B"
   },
   "source": [
    "# Create Predictor and Evaluator\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "prtJ_TGhC2PO"
   },
   "source": [
    "Create a predictor class. Just uses the prediction of the classifier/learner, but labels with confidentiality below a threshold get labeled as 'unrelated'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "qubb_Ka-C78O"
   },
   "outputs": [],
   "source": [
    "class Predictor:\n",
    "    def __init__(self, classifier, threshold=0.85, default_value =  'unrelated'):\n",
    "        self.classifier = classifier\n",
    "        self.threshold = threshold\n",
    "        self.classes = self.classifier.data.classes\n",
    "        self.default_value = default_value\n",
    "\n",
    "    def predict(self, text):\n",
    "        prediction = self.classifier.predict(text)\n",
    "        prediction_class = prediction[1]\n",
    "        prob = prediction[2][prediction_class].item()\n",
    "        if prob > self.threshold:\n",
    "            return self.classes[prediction_class]\n",
    "        else: return self.default_value   "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "KcJfuU7lkaT_"
   },
   "source": [
    "Create a evaluator class along with some useful functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ZGuA3I0h24ob"
   },
   "outputs": [],
   "source": [
    "# Eval builds and uses a confusion matrix M like this one:\n",
    "#        | Declare L1  |  Declare L2 | Declare L3 | \n",
    "# |Is L1 |             |             |            | \n",
    "# |Is L2 |             |             |            | \n",
    "# |Is L3 |             |             |            | \n",
    "# Precision for, e.g., L1 is then M_ii/Sum_j(M_ji) with i=0, so M_ii divided by the column for L1\n",
    "# Recall for, e.g., L1 is then M_ii/Sum_j(M_ij) with i=0, so M_ii divided by the row for L1\n",
    "\n",
    "def calculate_precisions(confusion_matrix):\n",
    "    column_val = np.sum(confusion_matrix, axis=0)\n",
    "    for i in range(len(confusion_matrix)):\n",
    "        if not column_val[i] == 0:\n",
    "            column_val[i] = confusion_matrix[i,i] / column_val[i]\n",
    "    return column_val\n",
    "\n",
    "def calculate_recalls(confusion_matrix):\n",
    "    row_val = np.sum(confusion_matrix, axis=1)\n",
    "    for i in range(len(confusion_matrix)):\n",
    "        if not row_val[i] == 0:\n",
    "            row_val[i] = confusion_matrix[i,i] / row_val[i]\n",
    "    return row_val\n",
    "\n",
    "def calculate_accuracy(confusion_matrix):\n",
    "    matrix_sum = confusion_matrix.sum()\n",
    "    true_sum = confusion_matrix.diagonal().sum()\n",
    "    accuracy = true_sum / matrix_sum\n",
    "    return accuracy\n",
    "\n",
    "class Evaluator:\n",
    "    def __init__(self, predictor):\n",
    "        self.predictor = predictor\n",
    "  \n",
    "    def evaluate(self, df_eval, num_labels=None):\n",
    "        num_labels = len(label_indices)\n",
    "        confusion_matrix = np.zeros(shape=(num_labels, num_labels))\n",
    "        for tuple in df_eval.itertuples():\n",
    "            gold_label = tuple.label\n",
    "            idx_gold_label = label_indices[gold_label]\n",
    "            pred_label = self.predictor.predict(tuple.text)\n",
    "            idx_pred_label = label_indices[pred_label]\n",
    "            confusion_matrix[idx_gold_label][idx_pred_label] += 1\n",
    "        return confusion_matrix\n",
    "   "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "zyVQS13d5Sft"
   },
   "source": [
    "# Cross-Validation\n",
    "\n",
    "Combine all the stuff above and do a cross-validation, where everything is executed and evaluated multiple times with different shuffling of the data. This way, we know better how stable our approach is regarding different training and test data. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "GeTaowHZzrSJ"
   },
   "source": [
    "Create the class CrossValidator that will do the training and validation.\n",
    "Properly mix the train and eval set!\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "T83UogVz5XJJ"
   },
   "outputs": [],
   "source": [
    "def split_dataframe(df, train_size = 0.9, random_state = None):\n",
    "    # split data into training and validation set\n",
    "    df_trn, df_valid = train_test_split(df, stratify = df['label'], train_size = train_size, random_state = random_state)\n",
    "    return df_trn, df_valid\n",
    "  \n",
    "class CrossValidator:\n",
    "    '''Cross Validation done as k-fold cross validation.'''\n",
    "\n",
    "    def __init__(self, config, df):\n",
    "        self.config = config\n",
    "        self.df = df.sample(frac=1, axis=0, random_state = config.seed) # shuffle data\n",
    "        self.num_labels = df['label'].nunique()\n",
    "\n",
    "\n",
    "    def __create_databunch(self, df_trn, df_valid):\n",
    "        bert_tok = BertTokenizer.from_pretrained(self.config.bert_model_name,)\n",
    "        fastai_tokenizer = Tokenizer(tok_func=FastAiBertTokenizer(bert_tok, max_seq_len=self.config.max_seq_len), pre_rules=[], post_rules=[])\n",
    "        fastai_bert_vocab = Vocab(list(bert_tok.vocab.keys()))\n",
    "        return BertDataBunch.from_df(\".\", \n",
    "                         train_df=df_trn,\n",
    "                         valid_df=df_valid,\n",
    "                         tokenizer=fastai_tokenizer,\n",
    "                         vocab=fastai_bert_vocab,\n",
    "                         bs=self.config.bs,\n",
    "                         text_cols='text',\n",
    "                         label_cols='label',\n",
    "                         collate_fn=partial(pad_collate, pad_first=False, pad_idx=0),\n",
    "                    )\n",
    "\n",
    "\n",
    "    def __create_learner(self, databunch):\n",
    "        bert_model = BertForSequenceClassification.from_pretrained(self.config.bert_model_name, num_labels=self.num_labels)\n",
    "\n",
    "        optimizer = AdamW # AdamW is the default optimizer of fastai.Learner\n",
    "        if self.config.use_bertAdam:\n",
    "          # BertAdam optimizer\n",
    "          optimizer = partial(BertAdam)\n",
    "\n",
    "        learner = Learner(\n",
    "            databunch, bert_model,\n",
    "            optimizer,\n",
    "            metrics=accuracy,\n",
    "            loss_func=self.config.loss_func\n",
    "        )\n",
    "        if self.config.use_fp16:\n",
    "            learner.to_fp16()\n",
    "        return learner\n",
    "\n",
    "    def split_for_fold(self, fold, num_folds = 10):\n",
    "        ''' Splits the data into two parts. the first part is the i'th fold of a k-fold and the second part is the rest of the data '''\n",
    "        n, i, k = len(self.df), fold, num_folds\n",
    "        df_eval = self.df[n*(i-1)//k:n*i//k]\n",
    "        df_train = self.df[:n*(i-1)//k].append(self.df[n*i//k:])\n",
    "        return df_train, df_eval\n",
    "\n",
    "    def validate_one_fold(self, fold, num_folds, info = None):\n",
    "        print('Shuffle data and create fold.')\n",
    "        df_trn, df_eval = self.split_for_fold(fold, num_folds)\n",
    "        df_trn, df_valid = split_dataframe(df_trn, train_size = self.config.train_size, random_state = self.config.seed)\n",
    "        databunch = self.__create_databunch(df_trn, df_valid)\n",
    "\n",
    "        if info: print(info)\n",
    "        print('Create classifier and start training it. Currently in fold {}.'.format(fold))\n",
    "        learner = self.__create_learner(databunch)\n",
    "        learner.fit_one_cycle(self.config.epochs, max_lr=self.config.max_lr, moms=self.config.moms)\n",
    "\n",
    "        print('Start evaluating the trained classifier on the evaluation data.')\n",
    "        evaluator = Evaluator(Predictor(learner, threshold=self.config.threshold))\n",
    "        confusion_matrix = evaluator.evaluate(df_eval)\n",
    "        precisions = calculate_precisions(confusion_matrix)\n",
    "        recalls = calculate_recalls(confusion_matrix)\n",
    "        accuracy = calculate_accuracy(confusion_matrix)\n",
    "        del learner\n",
    "        del databunch\n",
    "        return precisions, recalls, accuracy, confusion_matrix\n",
    "\n",
    "    def validate(self, num_folds):\n",
    "        current_fold = 0\n",
    "        accuracy_sum = 0\n",
    "        recall_sum = np.zeros(self.num_labels)\n",
    "        precision_sum = np.zeros(self.num_labels)\n",
    "        confusion_matrix_sum = np.zeros((self.num_labels, self.num_labels))\n",
    "        info = None\n",
    "\n",
    "        for i in range(num_folds):\n",
    "            current_fold = i+1\n",
    "            precisions, recalls, accuracy, confusion_matrix = self.validate_one_fold(current_fold, num_folds, info)\n",
    "\n",
    "            precision_sum += precisions  \n",
    "            recall_sum += recalls\n",
    "            accuracy_sum += accuracy\n",
    "            confusion_matrix_sum += confusion_matrix\n",
    "\n",
    "            recall = np.average(recall_sum / current_fold)\n",
    "            precision = np.average(precision_sum / current_fold)\n",
    "            avg_acc = accuracy_sum / current_fold\n",
    "            info = 'Finished fold {0} with average recall of {1:.2%}, average precision of {2:.2%}, and avg. accuracy of {3:.2%}'.format(current_fold, recall, precision, avg_acc)\n",
    "\n",
    "        recall = recall_sum / num_folds\n",
    "        precision = precision_sum / num_folds\n",
    "        avg_acc = accuracy_sum / num_folds\n",
    "        confusion_matrix = confusion_matrix_sum # CARE! Not divided by num_folds, just 'raw' confusion matrix (for analysing/debugging etc)\n",
    "        info = 'Finished validation with recalls of {0} and precisions of {1} and accuracy of {2}'.format(recall, precision, avg_acc)\n",
    "        print(info)\n",
    "        return precision, recall, avg_acc, confusion_matrix"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Method to set seeds to allow reproducibility"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def set_seed(seed):\n",
    "    random.seed(seed)\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "_1KUoNpCq8fh"
   },
   "source": [
    "Start the cross-validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "da-sBmbOCLYc"
   },
   "outputs": [],
   "source": [
    "for round in range(config_eval.num_repeats):\n",
    "    curr_seed = seeds[round]\n",
    "    set_seed(curr_seed)\n",
    "    logLine('Seed: {}'.format(curr_seed))\n",
    "\n",
    "    cross_validator = CrossValidator(config, df)\n",
    "    result = cross_validator.validate(config_eval.num_folds)  \n",
    "    logResult(result[0], result[1], result[2], result[3])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "TtMwAoU42YNW"
   },
   "source": [
    "Dict: {'audit': 0, 'authenticate': 1, 'heartbeat': 2, 'pooling': 3, 'scheduler': 4, 'unrelated': 5}"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "ClassBERTClassifier_Short_Eval.ipynb",
   "provenance": [],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}