source_code_p1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "uOcOtZ-WcH09"
   },
   "source": [
    "# Supervised Learning Coursework 2 - Part I"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "id": "8Qo_MGAhxKH3"
   },
   "source": [
    "## Import Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true,
    "id": "lUyumSzWvkoG"
   },
   "outputs": [],
   "source": [
    "# Import libraries\n",
    "import numpy as np\n",
    "from scipy.stats import mode \n",
    "import cupy as cp\n",
    "from matplotlib import pyplot as plt\n",
    "from itertools import combinations\n",
    "import seaborn as sns\n",
    "import pandas as pd\n",
    "import time\n",
    "import json\n",
    "import os"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Create Results Folder"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "Run the below code to create a folder where the results will be saved."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# The folder name can be modified to whatever you wish\n",
    "results_dir= \"./slcw2_results/\"\n",
    "\n",
    "if not os.path.exists(results_dir):\n",
    "    os.mkdir(results_dir)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "id": "o-i0us9uxORL"
   },
   "source": [
    "## Create Helper Functions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true,
    "id": "FSqZUjh8xq6l"
   },
   "source": [
    "These functions are essential for developing the models."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "### Kernel Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# Polynomial Kernel\n",
    "def polynomial_kernel_matrix(X1, X2, d=3):\n",
    "    \n",
    "    \"\"\" \n",
    "    Compute polynomial kernel matrix, given data matrices.\n",
    "    \n",
    "    Args\n",
    "    ----\n",
    "    X1 - training data matrix of shape (m, n) where m is the number of training instances and n is the number of features\n",
    "    X2 - matrix of shape (l, n) where l=m if we are generating matrix for training. Otherwise, l is the number of new points we are predicting on.\n",
    "    d - degree of polynomial kernel\n",
    "    \n",
    "    Returns\n",
    "    -------\n",
    "    kernel matrix of dimensions (m, l)\n",
    "    \"\"\"\n",
    "    \n",
    "    return (X1@X2.T)**d\n",
    "\n",
    "# Gaussian Kernel\n",
    "def gaussian_kernel_matrix(X1, X2, c=1):\n",
    "    \n",
    "    \"\"\" \n",
    "    Compute gaussian kernel matrix, given data matrices.\n",
    "    \n",
    "    Args\n",
    "    ----\n",
    "    X1 - training data matrix of shape (m, n) where m is the number of training instances and n is the number of features\n",
    "    X2 - matrix of shape (l, n) where l=m if we are generating matrix for training. Otherwise, l is the number of new points we are predicting on.\n",
    "    c - width of Gaussian kernel\n",
    "    \n",
    "    Returns\n",
    "    -------\n",
    "    kernel matrix of dimensions (m, l)\n",
    "    \"\"\"\n",
    "    \n",
    "    B = X1 @ X2.T\n",
    "    norm_sq = np.diagonal(X1@X1.T).reshape(-1, 1) - 2*B + np.diagonal(X2@X2.T).reshape(1, -1)\n",
    "    return np.exp(-1 * c * norm_sq)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "### Data Processing Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true,
    "id": "kLqrzTUh5DWi"
   },
   "outputs": [],
   "source": [
    "def preprocess(dataset, flat=True):\n",
    "    \"\"\" Preprocess dataset by separating labels and image pixels. If flat=True, then each datapoint is a flattened vector. \"\"\"\n",
    "\n",
    "    # Extract labels\n",
    "    y = dataset[:, 0]\n",
    "\n",
    "    # Extract pixel values\n",
    "    x = dataset[:, 1:]\n",
    "\n",
    "    if not flat:\n",
    "        # Reshape into image dimensions\n",
    "        x = x.reshape((x.shape[0], 16, 16))\n",
    "\n",
    "    return x, y\n",
    "\n",
    "def create_subsets(data, labels, classes):\n",
    "    \n",
    "    \"\"\" \n",
    "    Create subsets of the dataset which only contain examples and labels for selected class pairs.\n",
    "    \n",
    "    Args\n",
    "    ----\n",
    "    data - dataset of dimensions (m, n) where m is the number of instances and n is the number of features\n",
    "    labels - true labels for data, must be of shape (m, )\n",
    "    classes - the class pairs to select from the full data matrix to generate data subset. Should be a tuple/list.\n",
    "    \n",
    "    Returns\n",
    "    -------\n",
    "    subset_examples - set of data examples corresponding to the desired class pairs\n",
    "    subset_labels - true labels for the subset_examples.\n",
    "    \"\"\"\n",
    "    \n",
    "    # Find indices of rows with the desired classes\n",
    "    class_1 = np.where(labels == classes[0])\n",
    "    class_2 = np.where(labels == classes[1])\n",
    "    \n",
    "    # select rows from entire dataset\n",
    "    subset_examples = np.concatenate((data[class_1], data[class_2]))\n",
    "    subset_labels = np.concatenate((labels[class_1], labels[class_2]))\n",
    "    \n",
    "    return subset_examples, subset_labels\n",
    "\n",
    "def split_data(inputs, targets, test_proportion, shuffle=None):\n",
    "    \"\"\"\n",
    "    Splits the data into training and test sets.\n",
    "\n",
    "    Args\n",
    "    ----\n",
    "    inputs : NumPy array of input data. Should be of shape (# examples, # features)\n",
    "    targets : NumPy array of target data. Should be of shape (# examples, 1)\n",
    "    test_proportion : Value between 0 and 1 which specifies how much of the data to use for testing.\n",
    "    shuffle : Optional. Set to True if you want the data shuffled and then split.\n",
    "    seed : Optional. Set for reproducible results.\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    train_X : NumPy array of training examples. Should be of shape (# examples, # features)\n",
    "    train_Y : NumPy array of training targets. Should be of shape (# examples, 1)\n",
    "    test_X : NumPy array of testing examples. Should be of shape (# examples, # features)\n",
    "    test_Y : NumPy array of testing targets. Should be of shape (# examples, 1)\n",
    "    \"\"\"\n",
    "  \n",
    "    #Stores the number of data points\n",
    "    nData = inputs.shape[0]\n",
    "\n",
    "    # Shuffle data\n",
    "    if shuffle:\n",
    "        #Generate a shuffled version of the array indices\n",
    "        shuffled_indices = np.random.permutation(nData)\n",
    "        #Shuffle the inputs as per in the array of shuffled indices\n",
    "        shuffled_inputs = inputs[shuffled_indices, :]\n",
    "        shuffled_targets = targets[shuffled_indices]\n",
    "    else:\n",
    "        #If shuffle is set to False then we just work with the data in its original order\n",
    "        shuffled_indices = None\n",
    "        shuffled_inputs = inputs\n",
    "        shuffled_targets = targets\n",
    "\n",
    "    # Calculate the split index based on the specified proportions\n",
    "    split_index = int((1 - test_proportion) * nData)\n",
    "    \n",
    "    # Collect train and test indices\n",
    "    train_idxs = shuffled_indices[:split_index]\n",
    "    test_idxs = shuffled_indices[split_index:]\n",
    "    cache = (train_idxs, test_idxs)\n",
    "\n",
    "    # Select the examples up to the split index to be used as training set\n",
    "    train_X = shuffled_inputs[:split_index]\n",
    "    train_Y = shuffled_targets[:split_index]\n",
    "    # Select the examples from the split index onwards to be used as the test set\n",
    "    test_X = shuffled_inputs[split_index:]\n",
    "    test_Y = shuffled_targets[split_index:]\n",
    "\n",
    "    return train_X, train_Y, test_X, test_Y, cache"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Load full dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# Load the full dataset\n",
    "full_dataset = np.genfromtxt(\"data/zipcombo.dat\")\n",
    "\n",
    "# Preprocess full dataset using helper function\n",
    "x, y = preprocess(full_dataset)\n",
    "\n",
    "# Inpsect the full dataset \n",
    "print(\"Number of records in full dataset = {}\".format(full_dataset.shape[0]))\n",
    "print(\"Labels for the full dataset = {}\".format(np.unique(full_dataset[:, 0])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 202
    },
    "hidden": true,
    "id": "RUe-umQZ0Fzn",
    "outputId": "70bec44d-44f0-41e9-fee8-38f8c1f91018"
   },
   "outputs": [],
   "source": [
    "# Visualize a few images\n",
    "np.random.seed(1)\n",
    "idxs = np.random.choice(len(x), size=7)\n",
    "sample_imgs = x[idxs].reshape(7, 16, 16)\n",
    "sample_lbls = y[idxs]\n",
    "\n",
    "plt.figure(figsize=(17, 6))\n",
    "for i in range(len(sample_imgs)):\n",
    "    plt.subplot(1, len(sample_imgs), i+1)\n",
    "    plt.imshow(sample_imgs[i], plt.cm.gray)\n",
    "    plt.title(\"Image Label = {}\".format(sample_lbls[i]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "id": "0pTwCX_-xSKx"
   },
   "source": [
    "## Setup Multi-Class Kernel Perceptron Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true,
    "id": "28h63Lb7x2dB"
   },
   "source": [
    "My kernel perceptron model is represented using the below python class. It can work with the Polynomial kernel or the Gaussian kernel and is designed for one-vs-all classification."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true,
    "id": "qhTRY_c214aD"
   },
   "outputs": [],
   "source": [
    "class Kernel_Perceptron():\n",
    "\n",
    "    \"\"\" \n",
    "    \n",
    "    Kernel Perceptron Class. Can be modified to train in one-vs-all (ova) mode or one-vs-one (ovo) mode.\n",
    "    \n",
    "    Args\n",
    "    ----\n",
    "    train_x : training data of shape (m, n) where m is the number of examples and n is number of features\n",
    "    train_y : training labels of shape (m, )\n",
    "    test_x : testing data of shape (l, n) where l is number of testing examples\n",
    "    test_y : testing labels of shape (l, )\n",
    "    \n",
    "    Attributes\n",
    "    ----------\n",
    "    mode : can be ova or ovo\n",
    "    kernel_func : kernel matrix function\n",
    "    classes : list of unique classes in dataset\n",
    "    k : number of unique classes\n",
    "    m_train : number of training examples\n",
    "    m_test : number of test examples\n",
    "    train_K : kernel matrix or list of kernel matrices for training data, based on whether mode=ova or ovo\n",
    "    test_K : kernelm matrix of list of kernel matrices for test data, based on whether mode=ova or ovo\n",
    "    train_y : vector of training labels\n",
    "    test_y : vector of testing labels\n",
    "    alphas : matrix of shape (k, m_train) or list of vectors of shape (m_train, ) based on whether mode is ova or ovo\n",
    "    classifier_K : list of kernel matrices for each subset of data for class pairs\n",
    "    classifier_y : list of training labels for each subset of data for class pairs\n",
    "    \n",
    "    Methods\n",
    "    -------\n",
    "    fit - Fits model using fit_ova or fit_ovo method, based on the value of mode attribute\n",
    "    predict - generates predictions using predict_ovo or predict_ova, based on the value of mode attribute\n",
    "    evaluate - computes misclassification error\n",
    "    evaluate_binary_classifier - required for training indivitual classifiers as part of fit_ovo method\n",
    "    \n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, train_x, train_y, test_x, test_y, kernel_func, mode=\"ova\"):\n",
    "        \n",
    "        # Training mode - can be one-vs-all (ova) or one-vs-one (ovo)\n",
    "        self.mode = mode\n",
    "        \n",
    "        # Type of kernel function used to create kernel matrix\n",
    "        self.kernel_func = kernel_func\n",
    "        \n",
    "        # Number of unique classes\n",
    "        self.classes = np.unique(train_y)\n",
    "        self.k = len(self.classes)\n",
    "        \n",
    "        # Dimensions of datasets\n",
    "        self.m_train = len(train_y) \n",
    "        self.m_test = len(test_y)\n",
    "        \n",
    "        # Process training dataset differently based on the mode of training\n",
    "        if mode == \"ova\":\n",
    "            # create gram matrices\n",
    "            self.train_K = kernel_func(train_x, train_x)\n",
    "            self.test_K = kernel_func(train_x, test_x)\n",
    "            self.train_y = train_y\n",
    "            self.test_y = test_y\n",
    "            self.alphas = np.zeros((self.k, self.m_train))\n",
    "            \n",
    "        elif mode == \"ovo\":\n",
    "            \n",
    "            # creates all combinations of classes which will be used to design indivitual binary classifiers\n",
    "            self.classifiers = list(combinations(self.classes, 2))\n",
    "            \n",
    "            # initialize params\n",
    "            self.alphas = [] # This will be a list of (m_subset, ) vectors where m_subset is the number of examples in each binary classifier's data subset\n",
    "            self.classifier_K = [] # This will be a list of (m_subset, m_subset) matrices \n",
    "            self.classifier_y = [] # This will be a list of (m_subset, ) vectors\n",
    "            self.train_K = [] # This will be a list of (m_subset, m_train) matrices\n",
    "            self.train_y = train_y # This will be the original train labels\n",
    "            self.test_K = [] # This will be a list of (m_subset, m_test) matrices\n",
    "            self.test_y = test_y # This will be the original test labels\n",
    "            \n",
    "            for i, classifier in enumerate(self.classifiers):\n",
    "                \n",
    "                # Extract class values\n",
    "                class_1 = classifier[0]\n",
    "                class_2 = classifier[1]\n",
    "                \n",
    "                # Create subset\n",
    "                examples_subset, labels_subset = create_subsets(train_x, train_y, classifier)\n",
    "                \n",
    "                # Create sub-kernels for each classifier\n",
    "                classifier_kernel = cp.asnumpy(kernel_func(cp.asarray(examples_subset), cp.asarray(examples_subset)))\n",
    "                \n",
    "                # Create train kernels\n",
    "                train_kernel = cp.asnumpy(kernel_func(cp.asarray(examples_subset), cp.asarray(train_x)))\n",
    "                \n",
    "                # Create test kernel\n",
    "                test_kernel = cp.asnumpy(kernel_func(cp.asarray(examples_subset), cp.asarray(test_x)))\n",
    "                \n",
    "                # Save kernels\n",
    "                self.classifier_K.append(classifier_kernel)\n",
    "                self.train_K.append(train_kernel)\n",
    "                self.test_K.append(test_kernel)\n",
    "                \n",
    "                # Convert labels to {+1, -1} encoding.\n",
    "                # The first class is labelled as +1 while second is -1.\n",
    "                # i.e for a binary classifier of \"0 vs 1\", 0 = +1 and 1 = -1\n",
    "                labels_subset = np.where(labels_subset == class_1, 1, -1)\n",
    "                self.classifier_y.append(labels_subset)\n",
    "                \n",
    "                # Initialize weights as 0s\n",
    "                self.alphas.append(np.zeros(len(examples_subset)))\n",
    "\n",
    "        else:\n",
    "            raise Exception(\"mode has to be either \\\"ova\\\" for One-vs-All or \\\"ovo\\\" for One-vs-One \")\n",
    "        \n",
    "        \n",
    "    def fit(self, epsilon=0.001, tolerance=5, max_epochs=20, verbose=True):\n",
    "        \n",
    "        \"\"\" Fit model based on the selected mode of training \"\"\"\n",
    "        \n",
    "        if self.mode == \"ova\":\n",
    "            self.fit_ova(epsilon, tolerance, max_epochs, verbose)\n",
    "        elif self.mode == \"ovo\":\n",
    "            self.fit_ovo(epsilon, tolerance, max_epochs, verbose)\n",
    "        \n",
    "        \n",
    "    def fit_ova(self, epsilon=0.001, tolerance=5, max_epochs=30, verbose=True):\n",
    "\n",
    "        \"\"\" Fit model on the training set using one-vs-all approach \"\"\"\n",
    "        \n",
    "        # Stores the number of epochs over which the train error difference < epsilon\n",
    "        eta = 0\n",
    "        \n",
    "        # Log start time\n",
    "        s = time.time()\n",
    "\n",
    "        # Iterate over epochs\n",
    "        for epoch in range(max_epochs):\n",
    "            \n",
    "            if verbose:\n",
    "                print(\"Epoch {}/{}\".format(epoch+1, max_epochs), end=\" : \")\n",
    "            \n",
    "            # Record the old training error before learning\n",
    "            train_error_old = self.evaluate(self.train_K, self.train_y)\n",
    "\n",
    "            # Process each example online\n",
    "            for t in range(self.m_train):\n",
    "\n",
    "                # Recieve t-th input\n",
    "                input_t = self.train_K[:, t]\n",
    "\n",
    "                # Predict step - generate predictions on all K classes simultaneously - will produce a Kx1 vector\n",
    "                preds = self.alphas @ input_t\n",
    "                assert preds.shape == (self.k, )\n",
    "                \n",
    "                # Recieve true label\n",
    "                label_t = self.train_y[t]\n",
    "\n",
    "                # Update step\n",
    "                y = np.where(label_t == np.arange(0, self.k), 1, -1)\n",
    "                idxs = np.where(y*preds <= 0)\n",
    "                self.alphas[idxs, t] += y[idxs]\n",
    "\n",
    "            # Compute train error and use it to monitor convergence\n",
    "            train_error = self.evaluate(self.train_K, self.train_y)\n",
    "            difference = train_error - train_error_old\n",
    "            \n",
    "            # Output logs\n",
    "            if verbose:\n",
    "                print(\"Training error = {:.5f}\".format(train_error))\n",
    "            \n",
    "            #--------------------EARLY STOPPING CRITERIA---------------------#\n",
    "            \n",
    "            #If the difference in accuracy is lesser than epsilon and this occurs over lesser than 3 epochs, then add 1 to k\n",
    "            if (np.abs(difference) <= epsilon and eta < tolerance):\n",
    "                eta += 1\n",
    "\n",
    "            #If the difference is lesser than epsilon but k=3, then we have converged and can end training.\n",
    "            if (np.abs(difference) <= epsilon and eta == tolerance):\n",
    "                if verbose:\n",
    "                    print(\"Minimal change in training accuracy. Training completed early.\")\n",
    "                break\n",
    "\n",
    "            #Suppose we saw a small difference over atleast 1 (but < 5) consecutive epochs but then suddenly a large jump in accuracy occurs, then we want to reset k=0 and restart the counting.\n",
    "            #Note that if we saw a large difference and k = 0, that's just the same as going into the while loop and running another epoch again.\n",
    "            if (np.abs(difference) > epsilon and eta >= 1):\n",
    "                eta = 0\n",
    "        \n",
    "        # Log end time\n",
    "        e = time.time()\n",
    "                \n",
    "        if verbose:\n",
    "            print(\"TOTAL TIME TAKEN = {:.4f} seconds\".format(e - s))\n",
    "    \n",
    "    def fit_ovo(self, epsilon=0.001, tolerance=5, max_epochs=30, verbose=True):\n",
    "        \n",
    "        \"\"\" Fit indivitual classifiers as part of the one-vs-one approach \"\"\"\n",
    "        \n",
    "        # Log start time\n",
    "        s = time.time()\n",
    "        \n",
    "        for i, classifier in enumerate(self.classifiers):\n",
    "            \n",
    "            # Stores the number of epochs over which the train error difference < epsilon\n",
    "            eta = 0\n",
    "            \n",
    "            # Extract true class labels\n",
    "            class_1 = classifier[0]\n",
    "            class_2 = classifier[1]\n",
    "            \n",
    "            if verbose:\n",
    "                print(\"\\nTraining classifier {} : {} vs {}\".format(i+1, class_1, class_2))\n",
    "\n",
    "            # Iterate over epochs\n",
    "            for epoch in range(max_epochs):\n",
    "\n",
    "                if verbose:\n",
    "                    print(\"Epoch {}/{}\".format(epoch+1, max_epochs), end=\" : \")\n",
    "\n",
    "                # Record the old training error before learning\n",
    "                train_error_old = self.evaluate_binary_classifier(self.alphas[i], self.classifier_K[i], self.classifier_y[i])\n",
    "\n",
    "                # Process each example online\n",
    "                for t in range(self.classifier_K[i].shape[0]):\n",
    "\n",
    "                    # Recieve t-th input\n",
    "                    input_t = self.classifier_K[i][:, t]\n",
    "\n",
    "                    # Predict step \n",
    "                    preds = self.alphas[i] @ input_t\n",
    "\n",
    "                    # Recieve true label\n",
    "                    y = self.classifier_y[i][t]\n",
    "\n",
    "                    # Update step\n",
    "                    if y*preds <= 0:\n",
    "                        self.alphas[i][t] += y\n",
    "\n",
    "                # Compute train error and use it to monitor convergence\n",
    "                train_error = self.evaluate_binary_classifier(self.alphas[i], self.classifier_K[i], self.classifier_y[i])\n",
    "                difference = train_error - train_error_old\n",
    "\n",
    "                # Output logs\n",
    "                if verbose:\n",
    "                    print(\"Training error = {:.5f}\".format(train_error))\n",
    "\n",
    "                #--------------------EARLY STOPPING CRITERIA---------------------#\n",
    "\n",
    "                #If the difference in accuracy is lesser than epsilon and this occurs over lesser than 3 epochs, then add 1 to k\n",
    "                if (np.abs(difference) <= epsilon and eta < tolerance):\n",
    "                    eta += 1\n",
    "\n",
    "                #If the difference is lesser than epsilon but k=3, then we have converged and can end training.\n",
    "                if (np.abs(difference) <= epsilon and eta == tolerance):\n",
    "                    if verbose:\n",
    "                        print(\"Minimal change in training accuracy. Training completed early.\")\n",
    "                    break\n",
    "\n",
    "                #Suppose we saw a small difference over atleast 1 (but < 5) consecutive epochs but then suddenly a large jump in accuracy occurs, then we want to reset k=0 and restart the counting.\n",
    "                #Note that if we saw a large difference and k = 0, that's just the same as going into the while loop and running another epoch again.\n",
    "                if (np.abs(difference) > epsilon and eta >= 1):\n",
    "                    eta = 0\n",
    "                \n",
    "        # Log end time\n",
    "        e = time.time()\n",
    "            \n",
    "        if verbose:\n",
    "            print(\"TOTAL TIME TAKEN = {:.4f}s\".format(e - s))\n",
    "            \n",
    "    def predict(self, K, labels):\n",
    "        \n",
    "        \"\"\" Generate predictions for new examples \"\"\"\n",
    "        \n",
    "        if self.mode == \"ova\":\n",
    "            preds = self.predict_ova(K, labels)\n",
    "        elif self.mode == \"ovo\":\n",
    "            preds = self.predict_ovo(K, labels)\n",
    "            \n",
    "        return preds\n",
    "\n",
    "    def predict_ova(self, K, labels):\n",
    "\n",
    "        \"\"\" Compute predictions using trained model - for one vs all strategy\"\"\"\n",
    "        \n",
    "        # Generate predictions\n",
    "        preds = cp.asarray(self.alphas) @ cp.asarray(K)\n",
    "\n",
    "        # Set index of most confident entry as our predicted class\n",
    "        preds = np.argmax(cp.asnumpy(preds), 0)\n",
    "\n",
    "        return preds\n",
    "    \n",
    "    def predict_ovo(self, K, labels):\n",
    "        \n",
    "        \"\"\" Compute predictions using trained model - for one vs one strategy \"\"\"\n",
    "        \n",
    "        # Stores the classes predicted by the k(k-1)/2 indivitual binary classifiers in each row.\n",
    "        # cols correspond to number of test examples\n",
    "        class_matrix = np.zeros((len(self.classifiers), K[0].shape[1]))\n",
    "        \n",
    "        for i, classifier in enumerate(self.classifiers):\n",
    "            \n",
    "            # Extract class information\n",
    "            class_1 = classifier[0]\n",
    "            class_2 = classifier[1]\n",
    "            \n",
    "            # Generate predictions\n",
    "            pred = cp.asarray(self.alphas[i]) @ cp.asarray(K[i])\n",
    "            \n",
    "            # Encode as +1 or -1\n",
    "            pred = np.where(cp.asnumpy(pred) > 0, 1, -1)\n",
    "            \n",
    "            # Convert back to the original k class values\n",
    "            pred = np.where(pred == 1, class_1, class_2)\n",
    "            \n",
    "            # Update class matrix\n",
    "            class_matrix[i, :] = pred\n",
    "            \n",
    "        # Convert to a final predictions vector where each element is the class with maximum vote\n",
    "        class_predictions = mode(class_matrix)[0].squeeze()\n",
    "        \n",
    "        return class_predictions\n",
    "    \n",
    "    def evaluate(self, K, labels):\n",
    "        \n",
    "        \"\"\" Evaluate the model by computing error on the new points \"\"\"\n",
    "        \n",
    "        # Compute predictions\n",
    "        preds = self.predict(K, labels)\n",
    "        \n",
    "        # Find mistakes\n",
    "        mistakes = np.where(preds != labels, 1.0, 0.0)\n",
    "\n",
    "        # Error is the mean of the mistakes\n",
    "        error = np.mean(mistakes)\n",
    "        \n",
    "        return error\n",
    "    \n",
    "    def evaluate_binary_classifier(self, alphas, K, labels):\n",
    "        \n",
    "        \"\"\" Evaluates an indivitual binary classifier - required for monitoring indivitual classifiers in one vs one strategy \"\"\"\n",
    "    \n",
    "        # Generate predictions\n",
    "        preds = cp.asarray(alphas) @ cp.asarray(K)\n",
    "        \n",
    "        # Convert to {+1, -1} encoding\n",
    "        preds = np.where(cp.asnumpy(preds) > 0, 1, -1)\n",
    "        \n",
    "        # Count mistakes\n",
    "        mistakes = np.where(preds != labels, 1.0, 0.0)\n",
    "        error = np.mean(mistakes)\n",
    "        \n",
    "        return error"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "id": "3acjRLMQxgOh"
   },
   "source": [
    "## Perform a single run on the full-dataset (just to check if model running smoothly)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true,
    "id": "LbyFIrnh07W9"
   },
   "outputs": [],
   "source": [
    "# Split data into train and test set\n",
    "np.random.seed(3412)\n",
    "train_x, train_y, test_x, test_y, _ = split_data(inputs=x, targets=y, test_proportion=0.2, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "hidden": true,
    "id": "nsAtRJk22iAq",
    "outputId": "837fc90a-27ab-45eb-90e9-c26db9b5caa9",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Setup perclocal variable 'time' referenced before assignmenteptron model\n",
    "model = Kernel_Perceptron(train_x, train_y, test_x, test_y, gaussian_kernel_matrix, mode=\"ova\")\n",
    "model.fit(epsilon=0.001, tolerance=5, max_epochs=20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "id": "qMNjDFYIQmgv"
   },
   "source": [
    "## Creating function for performing multiple runs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "Given the model runs pretty smoothly, we can now create the functionality to perform multiple runs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true,
    "id": "V5NQDjA2ZxA_"
   },
   "outputs": [],
   "source": [
    "def perform_multiple_runs(runs,\n",
    "                          param_values,\n",
    "                          kernel_func,\n",
    "                          mode=\"ova\",\n",
    "                          save_results=False,\n",
    "                          path_to_results=\"\",\n",
    "                          seed=None,\n",
    "                          verbose=True):\n",
    "    \n",
    "    \"\"\"\n",
    "    Performs multiple runs of training with different parameter values.\n",
    "    \n",
    "    Args\n",
    "    ----\n",
    "    runs : number of runs to perform\n",
    "    param_values : list of parameter values to train on\n",
    "    kernel_func : kernel matrix computing function\n",
    "    mode : set to \"ova\" or \"ovo\"\n",
    "    save_results : Set true to save the results\n",
    "    path_to_results : Saves results to provided filepath\n",
    "    seed : to ensure reproducibility of results\n",
    "    verbose : prints outputs while training\n",
    "    \n",
    "    Returns\n",
    "    -------\n",
    "    results_df : pandas dataframe of the final results from experiment.\n",
    "    \"\"\"\n",
    "\n",
    "    # Save results here\n",
    "    results = np.zeros((4, len(param_values)))\n",
    "\n",
    "    # Set a random seed for reproducibility of results\n",
    "    np.random.seed(seed)\n",
    "\n",
    "    for i, p in enumerate(param_values):\n",
    "\n",
    "        if verbose:\n",
    "            print(\"\\nparamter value = {}\".format(p))\n",
    "            \n",
    "        # Create a version of the kernel function with hyperparameter d\n",
    "        kernel = lambda X1, X2: kernel_func(X1, X2, p)\n",
    "\n",
    "        # Save errors for all runs here\n",
    "        train_errors = []\n",
    "        test_errors = []\n",
    "\n",
    "        for run in range(runs):\n",
    "            \n",
    "            if verbose:\n",
    "                print(\">> Run {}\".format(run+1))\n",
    "\n",
    "            # generate random train-test split\n",
    "            train_x, train_y, test_x, test_y, _ = split_data(inputs=x, targets=y, test_proportion=0.20, shuffle=True)\n",
    "            \n",
    "            # Load a fresh model on every run\n",
    "            model = Kernel_Perceptron(train_x, train_y, test_x, test_y, kernel, mode=mode)\n",
    "            \n",
    "            # Fit model on training data\n",
    "            model.fit(epsilon=0.001, tolerance=5, max_epochs=20, verbose=False)\n",
    "\n",
    "            # Evaluate train and test error for that d\n",
    "            train_error = model.evaluate(model.train_K, model.train_y)\n",
    "            test_error = model.evaluate(model.test_K, model.test_y)\n",
    "            train_errors.append(np.mean(train_error))\n",
    "            test_errors.append(np.mean(test_error))\n",
    "\n",
    "        # Convert to numpy arrays\n",
    "        train_errors = np.array(train_errors)\n",
    "        test_errors = np.array(test_errors)\n",
    "\n",
    "        # Save results\n",
    "        results[0, i] = np.mean(train_errors)\n",
    "        results[1, i] = np.std(train_errors)\n",
    "        results[2, i] = np.mean(test_errors)\n",
    "        results[3, i] = np.std(test_errors)\n",
    "\n",
    "    # convert matrix to a pandas dataframe for easier visualization\n",
    "    results_df = pd.DataFrame(results, columns=[\"parameter value = {}\".format(p) for p in param_values], index=[\"Mean Train Error\", \"STD Train Error\", \"Mean Test Error\", \"STD Test Error\"])\n",
    "\n",
    "    if save_results:\n",
    "        results_df.to_csv(results_dir+path_to_results, header=True, index=True)\n",
    "\n",
    "    return results_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Performing for 20 runs on dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "hidden": true,
    "id": "azS84urP9uLm",
    "outputId": "a2e0cf92-438d-4e4d-e2eb-66f020c4b797",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Range of d_values\n",
    "d_values = np.arange(1, 8)\n",
    "\n",
    "# Run for 20 runs\n",
    "results = perform_multiple_runs(runs=20,\n",
    "                                mode=\"ova\",\n",
    "                                param_values=d_values,\n",
    "                                kernel_func=polynomial_kernel_matrix,\n",
    "                                save_results=True,\n",
    "                                path_to_results=\"q1_results.csv\",\n",
    "                                seed=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 173
    },
    "hidden": true,
    "id": "LEDwqCIrkNcb",
    "outputId": "4e1c20aa-0e69-406a-86a5-18dfb18546b2",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Check results\n",
    "results = pd.read_csv(results_dir+\"q1_results.csv\")\n",
    "results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "id": "1i7AtXAuQzNT"
   },
   "source": [
    "## Creating function for performing k-Fold Cross validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "def create_confusion_matrix(K, alphas, labels):\n",
    "    \n",
    "    \"\"\" \n",
    "    Creates confusion matrices for a given set of data \n",
    "    \n",
    "    Args\n",
    "    ----\n",
    "    K : Kernel matrix between the training data and the test points. Should be of shape (m, l) where m is the number of training points and l is the number of test points\n",
    "    alphas : dual solution vector for the model. Should be of shape (m, )\n",
    "    labels : true labels of shape (l, )\n",
    "    \n",
    "    Returns\n",
    "    -------\n",
    "    matrix - confusion matrix of shape (number of classes, number of classes)\n",
    "    \"\"\"\n",
    "    \n",
    "    # Initialize CM\n",
    "    matrix = np.zeros((10, 10))\n",
    "    \n",
    "    # Generate predictions\n",
    "    preds = cp.asarray(alphas) @ cp.asarray(K)\n",
    "    \n",
    "    # Set index of most confident entry as our predicted class\n",
    "    preds = np.argmax(cp.asnumpy(preds), 0)\n",
    "    \n",
    "    # Update cells in CM\n",
    "    for i in range(K.shape[1]):\n",
    "        if preds[i] != labels[i]:\n",
    "            matrix[int(preds[i]), int(labels[i])] += 1\n",
    "\n",
    "    return matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true,
    "id": "7md_yyiiclqP"
   },
   "outputs": [],
   "source": [
    "def perform_kfoldCV(k, x, y, mode, hparam, kernel_func, shuffle=True, verbose=True):\n",
    "    \n",
    "    \"\"\" \n",
    "    Performs k-fold cross validation for a given parameter\n",
    "    \n",
    "    Args\n",
    "    ----\n",
    "    k - number of folds of CV to perform\n",
    "    x - training data\n",
    "    y - training labels\n",
    "    mode - ova or ovo\n",
    "    hparam - parameter to use for kernel\n",
    "    kernel_func - kernel matrix computing function\n",
    "    shuffle - shuffles data before performing k-fold CV\n",
    "    verbose - prints output logs while running\n",
    "    \n",
    "    Returns\n",
    "    -------\n",
    "    cv_error_over_folds - the cross valiation error for each fold of cross validation.\n",
    "    \"\"\"\n",
    "\n",
    "    # Extract dimensions\n",
    "    m, n = x.shape\n",
    "    \n",
    "    # Configure kernel function to have a specific d value  \n",
    "    kernel = lambda X1, X2: kernel_func(X1, X2, hparam)\n",
    "\n",
    "    # Shuffle dataset randomly for splitting into groups\n",
    "    if shuffle:\n",
    "        perm = np.random.permutation(m)\n",
    "        x_shuffled = x[perm, :]\n",
    "        y_shuffled = y[perm]\n",
    "    else:\n",
    "        x_shuffled = x\n",
    "        y_shuffled = y\n",
    "\n",
    "    # Split data into k-groups\n",
    "    x_groups = np.array_split(x_shuffled, k)\n",
    "    y_groups = np.array_split(y_shuffled, k)\n",
    "\n",
    "    # Stores the mean CV error over all folds of CV\n",
    "    cv_error_over_folds = 0\n",
    "\n",
    "    for i in range(len(x_groups)):\n",
    "        \n",
    "        if verbose:\n",
    "            print(\">>>> Cross-validation Fold {}\".format(i+1), end=\"...\")\n",
    "\n",
    "        # Use the selected group as \"validation\" set\n",
    "        val_inputs, val_labels = x_groups[i], y_groups[i]\n",
    "\n",
    "        # Use rest of groups as training set\n",
    "        train_inputs = np.vstack([x_groups[j] for j in range(len(x_groups)) if j != i])\n",
    "        train_labels = np.concatenate([y_groups[j] for j in range(len(x_groups)) if j != i])\n",
    "\n",
    "        #-------------------------TRAIN MODEL--------------------------#\n",
    "\n",
    "        # Load the model\n",
    "        model = Kernel_Perceptron(train_inputs, train_labels, val_inputs, val_labels, kernel, mode=mode)\n",
    "\n",
    "        # Fit model to our training set\n",
    "        model.fit(epsilon=0.001, tolerance=5, max_epochs=20, verbose=False)\n",
    "\n",
    "        # Evaluate on validation data - the \"test\" K and y refer to the val set\n",
    "        cv_error_over_folds += np.mean(model.evaluate(model.test_K, model.test_y))\n",
    "\n",
    "        if verbose:\n",
    "            print(\"Done!\")\n",
    "\n",
    "    #Average the errors\n",
    "    cv_error_over_folds /= k\n",
    "\n",
    "    return cv_error_over_folds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true,
    "id": "-0fuwk0-EolL"
   },
   "outputs": [],
   "source": [
    "def perform_multiple_runs_with_kFold(param_values,\n",
    "                                     kernel_func,\n",
    "                                     mode=\"ova\",\n",
    "                                     k=5,\n",
    "                                     runs=20,\n",
    "                                     seed=None,\n",
    "                                     verbose=True,\n",
    "                                     save_results=True,\n",
    "                                     path_to_results=\"\",\n",
    "                                     create_cm=True,\n",
    "                                     path_to_cm=\"\",\n",
    "                                     save_mistake_counts=False,\n",
    "                                     path_to_mistake_counts=\"\"):\n",
    "    \n",
    "    \"\"\"\n",
    "    Performs multiple runs of training with different parameter values.\n",
    "    \n",
    "    Args\n",
    "    ----\n",
    "    param_values : list of parameter values to train on\n",
    "    kernel_func : kernel matrix computing function\n",
    "    mode : set to \"ova\" or \"ovo\"\n",
    "    k : number of folds of CV\n",
    "    runs : number of runs to perform\n",
    "    seed : to ensure reproducibility of results\n",
    "    verbose : prints outputs while training\n",
    "    save_results : Set true to save the results\n",
    "    path_to_results : Saves results to provided filepath\n",
    "    create_cm : Creates confusion matrices for each run of cross validation experiment\n",
    "    path_to_cm : saves confusion matrices as numpy arrays to the given path\n",
    "    save_mistake_counts : counts the number of mistakes for a specific example on each round of CV\n",
    "    path_to_mistake_counts : saves mistake counts to provided filepath\n",
    "    \n",
    "    Returns\n",
    "    -------\n",
    "    results_df : pandas dataframe of the final results from experiment.\n",
    "    confusion_matrices : numpy arrays for the confusion matrices\n",
    "    mistake_counts : vector containing counts of mistakes for each record in the dataset.\n",
    "    \"\"\"\n",
    "\n",
    "    # Set a random seed for reproducibility of results\n",
    "    np.random.seed(seed)\n",
    "\n",
    "    # Results will be stored here\n",
    "    results = np.zeros((runs, 2))\n",
    "\n",
    "    # Confusion matrices will be created\n",
    "    if create_cm:\n",
    "        confusion_matrices = np.zeros((runs, 10, 10))\n",
    "    else:\n",
    "        confusion_matrices = None\n",
    "    \n",
    "    # mistake counts for each image - required to identify which images are hardest to classify\n",
    "    if save_mistake_counts:\n",
    "        mistake_counts = np.zeros(x.shape[0])\n",
    "    else:\n",
    "        mistake_counts = None\n",
    "\n",
    "    for run in range(runs):\n",
    "\n",
    "        if verbose:\n",
    "            print(\"\\nRun {}\".format(run+1))\n",
    "\n",
    "        # I will record the errors for a single run in this vector\n",
    "        mean_cv_errors = np.zeros(len(param_values))\n",
    "\n",
    "        # generate random train-test split\n",
    "        train_x, train_y, test_x, test_y, cache = split_data(inputs=x, targets=y, test_proportion=0.20, shuffle=True)\n",
    "\n",
    "        # extract the indices for the examples in the original dataset that are being trained/tested on\n",
    "        train_idxs, test_idxs = cache\n",
    "\n",
    "        #----------------------------PERFORM k-FOLD CV over parameters-------------------------------#\n",
    "\n",
    "        for i, p in enumerate(param_values):\n",
    "\n",
    "            if verbose:\n",
    "                print(\"\\n>> parameter = {}\".format(p))\n",
    "\n",
    "            # Perform k-fold CV on the training set\n",
    "            mean_cv_errors[i] = perform_kfoldCV(k, train_x, train_y, mode, p, kernel_func, verbose=verbose)\n",
    "\n",
    "        #-----FIND BEST HYPERPARAM VALUE AND TRAIN THE WHOLE DATASET WITH THAT-----#\n",
    "\n",
    "        # The d that has lowest mean cv error is the optimal d\n",
    "        param_star = param_values[np.argmin(mean_cv_errors)]\n",
    "    \n",
    "        print(\"\\nbest parameter value = {}\".format(param_star))\n",
    "\n",
    "        # Set the kernel to have this d*\n",
    "        kernel = lambda X1, X2: kernel_func(X1, X2, param_star)\n",
    "\n",
    "        # Fit model on training set\n",
    "        model = Kernel_Perceptron(train_x, train_y, test_x, test_y, kernel, mode=mode)\n",
    "        model.fit(epsilon=0.001, tolerance=5, max_epochs=20, verbose=True)\n",
    "\n",
    "        # Evaluate on train and test set\n",
    "        train_error = model.evaluate(model.train_K, model.train_y)\n",
    "        test_error = model.evaluate(model.test_K, model.test_y)\n",
    "\n",
    "        if save_mistake_counts:\n",
    "            # Select the train and test indices that got misclassified\n",
    "            train_misclassifieds = train_idxs[np.where(train_error == 1.0)]\n",
    "            test_misclassifieds = test_idxs[np.where(test_error == 1.0)]\n",
    "\n",
    "            # Update the counts in the mistake vector for these indices\n",
    "            mistake_counts[train_misclassifieds] += 1\n",
    "            mistake_counts[test_misclassifieds] += 1\n",
    "\n",
    "        # Record results\n",
    "        results[run, 0] = param_star\n",
    "        results[run, 1] = np.mean(test_error)\n",
    "    \n",
    "        if create_cm:\n",
    "            # Record confusion matrices on test set\n",
    "            confusion_matrices[run] = create_confusion_matrix(model.test_K, model.alphas, model.test_y)\n",
    "\n",
    "        # Convert results matrix to pandas dataframe\n",
    "        results_df = pd.DataFrame(results, columns=[\"best parameter\", \"Test Error\"], index=[\"Run {}\".format(run+1) for run in range(runs)])\n",
    "\n",
    "        if save_results:\n",
    "            # Save errors\n",
    "            results_df.to_csv(results_dir+path_to_results, header=True, index=True)\n",
    "        if create_cm:\n",
    "            # Save confusion matrices\n",
    "            np.save(results_dir+path_to_cm, confusion_matrices)\n",
    "        if save_mistake_counts:\n",
    "            # Save vector of the mistake counts\n",
    "            np.save(results_dir+path_to_mistake_counts, mistake_counts)\n",
    "\n",
    "    return results_df, confusion_matrices, mistake_counts"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Performing 5-fold Cross validation for 20 runs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "hidden": true,
    "id": "VE48_FX1uF9V",
    "outputId": "78469122-a64e-4ce0-b822-c4f686eeeac5",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Range of d values\n",
    "d_values = np.arange(1, 8)\n",
    "\n",
    "# Run model for multiple runs with cross validation\n",
    "results_df, confusion_matrices, mistake_counts = perform_multiple_runs_with_kFold(param_values=d_values,\n",
    "                                                                                  kernel_func=polynomial_kernel_matrix,\n",
    "                                                                                  mode=\"ova\",\n",
    "                                                                                  k=5,\n",
    "                                                                                  runs=20,\n",
    "                                                                                  seed=231,\n",
    "                                                                                  path_to_results=\"q2_crossval_v2.csv\",\n",
    "                                                                                  create_cm=True,\n",
    "                                                                                  path_to_cm=\"q3_confusions_v2\",\n",
    "                                                                                  save_mistake_counts=True,\n",
    "                                                                                  path_to_mistake_counts=\"mistakes\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true,
    "id": "gTX2EdSTOIUq",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Results\n",
    "results_df = pd.read_csv(results_dir+\"q2_crossval.csv\")\n",
    "results_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "print(\"Mean d* \\u00B1 STD = {} \\u00B1 {}\".format(results_df[\"best parameter\"].mean(), results_df[\"best parameter\"].std()))\n",
    "print(\"Mean Test Error \\u00B1 STD = {} \\u00B1 {}\".format(results_df[\"Test Error\"].mean(), results_df[\"Test Error\"].std()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "id": "886M8Ih646wB"
   },
   "source": [
    "## Confusion Matrices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Confusion matrices\n",
    "confusion_matrices = np.load(results_dir+\"q3_confusions.npy\")\n",
    "means = np.mean(confusion_matrices, axis=0)\n",
    "stds = np.std(confusion_matrices, axis=0)\n",
    "\n",
    "annotations = np.empty(means.shape, dtype=object)\n",
    "\n",
    "for i in range(annotations.shape[0]):\n",
    "    for j in range(annotations.shape[1]):\n",
    "        if i != j:\n",
    "            annotations[i, j] = \"{:.4f} \\u00B1 {:.2f}\".format(means[i, j], stds[i, j])\n",
    "        else:\n",
    "            annotations[i, j] = \"0\"\n",
    "\n",
    "plt.figure(figsize=(20, 8))\n",
    "sns.heatmap(means, annot=annotations, fmt=\"\", annot_kws={\"fontweight\":\"bold\"}, linewidths=1)\n",
    "plt.ylabel(\"Predicted Class\")\n",
    "plt.xlabel(\"True Class\")\n",
    "plt.savefig(results_dir+\"q3_confusion_matrix.jpg\")\n",
    "plt.show()\n",
    "\n",
    "# plt.figure(figsize=(12, 52))\n",
    "# for i in range(len(confusion_matrices)):\n",
    "#     plt.subplot(10, 2, i+1)\n",
    "#     ax = sns.heatmap(confusion_matrices[i], annot=True)\n",
    "#     plt.ylabel(\"Predicted Class\")\n",
    "#     plt.xlabel(\"True Class\")\n",
    "#     plt.title(\"Run {}, d* = {}\".format(i+1, results_df[\"best parameter\"][i]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Find Hardest to Predict images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# load mistake counts file\n",
    "mistakes = np.load(results_dir+\"q4_mistakes.npy\")\n",
    "\n",
    "# store indices of top 5 hardest to predict images\n",
    "idxs = np.argsort(mistakes)[-5:]\n",
    "\n",
    "# Obtain the examples and their corresponding labels\n",
    "five_hardest_images = x[idxs].reshape(5, 16, 16)\n",
    "labels = y[idxs]\n",
    "\n",
    "# Plot results\n",
    "plt.figure(figsize=(18, 6))\n",
    "for i in range(5):\n",
    "    plt.subplot(1, 5, i+1)\n",
    "    plt.imshow(five_hardest_images[i], plt.cm.gray)\n",
    "    plt.title(\"Image Label = {}\".format(labels[i]))\n",
    "# plt.savefig(results_dir+\"hard2predict.png\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Repeat Experiments with Gaussian Kernel"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "### Perform 20 runs with Gaussian Kernel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Range of c_values\n",
    "c_values = [2**x for x in range(-10, -3)]\n",
    "\n",
    "# Run for 20 runs\n",
    "results = perform_multiple_runs(runs=20,\n",
    "                                mode=\"ova\",\n",
    "                                param_values=c_values,\n",
    "                                kernel_func=gaussian_kernel_matrix,\n",
    "                                save_results=True,\n",
    "                                path_to_results=\"q5_results.csv\",\n",
    "                                seed=1796)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# Check results\n",
    "results = pd.read_csv(results_dir+\"q5_results.csv\")\n",
    "results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "### Perform 20 runs with 5-fold cross-validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Range of c_values\n",
    "c_values = [2**x for x in range(-10, -3)]\n",
    "\n",
    "# Run model for multiple runs with cross validation\n",
    "results_df, _, _ = perform_multiple_runs_with_kFold(param_values=c_values,\n",
    "                                                    kernel_func=gaussian_kernel_matrix,\n",
    "                                                    mode=\"ova\",\n",
    "                                                    k=5,\n",
    "                                                    runs=20,\n",
    "                                                    seed=231,\n",
    "                                                    path_to_results=\"q5_crossval.csv\",\n",
    "                                                    create_cm=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# Results\n",
    "results_df = pd.read_csv(results_dir+\"q5_crossval.csv\")\n",
    "results_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "print(\"\\nMean d* \\u00B1 STD = {} \\u00B1 {}\".format(results_df[\"best parameter\"].mean(), results_df[\"best parameter\"].std()))\n",
    "print(\"Mean Test Error \\u00B1 STD = {} \\u00B1 {}\".format(results_df[\"Test Error\"].mean(), results_df[\"Test Error\"].std()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Alternative Method for Multi-class Classification: One vs One"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "### Perform 20 runs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Range of c_values\n",
    "d_values = np.arange(1, 8)\n",
    "\n",
    "# Run for 20 runs\n",
    "results = perform_multiple_runs(runs=20,\n",
    "                                mode=\"ovo\",\n",
    "                                param_values=d_values,\n",
    "                                kernel_func=polynomial_kernel_matrix,\n",
    "                                save_results=True,\n",
    "                                path_to_results=\"q6_results.csv\",\n",
    "                                seed=130399)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Check results\n",
    "results = pd.read_csv(results_dir+\"q6_results.csv\")\n",
    "results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "### Peform 20 runs of 5-fold cross-validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Range of d values\n",
    "d_values = np.arange(1, 8)\n",
    "\n",
    "# Run model for multiple runs with cross validation\n",
    "results_df, confusion_matrices = perform_multiple_runs_with_kFold(param_values=d_values,\n",
    "                                                                  kernel_func=polynomial_kernel_matrix,\n",
    "                                                                  mode=\"ovo\",\n",
    "                                                                  k=5,\n",
    "                                                                  runs=20,\n",
    "                                                                  seed=231,\n",
    "                                                                  path_to_results=\"q6_crossval.csv\",\n",
    "                                                                  create_cm=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# Results\n",
    "results_df = pd.read_csv(results_dir+\"q6_crossval.csv\")\n",
    "results_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "print(\"\\nMean d* \\u00B1 STD = {} \\u00B1 {}\".format(results_df[\"best parameter\"].mean(), results_df[\"best parameter\"].std()))\n",
    "print(\"Mean Test Error \\u00B1 STD = {} \\u00B1 {}\".format(results_df[\"Test Error\"].mean(), results_df[\"Test Error\"].std()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Implementing 2 other Classification Algorithms"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "### K-Nearest Neighbors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "class KNN():\n",
    "    \n",
    "    \"\"\" \n",
    "    Manual Implementation of KNN algorithm \n",
    "    \n",
    "    Args\n",
    "    ----\n",
    "    data - training data of shape (m, n) where m is the number of training examples and n is number of features\n",
    "    labels - vector of training labels of shape (m, )\n",
    "    k - number of neighbors to predict with.\n",
    "    \n",
    "    Methods\n",
    "    -------\n",
    "    calc_distance - computes euclidean distance between two points \n",
    "    get_nearest_neighbors - finds the indices in data of the k nearest neighbors\n",
    "    predict - predicts the class of a test point based on k nearest neighbors\n",
    "    \n",
    "    \"\"\"\n",
    "    \n",
    "    def __init__(self, data, labels, k):\n",
    "        self.k = k\n",
    "        self.data = data\n",
    "        self.labels = labels\n",
    "        \n",
    "    def calc_distance(self, point1, point2):\n",
    "        \"\"\" Calculates the euclidean distance between two points \"\"\"\n",
    "        distance = cp.linalg.norm(cp.asarray(point1) - cp.asarray(point2), axis=1)\n",
    "        return cp.sqrt(distance).get()\n",
    "    \n",
    "    def get_nearest_neighbors(self, test_point):\n",
    "        \"\"\" Returns the k nearest neighbors indices \"\"\"\n",
    "        distances = self.calc_distance(self.data, test_point)\n",
    "        nearest_neighbors = np.argsort(distances)[:self.k]\n",
    "        return nearest_neighbors\n",
    "        \n",
    "    def predict(self, test_point):\n",
    "        \"\"\" Predicts the class of a test point based on k nearest neighbors  \"\"\"\n",
    "        nearest_neighbors = self.get_nearest_neighbors(test_point)\n",
    "        nearest_labels = self.labels[nearest_neighbors]\n",
    "        pred = mode(nearest_labels)[0]\n",
    "        return pred"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "### Basic Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "def perform_multiple_runs(runs,\n",
    "                          param_values,\n",
    "                          save_results=False,\n",
    "                          path_to_results=\"\",\n",
    "                          seed=None,\n",
    "                          verbose=True):\n",
    "    \n",
    "    \"\"\"\n",
    "    Performs multiple runs of training with different parameter values.\n",
    "    \n",
    "    Args\n",
    "    ----\n",
    "    runs : number of runs to perform\n",
    "    param_values : list of parameter values to train on\n",
    "    save_results : Set true to save the results\n",
    "    path_to_results : Saves results to provided filepath\n",
    "    seed : to ensure reproducibility of results\n",
    "    verbose : prints outputs while training\n",
    "    \n",
    "    Returns\n",
    "    -------\n",
    "    results_df : pandas dataframe of the final results from experiment.\n",
    "    \"\"\"\n",
    "\n",
    "    # Save results here\n",
    "    results = np.zeros((4, len(param_values)))\n",
    "\n",
    "    # Set a random seed for reproducibility of results\n",
    "    np.random.seed(seed)\n",
    "\n",
    "    for i, p in enumerate(param_values):\n",
    "\n",
    "        if verbose:\n",
    "            print(\"paramter value = {}\".format(p))\n",
    "\n",
    "        # Save errors for all runs here\n",
    "        train_errors = []\n",
    "        test_errors = []\n",
    "\n",
    "        for run in range(runs):\n",
    "            \n",
    "            if verbose:\n",
    "                print(\">> Run {}\".format(run+1), end=\"...\")\n",
    "\n",
    "            # generate random train-test split\n",
    "            train_x, train_y, test_x, test_y, _ = split_data(inputs=x, targets=y, test_proportion=0.20, shuffle=True)\n",
    "            \n",
    "            # log start time\n",
    "            s = time.time()\n",
    "            # Load a fresh model on every run\n",
    "            model = KNN(train_x, train_y, p)\n",
    "            \n",
    "            # Generate predictions for training and testing set\n",
    "            train_preds = np.zeros(train_x.shape[0])\n",
    "            for k in range(len(train_x)):\n",
    "                train_preds[k] = model.predict(train_x[k])\n",
    "            \n",
    "            test_preds = np.zeros(test_x.shape[0])\n",
    "            for l in range(len(test_x)):\n",
    "                test_preds[l] = model.predict(test_x[l])\n",
    "                \n",
    "            # log end time\n",
    "            e = time.time()\n",
    "            print(\"time taken: {:.5f} seconds\".format(e-s))\n",
    "            \n",
    "            # Evaluate train and test error for that d\n",
    "            train_mistakes = np.where(train_preds != train_y, 1.0, 0.0)\n",
    "            test_mistakes = np.where(test_preds != test_y, 1.0, 0.0)\n",
    "            train_errors.append(np.mean(train_mistakes))\n",
    "            test_errors.append(np.mean(test_mistakes))\n",
    "\n",
    "        # Convert to numpy arrays\n",
    "        train_errors = np.array(train_errors)\n",
    "        test_errors = np.array(test_errors)\n",
    "\n",
    "        # Save results\n",
    "        results[0, i] = np.mean(train_errors)\n",
    "        results[1, i] = np.std(train_errors)\n",
    "        results[2, i] = np.mean(test_errors)\n",
    "        results[3, i] = np.std(test_errors)\n",
    "\n",
    "    # convert matrix to a pandas dataframe for easier visualization\n",
    "    results_df = pd.DataFrame(results, columns=[\"parameter value = {}\".format(p) for p in param_values], index=[\"Mean Train Error\", \"STD Train Error\", \"Mean Test Error\", \"STD Test Error\"])\n",
    "\n",
    "    if save_results:\n",
    "        results_df.to_csv(results_dir+path_to_results, header=True, index=True)\n",
    "\n",
    "    return results_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "k_values = 3**np.arange(5)\n",
    "\n",
    "results = perform_multiple_runs(runs=20,\n",
    "                                param_values=k_values,\n",
    "                                save_results=True,\n",
    "                                path_to_results=\"q7_knn_basic_results.csv\",\n",
    "                                seed=130399,\n",
    "                                verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "#Check results\n",
    "results = pd.read_csv(results_dir+\"q7_knn_basic_results.csv\")\n",
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "table = results.values[:, 1:]\n",
    "train_err = [\"{:.6f} \\u00B1 {:.6f}\".format(i, j) for i, j in zip(table[0], table[1])]\n",
    "test_err = [\"{:.6f} \\u00B1 {:.6f}\".format(i, j) for i, j in zip(table[2], table[3])]\n",
    "table = {\"K\":3**np.arange(5), \"Mean Train Error\":train_err, \"Mean Test Error\":test_err}\n",
    "df = pd.DataFrame.from_dict(table)\n",
    "latex = df.to_latex(index=False)\n",
    "print(latex)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "### KNN: Cross validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true,
    "id": "7md_yyiiclqP"
   },
   "outputs": [],
   "source": [
    "def perform_kfoldCV(k, x, y, hparam, shuffle=True, verbose=True):\n",
    "    \n",
    "    \"\"\" \n",
    "    Performs k-fold cross validation for a given parameter\n",
    "    \n",
    "    Args\n",
    "    ----\n",
    "    k - number of folds of CV to perform\n",
    "    x - training data\n",
    "    y - training labels\n",
    "    hparam - parameter to use for kernel\n",
    "    shuffle - shuffles data before performing k-fold CV\n",
    "    verbose - prints output logs while running\n",
    "    \n",
    "    Returns\n",
    "    -------\n",
    "    cv_error_over_folds - the cross valiation error for each fold of cross validation.\n",
    "    \"\"\"\n",
    "\n",
    "    # Extract dimensions\n",
    "    m, n = x.shape\n",
    "\n",
    "    # Shuffle dataset randomly for splitting into groups\n",
    "    if shuffle:\n",
    "        perm = np.random.permutation(m)\n",
    "        x_shuffled = x[perm, :]\n",
    "        y_shuffled = y[perm]\n",
    "    else:\n",
    "        x_shuffled = x\n",
    "        y_shuffled = y\n",
    "\n",
    "    # Split data into k-groups\n",
    "    x_groups = np.array_split(x_shuffled, k)\n",
    "    y_groups = np.array_split(y_shuffled, k)\n",
    "\n",
    "    # Stores the mean CV error over all folds of CV\n",
    "    cv_error_over_folds = 0\n",
    "\n",
    "    for i in range(len(x_groups)):\n",
    "        \n",
    "        if verbose:\n",
    "            print(\">>>> Cross-validation Fold {}\".format(i+1), end=\"...\")\n",
    "\n",
    "        # Use the selected group as \"validation\" set\n",
    "        val_inputs, val_labels = x_groups[i], y_groups[i]\n",
    "\n",
    "        # Use rest of groups as training set\n",
    "        train_inputs = np.vstack([x_groups[j] for j in range(len(x_groups)) if j != i])\n",
    "        train_labels = np.concatenate([y_groups[j] for j in range(len(x_groups)) if j != i])\n",
    "\n",
    "        #-------------------------TRAIN MODEL--------------------------#\n",
    "\n",
    "        # Load the model\n",
    "        model = KNN(train_inputs, train_labels, hparam)\n",
    "        \n",
    "        # Generate predictions\n",
    "        val_preds = np.zeros(val_inputs.shape[0])\n",
    "        for j in range(len(val_inputs)):\n",
    "            val_preds[j] = model.predict(val_inputs[j])\n",
    "\n",
    "        # Compute mistakes on val set\n",
    "        val_mistakes = np.where(val_preds != val_labels, 1.0, 0.0)\n",
    "        \n",
    "        # Evaluate on validation data\n",
    "        cv_error_over_folds += np.mean(val_mistakes)\n",
    "\n",
    "        if verbose:\n",
    "            print(\"Done!\")\n",
    "\n",
    "    #Average the errors\n",
    "    cv_error_over_folds /= k\n",
    "\n",
    "    return cv_error_over_folds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true,
    "id": "-0fuwk0-EolL"
   },
   "outputs": [],
   "source": [
    "def perform_multiple_runs_with_kFold(runs,\n",
    "                                     param_values,\n",
    "                                     k=5,\n",
    "                                     seed=None,\n",
    "                                     verbose=True,\n",
    "                                     save_results=True,\n",
    "                                     path_to_results=\"\"):\n",
    "    \n",
    "    \"\"\"\n",
    "    Performs multiple runs of training with different parameter values.\n",
    "    \n",
    "    Args\n",
    "    ----\n",
    "    param_values : list of parameter values to train on\n",
    "    k : number of folds of CV\n",
    "    runs : number of runs to perform\n",
    "    seed : to ensure reproducibility of results\n",
    "    verbose : prints outputs while training\n",
    "    save_results : Set true to save the results\n",
    "    path_to_results : Saves results to provided filepath\n",
    "    \n",
    "    Returns\n",
    "    -------\n",
    "    results_df : pandas dataframe of the final results from experiment.\n",
    "    \"\"\"\n",
    "\n",
    "    # Set a random seed for reproducibility of results\n",
    "    np.random.seed(seed)\n",
    "\n",
    "    # Results will be stored here\n",
    "    results = np.zeros((runs, 2))\n",
    "\n",
    "    for run in range(runs):\n",
    "\n",
    "        if verbose:\n",
    "            print(\"Run {}\".format(run+1))\n",
    "\n",
    "        # I will record the errors for a single run in this vector\n",
    "        mean_cv_errors = np.zeros(len(param_values))\n",
    "\n",
    "        # generate random train-test split\n",
    "        train_x, train_y, test_x, test_y, _ = split_data(inputs=x, targets=y, test_proportion=0.20, shuffle=True)\n",
    "\n",
    "        #----------------------------PERFORM k-FOLD CV-------------------------------#\n",
    "\n",
    "        for i, p in enumerate(param_values):\n",
    "\n",
    "            if verbose:\n",
    "                print(\">> parameter = {}\".format(p))\n",
    "\n",
    "            # Perform k-fold CV on the training set\n",
    "            mean_cv_errors[i] = perform_kfoldCV(k, train_x, train_y, p, verbose=verbose)\n",
    "\n",
    "        #-----FIND BEST HYPERPARAM VALUE AND TRAIN THE WHOLE DATASET WITH THAT-----#\n",
    "\n",
    "        # The d that has lowest mean cv error is the optimal d\n",
    "        param_star = param_values[np.argmin(mean_cv_errors)]\n",
    "\n",
    "        print(\"\\nbest parameter value = {}\".format(param_star), end=\"...\")\n",
    "\n",
    "        if verbose:\n",
    "            print(\"Evaluating\", end=\"...\")\n",
    "\n",
    "        # Fit model on training set\n",
    "        model = KNN(train_x, train_y, param_star)\n",
    "        \n",
    "        # Generate predictions\n",
    "        test_preds = np.zeros(test_x.shape[0])\n",
    "        for j in range(len(test_x)):\n",
    "            test_preds[j] = model.predict(test_x[j])\n",
    "\n",
    "        # Compute mistakes on val set\n",
    "        test_mistakes = np.where(test_preds != test_y, 1.0, 0.0)\n",
    "        \n",
    "        # Record results\n",
    "        results[run, 0] = param_star\n",
    "        results[run, 1] = np.mean(test_mistakes)\n",
    "        \n",
    "        if verbose:\n",
    "            print(\"Done!\")\n",
    "\n",
    "        # Convert results matrix to pandas dataframe\n",
    "        results_df = pd.DataFrame(results, columns=[\"best parameter\", \"Test Error\"], index=[\"Run {}\".format(run+1) for run in range(runs)])\n",
    "\n",
    "        if save_results:\n",
    "            # Save errors\n",
    "            results_df.to_csv(results_dir+path_to_results, header=True, index=True)\n",
    "\n",
    "    return results_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "hidden": true,
    "id": "VE48_FX1uF9V",
    "outputId": "78469122-a64e-4ce0-b822-c4f686eeeac5",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Range of d values\n",
    "k_values = 3**np.arange(5)\n",
    "\n",
    "# Run model for multiple runs with cross validation\n",
    "results_df= perform_multiple_runs_with_kFold(runs=20,\n",
    "                                             param_values=k_values,\n",
    "                                             k=5,\n",
    "                                             seed=231,\n",
    "                                             path_to_results=\"q7_knn_crossval.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# Results\n",
    "results_df = pd.read_csv(results_dir+\"q7_knn_crossval.csv\")\n",
    "results_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "print(\"\\nMean d* \\u00B1 STD = {} \\u00B1 {}\".format(results_df[\"best parameter\"].mean(), results_df[\"best parameter\"].std()))\n",
    "print(\"Mean Test Error \\u00B1 STD = {} \\u00B1 {}\".format(results_df[\"Test Error\"].mean(), results_df[\"Test Error\"].std()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## SVM: Refer to `svm_manual_implementation.ipynb`"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [
    "kzFBh7AmxXD_",
    "886M8Ih646wB"
   ],
   "name": "SL_CW2.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "769px",
    "left": "1444.41px",
    "top": "145px",
    "width": "237px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}