notebooks/art-for-tensorflow-v2-callable.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "x6ys1GCOZqhg"
      },
      "source": [
        "# ART for TensorFlow v2 - Callable Class/Function"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-gAsDzg_Zqhk"
      },
      "source": [
        "This notebook demonstrates applying ART with TensorFlow v2 using callable classes or functions to define models. The code follows and extends the examples on www.tensorflow.org."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "HzeOt5ydZqhl"
      },
      "outputs": [],
      "source": [
        "import warnings\n",
        "warnings.filterwarnings('ignore')\n",
        "import os\n",
        "os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'\n",
        "import tensorflow as tf\n",
        "from tensorflow.keras.layers import Dense, Flatten, Conv2D\n",
        "from tensorflow.keras import Model\n",
        "import numpy as np\n",
        "from matplotlib import pyplot as plt\n",
        "\n",
        "from art.estimators.classification import TensorFlowV2Classifier\n",
        "from art.attacks.evasion import FastGradientMethod, CarliniLInfMethod"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "p0caeFvVZqhn"
      },
      "outputs": [],
      "source": [
        "if tf.__version__[0] != '2':\n",
        "    raise ImportError('This notebook requires TensorFlow v2.')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EdK0rIxrZqhn"
      },
      "source": [
        "# Load MNIST dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QfX6OG05Zqhn",
        "outputId": "d9c90230-5c8a-44af-9410-0494a0312784"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/mnist.npz\n",
            "11490434/11490434 [==============================] - 0s 0us/step\n"
          ]
        }
      ],
      "source": [
        "(x_train, y_train), (x_test, y_test) = tf.keras.datasets.mnist.load_data()\n",
        "x_train, x_test = x_train / 255.0, x_test / 255.0\n",
        "\n",
        "x_train = x_train.astype(np.float32)\n",
        "x_test = x_test.astype(np.float32)\n",
        "\n",
        "x_test = x_test[0:10]\n",
        "y_test = y_test[0:10]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1W2g5CMZZqho"
      },
      "source": [
        "Add a dimension for color channel"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "-UudC_6xZqho"
      },
      "outputs": [],
      "source": [
        "x_train = x_train[..., tf.newaxis]\n",
        "x_test = x_test[..., tf.newaxis]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9yvI-CnTZqhp"
      },
      "source": [
        "Create loss object and optimizer"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "HbGPQz04Zqhp"
      },
      "outputs": [],
      "source": [
        "loss_object = tf.keras.losses.SparseCategoricalCrossentropy()\n",
        "optimizer = tf.keras.optimizers.Adam()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WeEHZ4P2Zqhq"
      },
      "source": [
        "Define metrics for training and testing"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "st7QtD-0Zqhq"
      },
      "outputs": [],
      "source": [
        "train_loss = tf.keras.metrics.Mean(name='train_loss')\n",
        "train_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name='train_accuracy')\n",
        "\n",
        "test_loss = tf.keras.metrics.Mean(name='test_loss')\n",
        "test_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name='test_accuracy')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8c_6fHJ5Zqhq"
      },
      "source": [
        "# TensorFlow with callable class"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wzm2FNhxZqhq"
      },
      "source": [
        "Create a custom model class."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "id": "s9qi6F-QZqhq"
      },
      "outputs": [],
      "source": [
        "class KerasModel(Model):\n",
        "    def __init__(self):\n",
        "        super(KerasModel, self).__init__()\n",
        "        self.conv1 = Conv2D(filters=3, kernel_size=3, activation='relu')\n",
        "        self.flatten = Flatten()\n",
        "        self.dense1 = Dense(10, activation='softmax')\n",
        "\n",
        "    def call(self, x):\n",
        "        x = self.conv1(x)\n",
        "        x = self.flatten(x)\n",
        "        x = self.dense1(x)\n",
        "        return x"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vwiJX-EpZqhq"
      },
      "source": [
        "Create callable model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "1oBsaWt3Zqhq"
      },
      "outputs": [],
      "source": [
        "model = KerasModel()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "llfepQ3CZqhq"
      },
      "source": [
        "Create input pipelines for training and testing"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "id": "Juvdq4rFZqhq"
      },
      "outputs": [],
      "source": [
        "train_ds = tf.data.Dataset.from_tensor_slices((x_train, y_train)).shuffle(10000).batch(32)\n",
        "test_ds = tf.data.Dataset.from_tensor_slices((x_test, y_test)).batch(32)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "__NONpRVZqhr"
      },
      "source": [
        "Define the training step."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "id": "tJcZ1sBfZqhr"
      },
      "outputs": [],
      "source": [
        "@tf.function\n",
        "def train_step(images, labels):\n",
        "    with tf.GradientTape() as tape:\n",
        "        predictions = model(images)\n",
        "        loss = loss_object(labels, predictions)\n",
        "    gradients = tape.gradient(loss, model.trainable_variables)\n",
        "    optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n",
        "\n",
        "    train_loss(loss)\n",
        "    train_accuracy(labels, predictions)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kfR92prOZqhr"
      },
      "source": [
        "Define the testing step."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "ssCesqdTZqhr"
      },
      "outputs": [],
      "source": [
        "@tf.function\n",
        "def test_step(images, labels):\n",
        "    predictions = model(images)\n",
        "    t_loss = loss_object(labels, predictions)\n",
        "\n",
        "    test_loss(t_loss)\n",
        "    test_accuracy(labels, predictions)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jWXChGcyZqhr"
      },
      "source": [
        "Fit the model on training data and collect metrics for training and testing."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8caHTvJgZqhr",
        "outputId": "8133fd71-0b69-4fcc-dab2-647f2d3c0e01"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1, Loss: 0.34, Accuracy: 90.55, Test Loss: 0.39, Test Accuracy: 90.00\n",
            "Epoch 2, Loss: 0.26, Accuracy: 92.70, Test Loss: 0.35, Test Accuracy: 90.00\n",
            "Epoch 3, Loss: 0.22, Accuracy: 93.75, Test Loss: 0.39, Test Accuracy: 90.00\n"
          ]
        }
      ],
      "source": [
        "epochs = 3\n",
        "\n",
        "for epoch in range(epochs):\n",
        "    for images, labels in train_ds:\n",
        "        train_step(images, labels)\n",
        "\n",
        "    for test_images, test_labels in test_ds:\n",
        "        test_step(test_images, test_labels)\n",
        "\n",
        "    template = 'Epoch {}, Loss: {:4.2f}, Accuracy: {:4.2f}, Test Loss: {:4.2f}, Test Accuracy: {:4.2f}'\n",
        "    print(template.format(epoch + 1,\n",
        "                          train_loss.result(),\n",
        "                          train_accuracy.result() * 100,\n",
        "                          test_loss.result(),\n",
        "                          test_accuracy.result() * 100))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ur_Oc5KvZqhs"
      },
      "source": [
        "Evaluate model accuracy on test data."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FschhFXSZqhs",
        "outputId": "ee41a4c5-3ec3-4514-adf0-ebb317fe33e8"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Accuracy on test data: 90.00%\n"
          ]
        }
      ],
      "source": [
        "y_test_pred = np.argmax(model(x_test), axis=1)\n",
        "accuracy_test = np.sum(y_test_pred == y_test) / y_test.shape[0]\n",
        "print('Accuracy on test data: {:4.2f}%'.format(accuracy_test * 100))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RH01U7_BZqhs"
      },
      "source": [
        "Create a ART TensorFlow v2 classifier for the TensorFlow custom model class."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "id": "ilKTAr8BZqhs"
      },
      "outputs": [],
      "source": [
        "classifier = TensorFlowV2Classifier(model=model, nb_classes=10, input_shape=(28, 28, 1), loss_object=loss_object,\n",
        "                                    clip_values=(0, 1), channels_first=False)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "p_bEYoaWZqhs"
      },
      "source": [
        "## Fast Gradient Sign Method attack"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "T1dfZKcwZqhs"
      },
      "source": [
        "Create a ART Fast Gradient Sign Method attack."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "id": "ThCnPgL6Zqhs"
      },
      "outputs": [],
      "source": [
        "attack_fgsm = FastGradientMethod(estimator=classifier)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Hhhnjw9GZqhs"
      },
      "source": [
        "Generate adversarial test data."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "scrolled": false,
        "id": "UiewqnIvZqht"
      },
      "outputs": [],
      "source": [
        "x_test_adv = attack_fgsm.generate(x_test)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pJqv2djMZqht"
      },
      "source": [
        "Evaluate accuracy on adversarial test data and calculate average perturbation."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dxh_xaDGZqht",
        "outputId": "fd9131c6-b734-401f-a939-561c63e0703f"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Accuracy on adversarial test data: 10.00%\n",
            "Average perturbation: 0.16\n"
          ]
        }
      ],
      "source": [
        "y_test_pred = np.argmax(model(x_test_adv), axis=1)\n",
        "accuracy_test_adv = np.sum(y_test_pred == y_test) / y_test.shape[0]\n",
        "perturbation = np.mean(np.abs((x_test_adv - x_test)))\n",
        "print('Accuracy on adversarial test data: {:4.2f}%'.format(accuracy_test_adv * 100))\n",
        "print('Average perturbation: {:4.2f}'.format(perturbation))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TnOSep49Zqht"
      },
      "source": [
        "Visualise the first adversarial test sample."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 437
        },
        "id": "fVRYBarHZqht",
        "outputId": "e1d6f62d-e8ca-4863-ae5e-735de95153ae"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 480x480 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaMAAAGkCAYAAACckEpMAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAeBElEQVR4nO3df3DU9b3v8dcSkgU1bIwx2aQEDIii8kOLkuagFEtKSGcYUKYj6vSA14GrBqeYWp10VNT2Tlo6x3rtpHj+aKF2RNS5AoPTSw9GE4YaYEA4XG5tDslNJQxJqJybLAQTQvK5f3BZuhIg32Wz783m+Zj5juzu55Pvez/5Zl9+s9+81+eccwIAwNAI6wIAACCMAADmCCMAgDnCCABgjjACAJgjjAAA5ggjAIA5wggAYI4wAgCYI4wAAOaGTBhVVVXppptu0qhRo1RYWKg9e/ZYlxR3L7/8snw+X8Q2efJk67LiYseOHVqwYIHy8vLk8/m0efPmiMedc3rppZeUm5ur0aNHq7i4WIcPH7YpdhBdaR2WLVt20TEyf/58m2IHUWVlpe655x6lp6crOztbixYtUn19fcSYrq4ulZWV6YYbbtB1112nxYsXq62tzajiwTGQdZgzZ85Fx8QTTzxhVPGlDYkwevfdd1VeXq7Vq1frs88+0/Tp01VSUqLjx49blxZ3d9xxh1paWsLbzp07rUuKi87OTk2fPl1VVVX9Pr5mzRq98cYbevPNN7V7925de+21KikpUVdXV5wrHVxXWgdJmj9/fsQx8s4778Sxwviora1VWVmZdu3ape3bt6unp0fz5s1TZ2dneMwzzzyjrVu36v3331dtba2OHTumBx980LDq2BvIOkjS8uXLI46JNWvWGFV8GW4ImDlzpisrKwvf7u3tdXl5ea6ystKwqvhbvXq1mz59unUZ5iS5TZs2hW/39fW5YDDofvnLX4bva29vd36/373zzjsGFcbH19fBOeeWLl3qFi5caFKPpePHjztJrra21jl37vufmprq3n///fCYzz//3ElydXV1VmUOuq+vg3POffvb33Y//OEP7YoaoIQ/Mzpz5oz27dun4uLi8H0jRoxQcXGx6urqDCuzcfjwYeXl5WnChAl69NFHdeTIEeuSzDU1Nam1tTXiGAkEAiosLByWx0hNTY2ys7N166236sknn9SJEyesSxp0HR0dkqTMzExJ0r59+9TT0xNxTEyePFnjxo1L6mPi6+tw3ttvv62srCxNmTJFFRUVOn36tEV5lzXSuoAr+fLLL9Xb26ucnJyI+3NycvTXv/7VqCobhYWFWr9+vW699Va1tLTolVde0X333adDhw4pPT3dujwzra2tktTvMXL+seFi/vz5evDBB1VQUKDGxkb95Cc/UWlpqerq6pSSkmJd3qDo6+vTqlWrNGvWLE2ZMkXSuWMiLS1NGRkZEWOT+Zjobx0k6ZFHHtH48eOVl5engwcP6vnnn1d9fb0++OADw2ovlvBhhAtKS0vD/542bZoKCws1fvx4vffee3r88ccNK0OiWLJkSfjfU6dO1bRp0zRx4kTV1NRo7ty5hpUNnrKyMh06dGjYvH96KZdahxUrVoT/PXXqVOXm5mru3LlqbGzUxIkT413mJSX8r+mysrKUkpJy0VUwbW1tCgaDRlUlhoyMDN1yyy1qaGiwLsXU+eOAY+RiEyZMUFZWVtIeIytXrtSHH36oTz75RGPHjg3fHwwGdebMGbW3t0eMT9Zj4lLr0J/CwkJJSrhjIuHDKC0tTTNmzFB1dXX4vr6+PlVXV6uoqMiwMnunTp1SY2OjcnNzrUsxVVBQoGAwGHGMhEIh7d69e9gfI0ePHtWJEyeS7hhxzmnlypXatGmTPv74YxUUFEQ8PmPGDKWmpkYcE/X19Tpy5EhSHRNXWof+HDhwQJIS75iwvoJiIDZu3Oj8fr9bv369+8tf/uJWrFjhMjIyXGtrq3VpcfWjH/3I1dTUuKamJvfnP//ZFRcXu6ysLHf8+HHr0gbdyZMn3f79+93+/fudJPfaa6+5/fv3uy+++MI559zPf/5zl5GR4bZs2eIOHjzoFi5c6AoKCtxXX31lXHlsXW4dTp486Z599llXV1fnmpqa3EcffeS++c1vukmTJrmuri7r0mPqySefdIFAwNXU1LiWlpbwdvr06fCYJ554wo0bN859/PHHbu/eva6oqMgVFRUZVh17V1qHhoYG9+qrr7q9e/e6pqYmt2XLFjdhwgQ3e/Zs48ovNiTCyDnnfv3rX7tx48a5tLQ0N3PmTLdr1y7rkuLuoYcecrm5uS4tLc194xvfcA899JBraGiwLisuPvnkEyfpom3p0qXOuXOXd7/44osuJyfH+f1+N3fuXFdfX29b9CC43DqcPn3azZs3z914440uNTXVjR8/3i1fvjwp/6etvzWQ5NatWxce89VXX7mnnnrKXX/99e6aa65xDzzwgGtpabErehBcaR2OHDniZs+e7TIzM53f73c333yz+/GPf+w6OjpsC++Hzznn4nceBgDAxRL+PSMAQPIjjAAA5ggjAIA5wggAYI4wAgCYI4wAAOaGVBh1d3fr5ZdfVnd3t3UppliHC1iLc1iHC1iLc4baOgypvzMKhUIKBALq6OjQmDFjrMsxwzpcwFqcwzpcwFqcM9TWYUidGQEAkhNhBAAwl3CfZ9TX16djx44pPT1dPp8v4rFQKBTx3+GKdbiAtTiHdbiAtTgnEdbBOaeTJ08qLy9PI0Zc/twn4d4zOnr0qPLz863LAADESHNz8xU/ZynhzozOf3z2tMUvKiV11IDnBTbuGaySLtKxZKbnOfGsL16iWQeJtTgvGdcB+Edn1aOd+mP4df1yEi6Mzv9qLiV1lFLSBh5GI32pg1XSRbzUdV4864uXaNZBYi3OS8Z1ACL8/9+7ff0tl/4M2gUMVVVVuummmzRq1CgVFhZqzx7+LxAA0L9BCaN3331X5eXlWr16tT777DNNnz5dJSUlOn78+GDsDgAwxA1KGL322mtavny5HnvsMd1+++168803dc011+h3v/vdYOwOADDExTyMzpw5o3379qm4uPjCTkaMUHFxserq6i4a393drVAoFLEBAIaXmIfRl19+qd7eXuXk5ETcn5OTo9bW1ovGV1ZWKhAIhDcu6waA4ce8A0NFRYU6OjrCW3Nzs3VJAIA4i/ml3VlZWUpJSVFbW1vE/W1tbQoGgxeN9/v98vv9sS4DADCExPzMKC0tTTNmzFB1dXX4vr6+PlVXV6uoqCjWuwMAJIFB+aPX8vJyLV26VHfffbdmzpyp119/XZ2dnXrssccGY3cAgCFuUMLooYce0t///ne99NJLam1t1Z133qlt27ZddFEDAABSAjZKPf+BUHct+W+eWqxk/OHiy8aRmNp/4P3XtdF8f6PZTzxxzCLZnXU9qtGWAX3An/nVdAAAEEYAAHOEEQDAHGEEADBHGAEAzBFGAABzhBEAwBxhBAAwRxgBAMwRRgAAc4QRAMAcYQQAMEcYAQDMJWzX7jlaqJG+VOtyhqRE71YNDCV0V48eXbsBAEMKYQQAMEcYAQDMEUYAAHOEEQDAHGEEADBHGAEAzBFGAABzhBEAwBxhBAAwRxgBAMwRRgAAcyOtC8Cl0fAUsBfPn8Ph3JSVMyMAgDnCCABgjjACAJgjjAAA5ggjAIA5wggAYI4wAgCYI4wAAOYIIwCAOcIIAGCOMAIAmCOMAADmErZRaseSmUpJGzXg8YneYJCmpxdE871i/RAL8Tz2Evk4T8TXS86MAADmCCMAgDnCCABgjjACAJgjjAAA5ggjAIA5wggAYI4wAgCYI4wAAOYIIwCAOcIIAGCOMAIAmEvYRqmBjXs00pdqXcawkYiNEy3Ecx1o/hp/ybjmiXzM9p7pkjZuGdBYzowAAOYIIwCAuZiH0csvvyyfzxexTZ48Oda7AQAkkUF5z+iOO+7QRx99dGEnIxP2rSkAQAIYlJQYOXKkgsHgYHxpAEASGpT3jA4fPqy8vDxNmDBBjz76qI4cOXLJsd3d3QqFQhEbAGB4iXkYFRYWav369dq2bZvWrl2rpqYm3XfffTp58mS/4ysrKxUIBMJbfn5+rEsCACS4mIdRaWmpvv/972vatGkqKSnRH//4R7W3t+u9997rd3xFRYU6OjrCW3Nzc6xLAgAkuEG/siAjI0O33HKLGhoa+n3c7/fL7/cPdhkAgAQ26H9ndOrUKTU2Nio3N3ewdwUAGKJiHkbPPvusamtr9be//U2ffvqpHnjgAaWkpOjhhx+O9a4AAEki5r+mO3r0qB5++GGdOHFCN954o+69917t2rVLN954Y6x3BQBIEjEPo40bN8b6SwIAkhytERB3ydg5ORmfUzSi6SAdz7VL5A7Xwx2NUgEA5ggjAIA5wggAYI4wAgCYI4wAAOYIIwCAOcIIAGCOMAIAmCOMAADmCCMAgDnCCABgjjACAJijUWoSyvwf/+55zhcv/JPnOde0OM9zJGlkl/d5bff1eZ5zyxN7PM/BBfFqKhrP5qWJbjivBWdGAABzhBEAwBxhBAAwRxgBAMwRRgAAc4QRAMAcYQQAMEcYAQDMEUYAAHOEEQDAHGEEADBHGAEAzCVso9SOJTOVkjZqwOOTscFgtM+pfv0Mz3Mm5jdHta942XXbVs9z7spd4nnOqc+v9zxHkg7/89qo5iWySXc8GZf9RLt2X/Z2ep7zaP6sqPYVDV6TpLOuZ8BjOTMCAJgjjAAA5ggjAIA5wggAYI4wAgCYI4wAAOYIIwCAOcIIAGCOMAIAmCOMAADmCCMAgDnCCABgjjACAJhL2K7dgY17NNKXal3GkOT6fJ7npDw7xvt+9v9vz3MkyXfXHZ7nzJjpvYP0ydldnucU3HPU85xofffzBZ7nbI+ie3m0cu9q9Tyn5URgECrp38P/4b0re/eiXM9zRm/e43lOsmr/QZGn8b1nuqSNWwY0ljMjAIA5wggAYI4wAgCYI4wAAOYIIwCAOcIIAGCOMAIAmCOMAADmCCMAgDnCCABgjjACAJgjjAAA5hK2USqil/2J9wazbv/eQajkUvvy3mA1a7/3/WT9q/c50bqr7CnPc7KrPvU8p0R3ep4TrdFq8jxnQjQ7OhbNpOhc9/l/ep7TG+W+vDYVlaSMP9RFubehjzMjAIA5wggAYM5zGO3YsUMLFixQXl6efD6fNm/eHPG4c04vvfSScnNzNXr0aBUXF+vw4cOxqhcAkIQ8h1FnZ6emT5+uqqqqfh9fs2aN3njjDb355pvavXu3rr32WpWUlKiry/sHnQEAhgfPFzCUlpaqtLS038ecc3r99df1wgsvaOHChZKkt956Szk5Odq8ebOWLPH+yYwAgOQX0/eMmpqa1NraquLi4vB9gUBAhYWFqqvr/yqR7u5uhUKhiA0AMLzENIxaW1slSTk5ORH35+TkhB/7usrKSgUCgfCWn58fy5IAAEOA+dV0FRUV6ujoCG/Nzc3WJQEA4iymYRQMBiVJbW1tEfe3tbWFH/s6v9+vMWPGRGwAgOElpmFUUFCgYDCo6urq8H2hUEi7d+9WUZH3v0YGAAwPnq+mO3XqlBoaGsK3m5qadODAAWVmZmrcuHFatWqVfvazn2nSpEkqKCjQiy++qLy8PC1atCiWdQMAkojnMNq7d6/uv//+8O3y8nJJ0tKlS7V+/Xo999xz6uzs1IoVK9Te3q57771X27Zt06hRo2JXNQAgqficc866iH8UCoUUCAQ0Rws10ue94Wc8JHoDxESvL16iWYdEl+jHUeb73jvanvkw2/McSWr6a67nOZNW7o5qX4jOWdejGm1RR0fHFa8HML+aDgAAwggAYI4wAgCYI4wAAOYIIwCAOcIIAGCOMAIAmCOMAADmCCMAgDnCCABgjjACAJgjjAAA5jx37UbiNxVN9PoQvWibv0ZzTHyV7fM8p+mtSZ7njPf9X89zJOnWZ//d85y+KPYTzzUfzjgzAgCYI4wAAOYIIwCAOcIIAGCOMAIAmCOMAADmCCMAgDnCCABgjjACAJgjjAAA5ggjAIA5wggAYI4wAgCYS9iu3R1LZiolbdSAx9MhN7lF2zkZ50Szfov/ucbznJ1fTvQ851hojOc5kpTXddTznGjWgdeW+ODMCABgjjACAJgjjAAA5ggjAIA5wggAYI4wAgCYI4wAAOYIIwCAOcIIAGCOMAIAmCOMAADmCCMAgLmEbZQa2LhHI32p1mUASeH4P/V6nrO9ZbLnOaNTezzPyXvgL57nSDTPvRrxWrveM13Sxi0DGsuZEQDAHGEEADBHGAEAzBFGAABzhBEAwBxhBAAwRxgBAMwRRgAAc4QRAMAcYQQAMEcYAQDMEUYAAHMJ2ygVQOw8dV+15zn/1na75zlHa/M9z7nuv4z1PEeSRnjvyaqMP9RFta9oJGMjV6/rd9YN/JvEmREAwBxhBAAw5zmMduzYoQULFigvL08+n0+bN2+OeHzZsmXy+XwR2/z582NVLwAgCXkOo87OTk2fPl1VVVWXHDN//ny1tLSEt3feeeeqigQAJDfPFzCUlpaqtLT0smP8fr+CwWDURQEAhpdBec+opqZG2dnZuvXWW/Xkk0/qxIkTlxzb3d2tUCgUsQEAhpeYh9H8+fP11ltvqbq6Wr/4xS9UW1ur0tJS9fb29ju+srJSgUAgvOXne780FAAwtMX874yWLFkS/vfUqVM1bdo0TZw4UTU1NZo7d+5F4ysqKlReXh6+HQqFCCQAGGYG/dLuCRMmKCsrSw0NDf0+7vf7NWbMmIgNADC8DHoYHT16VCdOnFBubu5g7woAMER5/jXdqVOnIs5ympqadODAAWVmZiozM1OvvPKKFi9erGAwqMbGRj333HO6+eabVVJSEtPCAQDJw3MY7d27V/fff3/49vn3e5YuXaq1a9fq4MGD+v3vf6/29nbl5eVp3rx5+ulPfyq/3x+7qgEAScVzGM2ZM0fOuUs+/qc//emqCgIADD907Y5ConfjjVdn4kRfh2R0dnR089786Lue5wQnH/c8Z9yrn3qeE+1xFM8O3MkmEdeORqkAAHOEEQDAHGEEADBHGAEAzBFGAABzhBEAwBxhBAAwRxgBAMwRRgAAc4QRAMAcYQQAMEcYAQDMJWyj1I4lM5WSNmrQ95OIDQOvFg1Mz4nmexvPtYumvt4534xqX5kP/d3znO23bfU8p0R3ep6T6BL95ylZXsM4MwIAmCOMAADmCCMAgDnCCABgjjACAJgjjAAA5ggjAIA5wggAYI4wAgCYI4wAAOYIIwCAOcIIAGAuYRulepXozQITvWlnMkrG9Tv7k/+Mal5NFE1PJ2z6r57nTNJuz3PiKdGPiXi9TiTi6yVnRgAAc4QRAMAcYQQAMEcYAQDMEUYAAHOEEQDAHGEEADBHGAEAzBFGAABzhBEAwBxhBAAwRxgBAMwlTaNUwFLXDT7Pc/5j3QzPc25OOe55jiQdOXvK85xJZfFpepqITTsRf5wZAQDMEUYAAHOEEQDAHGEEADBHGAEAzBFGAABzhBEAwBxhBAAwRxgBAMwRRgAAc4QRAMAcYQQAMEcYAQDM0bU7Ttp/UGRdAgYg2g7Sjf/yLc9zbh7nvQP39tu2ep4jSbf963Oe54zTp1HtK16i+ZmK5vub6D+7ydL1nDMjAIA5wggAYM5TGFVWVuqee+5Renq6srOztWjRItXX10eM6erqUllZmW644QZdd911Wrx4sdra2mJaNAAguXgKo9raWpWVlWnXrl3avn27enp6NG/ePHV2dobHPPPMM9q6davef/991dbW6tixY3rwwQdjXjgAIHl4uoBh27ZtEbfXr1+v7Oxs7du3T7Nnz1ZHR4d++9vfasOGDfrOd74jSVq3bp1uu+027dq1S9/61sVv8nZ3d6u7uzt8OxQKRfM8AABD2FW9Z9TR0SFJyszMlCTt27dPPT09Ki4uDo+ZPHmyxo0bp7q6/q/4qKysVCAQCG/5+flXUxIAYAiKOoz6+vq0atUqzZo1S1OmTJEktba2Ki0tTRkZGRFjc3Jy1Nra2u/XqaioUEdHR3hrbm6OtiQAwBAV9d8ZlZWV6dChQ9q5c+dVFeD3++X3+6/qawAAhraozoxWrlypDz/8UJ988onGjh0bvj8YDOrMmTNqb2+PGN/W1qZgMHhVhQIAkpenMHLOaeXKldq0aZM+/vhjFRQURDw+Y8YMpaamqrq6OnxffX29jhw5oqKixP4rZgCAHU+/pisrK9OGDRu0ZcsWpaenh98HCgQCGj16tAKBgB5//HGVl5crMzNTY8aM0dNPP62ioqJ+r6QDAEDyGEZr166VJM2ZMyfi/nXr1mnZsmWSpF/96lcaMWKEFi9erO7ubpWUlOg3v/lNTIoFACQnT2HknLvimFGjRqmqqkpVVVVRFyVJgY17NNKXelVfYyASvQki4uuLV6M7HgruPBrjSvp3S+3SqOZd/3+u/LMbC/FqXooLkmXN6U0HADBHGAEAzBFGAABzhBEAwBxhBAAwRxgBAMwRRgAAc4QRAMAcYQQAMEcYAQDMEUYAAHOEEQDAXNSf9IrBF20zw3g1Tkz0JrMnx/k8zzlzU9cgVNK/sde2e54T+p9jrzyoH4nYGBP4R5wZAQDMEUYAAHOEEQDAHGEEADBHGAEAzBFGAABzhBEAwBxhBAAwRxgBAMwRRgAAc4QRAMAcYQQAMEcYAQDMJU3X7kTvII34e+zhP3me829ttw9CJf3bsesOz3OyXHT7itfPB93Br85wfh3jzAgAYI4wAgCYI4wAAOYIIwCAOcIIAGCOMAIAmCOMAADmCCMAgDnCCABgjjACAJgjjAAA5ggjAIC5hG2U2rFkplLSRlmXMWwkeoPG47N6Pc/5cWaj5znRNkrdfttWz3Mm/q8notpXvND0NP6iWfNE/9kdKM6MAADmCCMAgDnCCABgjjACAJgjjAAA5ggjAIA5wggAYI4wAgCYI4wAAOYIIwCAOcIIAGCOMAIAmEvYRqnxksjNIJOlAWIsLLh7v+c53/18wSBUEjsjT/msS8AAJPJrhBTf+ry+JvWe6ZI2bhnQWM6MAADmCCMAgDlPYVRZWal77rlH6enpys7O1qJFi1RfXx8xZs6cOfL5fBHbE08k9ue2AABseQqj2tpalZWVadeuXdq+fbt6eno0b948dXZ2Roxbvny5WlpawtuaNWtiWjQAILl4uoBh27ZtEbfXr1+v7Oxs7du3T7Nnzw7ff8011ygYDMamQgBA0ruq94w6OjokSZmZmRH3v/3228rKytKUKVNUUVGh06dPX/JrdHd3KxQKRWwAgOEl6ku7+/r6tGrVKs2aNUtTpkwJ3//II49o/PjxysvL08GDB/X888+rvr5eH3zwQb9fp7KyUq+88kq0ZQAAkkDUYVRWVqZDhw5p586dEfevWLEi/O+pU6cqNzdXc+fOVWNjoyZOnHjR16moqFB5eXn4digUUn5+frRlAQCGoKjCaOXKlfrwww+1Y8cOjR079rJjCwsLJUkNDQ39hpHf75ff74+mDABAkvAURs45Pf3009q0aZNqampUUFBwxTkHDhyQJOXm5kZVIAAg+XkKo7KyMm3YsEFbtmxRenq6WltbJUmBQECjR49WY2OjNmzYoO9973u64YYbdPDgQT3zzDOaPXu2pk2bNihPAAAw9HkKo7Vr10o694et/2jdunVatmyZ0tLS9NFHH+n1119XZ2en8vPztXjxYr3wwgsxKxgAkHw8/5rucvLz81VbW3tVBZ0X2LhHI32pMflaGJ6237Y1bvua/ounPM+56b9/6nlOtM1zE73ZZzSS8TklOq9rftb1DHgsvekAAOYIIwCAOcIIAGCOMAIAmCOMAADmCCMAgDnCCABgjjACAJgjjAAA5ggjAIA5wggAYI4wAgCYI4wAAOai/tjx4SzazsnJJp5dk3fL+5oX6m7Pc6J9TkHFrwN3IovmOUW75vHcV7KJ17HXe6ZL2rhlQGM5MwIAmCOMAADmCCMAgDnCCABgjjACAJgjjAAA5ggjAIA5wggAYI4wAgCYI4wAAOYIIwCAuYTrTeeckySdVY/kjIu5hN4zXdYlJISzridu+4rXmifjc5Li97yieU7R1hbPfSWbeB17vT3n9nP+df1yfG4go+Lo6NGjys/Pty4DABAjzc3NGjt27GXHJFwY9fX16dixY0pPT5fP54t4LBQKKT8/X83NzRozZoxRhfZYhwtYi3NYhwtYi3MSYR2cczp58qTy8vI0YsTl3xVKuF/TjRgx4ooJOmbMmGF9kJ3HOlzAWpzDOlzAWpxjvQ6BQGBA47iAAQBgjjACAJgbUmHk9/u1evVq+f1+61JMsQ4XsBbnsA4XsBbnDLV1SLgLGAAAw8+QOjMCACQnwggAYI4wAgCYI4wAAOYIIwCAOcIIAGCOMAIAmCOMAADm/h8IFh9WmX0LFgAAAABJRU5ErkJggg==\n"
          },
          "metadata": {}
        }
      ],
      "source": [
        "plt.matshow(x_test_adv[0, :, :, 0])\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EzsaRb6HZqht"
      },
      "source": [
        "## Carlini&Wagner Infinity-norm attack"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "73nss-tLZqhy"
      },
      "source": [
        "Create a ART Carlini&Wagner Infinity-norm attack."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {
        "id": "Nzo6LLv9Zqhz"
      },
      "outputs": [],
      "source": [
        "attack_cw = CarliniLInfMethod(classifier=classifier,\n",
        "                              max_iter=100,\n",
        "                              learning_rate=0.01,\n",
        "                              initial_const=1e0,\n",
        "                              largest_const=2e0)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tiS68nqhZqhz"
      },
      "source": [
        "Generate adversarial test data."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 87,
          "referenced_widgets": [
            "0b8242407f8245e197f108e489b709f3",
            "ee9bf3456da448a9a6fd9c32b4b55d1c",
            "63cc72f10306471bb259f407b86d40b3",
            "571690ae3d4848b2b144e309baa6db62",
            "7a3ba25da2ec43c5a02f33256e908a2d",
            "290351d386ad460c89f4a6518fbca91b",
            "0a8aa57f7b904d0784b292823e9c2407",
            "b0138b0c3dc14596a372a7f7a6353c71",
            "773990d53c4f498a83c467ea828a6396",
            "430139380b54472ca2753c9392e7917a",
            "8d62f30dc13341bbb50aa01551102379"
          ]
        },
        "id": "Pti4oVUGZqhz",
        "outputId": "6a7dc9fa-0dbf-480a-8936-4c12df05e77f"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "C&W L_inf:   0%|          | 0/10 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "0b8242407f8245e197f108e489b709f3"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:tensorflow:Calling GradientTape.gradient on a persistent tape inside its context is significantly less efficient than calling it outside the context (it causes the gradient ops to be recorded on the tape, leading to increased CPU and memory usage). Only call GradientTape.gradient inside the context if you actually want to trace the gradient in order to compute higher order derivatives.\n"
          ]
        }
      ],
      "source": [
        "x_test_adv = attack_cw.generate(x_test)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6aJMVMryZqhz"
      },
      "source": [
        "Evaluate accuracy on adversarial test data and calculate average perturbation."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MmjumC6oZqhz",
        "outputId": "765c408d-7d71-4ad9-dda0-bd4d3da888da"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Accuracy on adversarial test data: 10.00%\n",
            "Average perturbation: 0.03\n"
          ]
        }
      ],
      "source": [
        "y_test_pred = np.argmax(model(x_test_adv), axis=1)\n",
        "accuracy_test_adv = np.sum(y_test_pred == y_test) / y_test.shape[0]\n",
        "perturbation = np.mean(np.abs((x_test_adv - x_test)))\n",
        "print('Accuracy on adversarial test data: {:4.2f}%'.format(accuracy_test_adv * 100))\n",
        "print('Average perturbation: {:4.2f}'.format(perturbation))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "A8VqF-oeZqhz"
      },
      "source": [
        "Visualise the first adversarial test sample."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 23,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 437
        },
        "id": "YU0yPQLzZqhz",
        "outputId": "a601ec56-7fae-4b61-dd87-39a5a8668141"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 480x480 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaMAAAGkCAYAAACckEpMAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAb/ElEQVR4nO3df3DU9b3v8dcGkgU02TSEZBMJGFChlR+9pZCmKMWSIcS5XlDOvf6aO+AwMNBghWh10lHA2jtp6RnKsYfimXss1BlR6z0Co3MuvRhNONgEhwjDMNpckpsWKCQUziUbgiwh+dw/uC5dCeB32c072TwfM98x+/1+3/t98/ELr3x3v/tZn3POCQAAQynWDQAAQBgBAMwRRgAAc4QRAMAcYQQAMEcYAQDMEUYAAHOEEQDAHGEEADBHGAEAzA2YMNq0aZNuv/12DRs2TEVFRfr444+tW+pz69atk8/ni1omTpxo3Vaf2LNnjx544AHl5+fL5/Npx44dUdudc1qzZo3y8vI0fPhwlZSU6MiRIzbNJtCNxmHx4sVXnSPz5s2zaTaBqqqqNH36dKWnpysnJ0cLFixQY2Nj1D4XLlxQeXm5Ro4cqVtvvVULFy5UW1ubUceJ8VXGYfbs2VedE8uXLzfq+NoGRBi99dZbqqio0Nq1a/XJJ59o6tSpKi0t1alTp6xb63N33323Tp48GVn27t1r3VKf6Ozs1NSpU7Vp06Zet69fv14vv/yyXnnlFe3bt0+33HKLSktLdeHChT7uNLFuNA6SNG/evKhz5I033ujDDvtGbW2tysvLVV9fr927d6urq0tz585VZ2dnZJ/Vq1fr3Xff1dtvv63a2lqdOHFCDz30kGHX8fdVxkGSli5dGnVOrF+/3qjj63ADwIwZM1x5eXnkcXd3t8vPz3dVVVWGXfW9tWvXuqlTp1q3YU6S2759e+RxT0+PCwaD7he/+EVk3dmzZ53f73dvvPGGQYd948vj4JxzixYtcvPnzzfpx9KpU6ecJFdbW+ucu/z/PzU11b399tuRfT777DMnydXV1Vm1mXBfHgfnnPve977nnnrqKbumvqJ+f2V08eJFNTQ0qKSkJLIuJSVFJSUlqqurM+zMxpEjR5Sfn69x48bp8ccf19GjR61bMtfS0qLW1taocyQQCKioqGhQniM1NTXKycnRhAkTtGLFCp05c8a6pYRrb2+XJGVlZUmSGhoa1NXVFXVOTJw4UWPGjEnqc+LL4/CF119/XdnZ2Zo0aZIqKyt1/vx5i/aua6h1Azdy+vRpdXd3Kzc3N2p9bm6u/vjHPxp1ZaOoqEhbt27VhAkTdPLkSb344ou69957dfjwYaWnp1u3Z6a1tVWSej1Hvtg2WMybN08PPfSQCgsL1dzcrB//+McqKytTXV2dhgwZYt1eQvT09GjVqlWaOXOmJk2aJOnyOZGWlqbMzMyofZP5nOhtHCTpscce09ixY5Wfn69Dhw7pueeeU2Njo9555x3Dbq/W78MIV5SVlUV+njJlioqKijR27Fj97ne/05IlSww7Q3/xyCOPRH6ePHmypkyZovHjx6umpkZz5swx7CxxysvLdfjw4UHz/um1XGscli1bFvl58uTJysvL05w5c9Tc3Kzx48f3dZvX1O9fpsvOztaQIUOuugumra1NwWDQqKv+ITMzU3fddZeampqsWzH1xXnAOXK1cePGKTs7O2nPkZUrV+q9997Thx9+qNGjR0fWB4NBXbx4UWfPno3aP1nPiWuNQ2+Kiookqd+dE/0+jNLS0jRt2jRVV1dH1vX09Ki6ulrFxcWGndk7d+6cmpublZeXZ92KqcLCQgWDwahzJBQKad++fYP+HDl+/LjOnDmTdOeIc04rV67U9u3b9cEHH6iwsDBq+7Rp05Samhp1TjQ2Nuro0aNJdU7caBx6c/DgQUnqf+eE9R0UX8Wbb77p/H6/27p1q/v000/dsmXLXGZmpmttbbVurU89/fTTrqamxrW0tLiPPvrIlZSUuOzsbHfq1Cnr1hKuo6PDHThwwB04cMBJchs2bHAHDhxwf/7zn51zzv3sZz9zmZmZbufOne7QoUNu/vz5rrCw0H3++efGncfX9caho6PDPfPMM66urs61tLS4999/333rW99yd955p7tw4YJ163G1YsUKFwgEXE1NjTt58mRkOX/+fGSf5cuXuzFjxrgPPvjA7d+/3xUXF7vi4mLDruPvRuPQ1NTkfvKTn7j9+/e7lpYWt3PnTjdu3Dg3a9Ys486vNiDCyDnnfvWrX7kxY8a4tLQ0N2PGDFdfX2/dUp97+OGHXV5enktLS3O33Xabe/jhh11TU5N1W33iww8/dJKuWhYtWuScu3x79wsvvOByc3Od3+93c+bMcY2NjbZNJ8D1xuH8+fNu7ty5btSoUS41NdWNHTvWLV26NCl/aettDCS5LVu2RPb5/PPP3Q9+8AP3ta99zY0YMcI9+OCD7uTJk3ZNJ8CNxuHo0aNu1qxZLisry/n9fnfHHXe4H/3oR669vd228V74nHOu767DAAC4Wr9/zwgAkPwIIwCAOcIIAGCOMAIAmCOMAADmCCMAgLkBFUbhcFjr1q1TOBy2bsUU43AFY3EZ43AFY3HZQBuHAfU5o1AopEAgoPb2dmVkZFi3Y4ZxuIKxuIxxuIKxuGygjcOAujICACQnwggAYK7ffZ9RT0+PTpw4ofT0dPl8vqhtoVAo6r+DFeNwBWNxGeNwBWNxWX8YB+ecOjo6lJ+fr5SU61/79Lv3jI4fP66CggLrNgAAcXLs2LEbfs9Sv7sy+uLrs+/R/RqqVONuAACxuqQu7dW/Rv5dv55+F0ZfvDQ3VKka6iOMAGDA+v+vu335LZfeJOwGhk2bNun222/XsGHDVFRUpI8//jhRhwIADHAJCaO33npLFRUVWrt2rT755BNNnTpVpaWlOnXqVCIOBwAY4BISRhs2bNDSpUv1xBNP6Bvf+IZeeeUVjRgxQr/5zW8ScTgAwAAX9zC6ePGiGhoaVFJScuUgKSkqKSlRXV3dVfuHw2GFQqGoBQAwuMQ9jE6fPq3u7m7l5uZGrc/NzVVra+tV+1dVVSkQCEQWbusGgMHHfAaGyspKtbe3R5Zjx45ZtwQA6GNxv7U7OztbQ4YMUVtbW9T6trY2BYPBq/b3+/3y+/3xbgMAMIDE/cooLS1N06ZNU3V1dWRdT0+PqqurVVxcHO/DAQCSQEI+9FpRUaFFixbp29/+tmbMmKGNGzeqs7NTTzzxRCIOBwAY4BISRg8//LD++te/as2aNWptbdU3v/lN7dq166qbGgAAkPrhRKlffCHUbM1nOiAAGMAuuS7VaOdX+oI/87vpAAAgjAAA5ggjAIA5wggAYI4wAgCYI4wAAOYIIwCAOcIIAGCOMAIAmCOMAADmCCMAgDnCCABgjjACAJgjjAAA5ggjAIA5wggAYI4wAgCYI4wAAOYIIwCAOcIIAGCOMAIAmCOMAADmCCMAgDnCCABgjjACAJgjjAAA5ggjAIA5wggAYI4wAgCYI4wAAOYIIwCAOcIIAGCOMAIAmCOMAADmCCMAgDnCCABgjjACAJgjjAAA5ggjAIA5wggAYI4wAgCYI4wAAOYIIwCAOcIIAGCOMAIAmCOMAADmCCMAgDnCCABgjjACAJgjjAAA5ggjAIA5wggAYI4wAgCYI4wAAOYIIwCAubiH0bp16+Tz+aKWiRMnxvswAIAkMjQRT3r33Xfr/fffv3KQoQk5DAAgSSQkJYYOHapgMJiIpwYAJKGEvGd05MgR5efna9y4cXr88cd19OjRa+4bDocVCoWiFgDA4BL3MCoqKtLWrVu1a9cubd68WS0tLbr33nvV0dHR6/5VVVUKBAKRpaCgIN4tAQD6OZ9zziXyAGfPntXYsWO1YcMGLVmy5Krt4XBY4XA48jgUCqmgoECzNV9DfamJbA0AkECXXJdqtFPt7e3KyMi47r4Jv7MgMzNTd911l5qamnrd7vf75ff7E90GAKAfS/jnjM6dO6fm5mbl5eUl+lAAgAEq7mH0zDPPqLa2Vn/605/0hz/8QQ8++KCGDBmiRx99NN6HAgAkibi/THf8+HE9+uijOnPmjEaNGqV77rlH9fX1GjVqVLwPBQBIEnEPozfffDPeTwkASHLMTQcAMEcYAQDMEUYAAHOEEQDAHGEEADBHGAEAzBFGAABzhBEAwBxhBAAwRxgBAMwRRgAAc4QRAMAcYQQAMJfwb3pF3/PF8M25F++d5Llm2LF2zzWSpKFDPJe4oyc816TcMsJzzaXWNs81AG4eV0YAAHOEEQDAHGEEADBHGAEAzBFGAABzhBEAwBxhBAAwRxgBAMwRRgAAc4QRAMAcYQQAMEcYAQDMMVFqEvrLD6d5rnn1B//guabHxfa7TIqvx3PNqJSw55o57z7tucYNHeu5RpL+qWSL55rvDT8f07H6ypwfruyT41S//I8x1XW5bs81f3f7PZ5r3KVLnmvgHVdGAABzhBEAwBxhBAAwRxgBAMwRRgAAc4QRAMAcYQQAMEcYAQDMEUYAAHOEEQDAHGEEADBHGAEAzBFGAABzzNqdhL77nw94rln37VLPNaH77vRcI0kZ/+sz70Wjg55L0kuHeK757z/0Pnu5JG09fa/nmvuGf+S5JtXn/c8Uq9TlrZ5rTu++zXON35fquUaS/trtfSb3lMyA55ru02c818A7rowAAOYIIwCAOcIIAGCOMAIAmCOMAADmCCMAgDnCCABgjjACAJgjjAAA5ggjAIA5wggAYI4wAgCYY6LUJPSnGZ/HUOW95pb/sS+G40jdsRR9GvJcEvz0f3uuWf2XJz3XSFJPDH+TSk/d7blm2MdHvB9IUs/nFzzX+N1fPNfkdx/1XNP9VI/nGkk6fmm45xoXvhjTsZB4XBkBAMwRRgAAc57DaM+ePXrggQeUn58vn8+nHTt2RG13zmnNmjXKy8vT8OHDVVJSoiNHYntpAQAwOHgOo87OTk2dOlWbNm3qdfv69ev18ssv65VXXtG+fft0yy23qLS0VBcueH/NGgAwOHh+27WsrExlZWW9bnPOaePGjXr++ec1f/58SdJrr72m3Nxc7dixQ4888sjNdQsASEpxfc+opaVFra2tKikpiawLBAIqKipSXV1drzXhcFihUChqAQAMLnENo9bWVklSbm5u1Prc3NzIti+rqqpSIBCILAUFBfFsCQAwAJjfTVdZWan29vbIcuzYMeuWAAB9LK5hFAwGJUltbW1R69va2iLbvszv9ysjIyNqAQAMLnENo8LCQgWDQVVXV0fWhUIh7du3T8XFxfE8FAAgiXi+m+7cuXNqamqKPG5padHBgweVlZWlMWPGaNWqVfrpT3+qO++8U4WFhXrhhReUn5+vBQsWxLNvAEAS8RxG+/fv13333Rd5XFFRIUlatGiRtm7dqmeffVadnZ1atmyZzp49q3vuuUe7du3SsGHD4tc1ACCp+JxzzrqJvxUKhRQIBDRb8zXUl2rdDgAPfH6/55oFB47HdKx/3vCfPNeMfLXe+4H61z+RA8ol16Ua7VR7e/sN7wcwv5sOAADCCABgjjACAJgjjAAA5ggjAIA5wggAYI4wAgCYI4wAAOYIIwCAOcIIAGCOMAIAmCOMAADmPM/aDWBwOPdfvuO55ux477/fLs/c57lGkl7/9x7vRUx62m9xZQQAMEcYAQDMEUYAAHOEEQDAHGEEADBHGAEAzBFGAABzhBEAwBxhBAAwRxgBAMwRRgAAc4QRAMAcYQQAMMes3QB6dWLeJc81b9232XPNNzY95blGkm4/3Oa5pjumI6EvcGUEADBHGAEAzBFGAABzhBEAwBxhBAAwRxgBAMwRRgAAc4QRAMAcYQQAMEcYAQDMEUYAAHOEEQDAHBOlAoNB9WjPJZnnznmu6XHef78d8/cNnmskqTscjqkO/RNXRgAAc4QRAMAcYQQAMEcYAQDMEUYAAHOEEQDAHGEEADBHGAEAzBFGAABzhBEAwBxhBAAwRxgBAMwxUSowCIwa7n3S05fGbfdc89zyFZ5rhs7o9lwjSSn/diCmOvRPXBkBAMwRRgAAc57DaM+ePXrggQeUn58vn8+nHTt2RG1fvHixfD5f1DJv3rx49QsASEKew6izs1NTp07Vpk2brrnPvHnzdPLkycjyxhtv3FSTAIDk5vkGhrKyMpWVlV13H7/fr2AwGHNTAIDBJSHvGdXU1CgnJ0cTJkzQihUrdObMmWvuGw6HFQqFohYAwOAS9zCaN2+eXnvtNVVXV+vnP/+5amtrVVZWpu7u3m/frKqqUiAQiCwFBQXxbgkA0M/F/XNGjzzySOTnyZMna8qUKRo/frxqamo0Z86cq/avrKxURUVF5HEoFCKQAGCQSfit3ePGjVN2draampp63e73+5WRkRG1AAAGl4SH0fHjx3XmzBnl5eUl+lAAgAHK88t0586di7rKaWlp0cGDB5WVlaWsrCy9+OKLWrhwoYLBoJqbm/Xss8/qjjvuUGlpaVwbBwAkD89htH//ft13332Rx1+837No0SJt3rxZhw4d0m9/+1udPXtW+fn5mjt3rl566SX5/f74dQ0ASCqew2j27Nlyzl1z++9///ubaggAMPgwazcwgKSMGBFTXfM/TvRc81/nj/FcU1h90HONu3TJcw2SDxOlAgDMEUYAAHOEEQDAHGEEADBHGAEAzBFGAABzhBEAwBxhBAAwRxgBAMwRRgAAc4QRAMAcYQQAMMdEqcAA4rstGFPdP/y3X3mu+ffuWz3X/PLS1z3XABJXRgCAfoAwAgCYI4wAAOYIIwCAOcIIAGCOMAIAmCOMAADmCCMAgDnCCABgjjACAJgjjAAA5ggjAIA5JkoFjPj8fs81w17tiOlYM/ypnmsea/luDEf6vzHUAFwZAQD6AcIIAGCOMAIAmCOMAADmCCMAgDnCCABgjjACAJgjjAAA5ggjAIA5wggAYI4wAgCYI4wAAOaYKBWIg/+zvthzza1HfZ5rfjtmg+caSWrvcZ5rzsxk0lP0Ha6MAADmCCMAgDnCCABgjjACAJgjjAAA5ggjAIA5wggAYI4wAgCYI4wAAOYIIwCAOcIIAGCOMAIAmCOMAADmmLUb+BtDx90eU53/jPcZuP/56Y2ea1LV47lGkmbtX+K5JqjPYjoWEAuujAAA5ggjAIA5T2FUVVWl6dOnKz09XTk5OVqwYIEaGxuj9rlw4YLKy8s1cuRI3XrrrVq4cKHa2tri2jQAILl4CqPa2lqVl5ervr5eu3fvVldXl+bOnavOzs7IPqtXr9a7776rt99+W7W1tTpx4oQeeuihuDcOAEgenm5g2LVrV9TjrVu3KicnRw0NDZo1a5ba29v16quvatu2bfr+978vSdqyZYu+/vWvq76+Xt/5zneues5wOKxwOBx5HAqFYvlzAAAGsJt6z6i9vV2SlJWVJUlqaGhQV1eXSkpKIvtMnDhRY8aMUV1dXa/PUVVVpUAgEFkKCgpupiUAwAAUcxj19PRo1apVmjlzpiZNmiRJam1tVVpamjIzM6P2zc3NVWtra6/PU1lZqfb29shy7NixWFsCAAxQMX/OqLy8XIcPH9bevXtvqgG/3y+/339TzwEAGNhiujJauXKl3nvvPX344YcaPXp0ZH0wGNTFixd19uzZqP3b2toUDAZvqlEAQPLyFEbOOa1cuVLbt2/XBx98oMLCwqjt06ZNU2pqqqqrqyPrGhsbdfToURUXF8enYwBA0vH0Ml15ebm2bdumnTt3Kj09PfI+UCAQ0PDhwxUIBLRkyRJVVFQoKytLGRkZevLJJ1VcXNzrnXQAAEgew2jz5s2SpNmzZ0et37JlixYvXixJ+uUvf6mUlBQtXLhQ4XBYpaWl+vWvfx2XZgEAycnnnHPWTfytUCikQCCg2Zqvob5U63YwyJx45rsx1f1u5d/HuZPebQ/9h5jqaqcMj3MnwI1dcl2q0U61t7crIyPjuvsyNx0AwBxhBAAwRxgBAMwRRgAAc4QRAMAcYQQAMEcYAQDMEUYAAHOEEQDAHGEEADBHGAEAzBFGAABzMX/TK9DfHf+Xuz3XDP232I7V4bxP6nu+x/s3HH90/x2eay77S4x1QN/gyggAYI4wAgCYI4wAAOYIIwCAOcIIAGCOMAIAmCOMAADmCCMAgDnCCABgjjACAJgjjAAA5ggjAIA5wggAYI5Zu5G0fj/9nzzXnP6W99m3JalbPs81L7X8R881Q48f9VwDDARcGQEAzBFGAABzhBEAwBxhBAAwRxgBAMwRRgAAc4QRAMAcYQQAMEcYAQDMEUYAAHOEEQDAHGEEADDHRKkYEPLr0z3XtPcMSUAnvZvmT/NcM3xol+ca7xXAwMCVEQDAHGEEADBHGAEAzBFGAABzhBEAwBxhBAAwRxgBAMwRRgAAc4QRAMAcYQQAMEcYAQDMEUYAAHNMlIoBYU3e//Rckz/U77nm8EXnuUaSuly355rPjgc919yhk55rgIGAKyMAgDnCCABgzlMYVVVVafr06UpPT1dOTo4WLFigxsbGqH1mz54tn88XtSxfvjyuTQMAkounMKqtrVV5ebnq6+u1e/dudXV1ae7cuers7Izab+nSpTp58mRkWb9+fVybBgAkF083MOzatSvq8datW5WTk6OGhgbNmjUrsn7EiBEKBr2/OQsAGJxu6j2j9vZ2SVJWVlbU+tdff13Z2dmaNGmSKisrdf78+Ws+RzgcVigUiloAAINLzLd29/T0aNWqVZo5c6YmTZoUWf/YY49p7Nixys/P16FDh/Tcc8+psbFR77zzTq/PU1VVpRdffDHWNgAASSDmMCovL9fhw4e1d+/eqPXLli2L/Dx58mTl5eVpzpw5am5u1vjx4696nsrKSlVUVEQeh0IhFRQUxNoWAGAAiimMVq5cqffee0979uzR6NGjr7tvUVGRJKmpqanXMPL7/fL7vX84EQCQPDyFkXNOTz75pLZv366amhoVFhbesObgwYOSpLy8vJgaBAAkP09hVF5erm3btmnnzp1KT09Xa2urJCkQCGj48OFqbm7Wtm3bdP/992vkyJE6dOiQVq9erVmzZmnKlCkJ+QMAAAY+T2G0efNmSZc/2Pq3tmzZosWLFystLU3vv/++Nm7cqM7OThUUFGjhwoV6/vnn49YwACD5eH6Z7noKCgpUW1t7Uw0BvRni817j96V6rpmS5n3CU0masHvZjXf6kjsXN8R0LCAZMTcdAMAcYQQAMEcYAQDMEUYAAHOEEQDAHGEEADBHGAEAzBFGAABzhBEAwBxhBAAwRxgBAMwRRgAAc4QRAMBczF87DvSlpWPusW7huu4UM3ADN4MrIwCAOcIIAGCOMAIAmCOMAADmCCMAgDnCCABgjjACAJgjjAAA5ggjAIA5wggAYI4wAgCY63dz0znnJEmX1CU542YAADG7pC5JV/5dv55+F0YdHR2SpL36V+NOAADx0NHRoUAgcN19fO6rRFYf6unp0YkTJ5Seni6fzxe1LRQKqaCgQMeOHVNGRoZRh/YYhysYi8sYhysYi8v6wzg459TR0aH8/HylpFz/XaF+d2WUkpKi0aNHX3efjIyMQX2SfYFxuIKxuIxxuIKxuMx6HG50RfQFbmAAAJgjjAAA5gZUGPn9fq1du1Z+v9+6FVOMwxWMxWWMwxWMxWUDbRz63Q0MAIDBZ0BdGQEAkhNhBAAwRxgBAMwRRgAAc4QRAMAcYQQAMEcYAQDMEUYAAHP/D/ifH2fb+rUyAAAAAElFTkSuQmCC\n"
          },
          "metadata": {}
        }
      ],
      "source": [
        "plt.matshow(x_test_adv[0, :, :, 0])\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "22fDvv7dZqhz"
      },
      "source": [
        "# TensorFlow with custom function"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uf5o2mVwZqh0"
      },
      "source": [
        "Reshape dataset in feature vectors because the model in this example requires feature vectors."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 43,
      "metadata": {
        "id": "kjo9-ONXZqh0"
      },
      "outputs": [],
      "source": [
        "x_train = x_train.reshape((60000, 784))\n",
        "x_test = x_test.reshape((x_test.shape[0], 784))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CxoZ583LZqh0"
      },
      "source": [
        "Create input pipelines for training and testing"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 45,
      "metadata": {
        "id": "VncrPWpVZqh0"
      },
      "outputs": [],
      "source": [
        "train_ds = tf.data.Dataset.from_tensor_slices((x_train, y_train)).shuffle(10000).batch(32)\n",
        "test_ds = tf.data.Dataset.from_tensor_slices((x_test, y_test)).batch(32)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "y3jKSVKRZqh0"
      },
      "source": [
        "Create variables and keep track of them"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 46,
      "metadata": {
        "id": "X9a9mKLLZqh0"
      },
      "outputs": [],
      "source": [
        "W = tf.Variable(initial_value=tf.random.normal(shape=(784, 10)), name=\"W\")\n",
        "b = tf.Variable(tf.zeros(shape=(10)), name=\"b\")"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "optimizer = tf.keras.optimizers.legacy.Adam()"
      ],
      "metadata": {
        "id": "G3tgJHIDcCZM"
      },
      "execution_count": 44,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "izFp8FS8Zqh0"
      },
      "source": [
        "Define a function representing the model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 47,
      "metadata": {
        "id": "CiFMVZBYZqh0"
      },
      "outputs": [],
      "source": [
        "@tf.function\n",
        "def forward(x, training=None):\n",
        "    x = tf.matmul(x, W) + b\n",
        "    denominator = tf.expand_dims(tf.reduce_sum(tf.exp(x), axis=1), axis=1)\n",
        "    softmax = (1.0 / denominator) * tf.exp(x)\n",
        "    return softmax"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rfQdl5vRZqh0"
      },
      "source": [
        "Define the training step."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 48,
      "metadata": {
        "id": "s0Lg-2Q3Zqh0"
      },
      "outputs": [],
      "source": [
        "@tf.function\n",
        "def train_step(images, labels):\n",
        "    with tf.GradientTape() as tape:\n",
        "        predictions = forward(images)\n",
        "        loss = loss_object(labels, predictions)\n",
        "    gradients = tape.gradient(loss, [W, b])\n",
        "    optimizer.apply_gradients(zip(gradients, [W, b]))\n",
        "\n",
        "    train_loss(loss)\n",
        "    train_accuracy(labels, predictions)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "A9hoHE1bZqh1"
      },
      "source": [
        "Define the testing step."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 49,
      "metadata": {
        "id": "DMdWnJVAZqh1"
      },
      "outputs": [],
      "source": [
        "@tf.function\n",
        "def test_step(images, labels):\n",
        "    predictions = forward(images)\n",
        "    t_loss = loss_object(labels, predictions)\n",
        "\n",
        "    test_loss(t_loss)\n",
        "    test_accuracy(labels, predictions)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "w7c618Y3Zqh1"
      },
      "source": [
        "Fit the model on training data and collect metrics for training and testing."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 50,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9j1NyOiRZqh1",
        "outputId": "2bc12a6b-c61f-4925-fb34-73264e179800"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1, Loss: 0.82, Accuracy: 85.42, Test Loss: 0.36, Test Accuracy: 87.50\n",
            "Epoch 2, Loss: 0.83, Accuracy: 84.71, Test Loss: 0.30, Test Accuracy: 90.00\n",
            "Epoch 3, Loss: 0.80, Accuracy: 84.90, Test Loss: 0.26, Test Accuracy: 91.67\n"
          ]
        }
      ],
      "source": [
        "epochs = 3\n",
        "\n",
        "for epoch in range(epochs):\n",
        "    for images, labels in train_ds:\n",
        "        train_step(images, labels)\n",
        "\n",
        "    for test_images, test_labels in test_ds:\n",
        "        test_step(test_images, test_labels)\n",
        "\n",
        "    template = 'Epoch {}, Loss: {:4.2f}, Accuracy: {:4.2f}, Test Loss: {:4.2f}, Test Accuracy: {:4.2f}'\n",
        "    print(template.format(epoch + 1,\n",
        "                          train_loss.result(),\n",
        "                          train_accuracy.result() * 100,\n",
        "                          test_loss.result(),\n",
        "                          test_accuracy.result() * 100))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0N1_z0iWZqh1"
      },
      "source": [
        "Evaluate model accuracy on test data."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 51,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kv95doyJZqh1",
        "outputId": "a4b8bcc4-db5d-4452-f868-7645ae714536"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Accuracy on test data: 100.00%\n"
          ]
        }
      ],
      "source": [
        "y_test_pred = np.argmax(forward(x_test), axis=1)\n",
        "accuracy_test = np.sum(y_test_pred == y_test) / y_test.shape[0]\n",
        "print('Accuracy on test data: {:4.2f}%'.format(accuracy_test * 100))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "R9sn03cBZqh1"
      },
      "source": [
        "Create a ART TensorFlow v2 classifier for the TensorFlow custom model function."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 52,
      "metadata": {
        "id": "D-yeLHmUZqh1"
      },
      "outputs": [],
      "source": [
        "classifier = TensorFlowV2Classifier(model=forward, nb_classes=10, input_shape=(28, 28, 1),\n",
        "                                    loss_object=loss_object, clip_values=(0, 1), channels_first=False)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "84NYQCFiZqh1"
      },
      "source": [
        "## Fast Gradient Sign Method attack"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mAXJqjEdZqh1"
      },
      "source": [
        "Create a ART Fast Gradient Sign Method attack."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 53,
      "metadata": {
        "id": "7wPINgYfZqh2"
      },
      "outputs": [],
      "source": [
        "attack_fgsm = FastGradientMethod(estimator=classifier)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gqulQbmyZqh2"
      },
      "source": [
        "Generate adversarial test data."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 54,
      "metadata": {
        "scrolled": false,
        "id": "34Xm1s1EZqh2"
      },
      "outputs": [],
      "source": [
        "x_test_adv = attack_fgsm.generate(x_test)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NBwMJdqtZqh2"
      },
      "source": [
        "Evaluate accuracy on adversarial test data and calculate average perturbation."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 55,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eVmSLBAhZqh2",
        "outputId": "49cfc691-2e36-4a8b-febe-6836dc6c5b2e"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Accuracy on adversarial test data: 0.00%\n",
            "Average perturbation: 0.17\n"
          ]
        }
      ],
      "source": [
        "y_test_pred = np.argmax(forward(x_test_adv), axis=1)\n",
        "accuracy_test_adv = np.sum(y_test_pred == y_test) / y_test.shape[0]\n",
        "perturbation = np.mean(np.abs((x_test_adv - x_test)))\n",
        "print('Accuracy on adversarial test data: {:4.2f}%'.format(accuracy_test_adv * 100))\n",
        "print('Average perturbation: {:4.2f}'.format(perturbation))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NCOIsVdJZqh2"
      },
      "source": [
        "Visualise the first adversarial test sample."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 56,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 437
        },
        "id": "6h8gmnhKZqh2",
        "outputId": "fde6426b-59be-4268-f9b2-a818cec6e08d"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 480x480 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaMAAAGkCAYAAACckEpMAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAfz0lEQVR4nO3df3DU9b3v8dcSyIKaLI2R/CgBAypY+WGlEjkoxZIS0hkOKNMRdc4BjwdGG7xFavWmVUHbnrR0rnXsULz3TAv1jIg6FRi9LR2MJowtPw4IZWhtLuSmBS4kVE7ZhWBCTD73D4bFSMB8v+x+398sz8fMdyS730++7/3sZ/Pym/3mvRHnnBMAAIb6WRcAAABhBAAwRxgBAMwRRgAAc4QRAMAcYQQAMEcYAQDMEUYAAHOEEQDAHGEEADDXZ8JoxYoVuvbaazVw4ECVlZVp+/bt1iUFbtmyZYpEIt220aNHW5cViM2bN2vmzJkqLi5WJBLR+vXru93vnNPTTz+toqIiDRo0SOXl5dq3b59NsWn0WfMwf/7889bIjBkzbIpNo5qaGt16663KycnRkCFDNHv2bDU0NHTbp62tTVVVVbr66qt11VVXac6cOWppaTGqOD16Mw9Tp049b0089NBDRhVfWJ8Io1dffVVLlizR0qVL9f7772v8+PGqqKjQ0aNHrUsL3E033aQjR44kt/fee8+6pEC0trZq/PjxWrFiRY/3L1++XC+88IJefPFFbdu2TVdeeaUqKirU1tYWcKXp9VnzIEkzZszotkZeeeWVACsMRn19vaqqqrR161Zt2rRJHR0dmj59ulpbW5P7PProo3rzzTf1+uuvq76+XocPH9bdd99tWHXq9WYeJGnBggXd1sTy5cuNKr4I1wdMnDjRVVVVJb/u7Ox0xcXFrqamxrCq4C1dutSNHz/eugxzkty6deuSX3d1dbnCwkL34x//OHnb8ePHXTQada+88opBhcH49Dw459y8efPcrFmzTOqxdPToUSfJ1dfXO+fOPP8DBgxwr7/+enKfDz74wElyW7ZssSoz7T49D8459+Uvf9l985vftCuql0J/ZnT69Gnt3LlT5eXlydv69eun8vJybdmyxbAyG/v27VNxcbFGjBih+++/XwcOHLAuyVxTU5Oam5u7rZFYLKaysrLLco3U1dVpyJAhGjVqlB5++GEdO3bMuqS0i8fjkqS8vDxJ0s6dO9XR0dFtTYwePVrDhg3L6DXx6Xk46+WXX1Z+fr7GjBmj6upqnTp1yqK8i+pvXcBn+fDDD9XZ2amCgoJutxcUFOjPf/6zUVU2ysrKtHr1ao0aNUpHjhzRM888ozvuuEN79+5VTk6OdXlmmpubJanHNXL2vsvFjBkzdPfdd6u0tFSNjY36zne+o8rKSm3ZskVZWVnW5aVFV1eXFi9erMmTJ2vMmDGSzqyJ7OxsDR48uNu+mbwmepoHSbrvvvs0fPhwFRcXa8+ePXriiSfU0NCgN954w7Da84U+jHBOZWVl8t/jxo1TWVmZhg8frtdee00PPvigYWUIi7lz5yb/PXbsWI0bN04jR45UXV2dpk2bZlhZ+lRVVWnv3r2XzfunF3KheVi4cGHy32PHjlVRUZGmTZumxsZGjRw5MugyLyj0v6bLz89XVlbWeVfBtLS0qLCw0KiqcBg8eLBuuOEG7d+/37oUU2fXAWvkfCNGjFB+fn7GrpFFixbprbfe0rvvvquhQ4cmby8sLNTp06d1/Pjxbvtn6pq40Dz0pKysTJJCtyZCH0bZ2dmaMGGCamtrk7d1dXWptrZWkyZNMqzM3smTJ9XY2KiioiLrUkyVlpaqsLCw2xpJJBLatm3bZb9GDh06pGPHjmXcGnHOadGiRVq3bp3eeecdlZaWdrt/woQJGjBgQLc10dDQoAMHDmTUmviseejJ7t27JSl8a8L6CoreWLt2rYtGo2716tXuT3/6k1u4cKEbPHiwa25uti4tUN/61rdcXV2da2pqcr/73e9ceXm5y8/Pd0ePHrUuLe1OnDjhdu3a5Xbt2uUkueeee87t2rXL/fWvf3XOOffDH/7QDR482G3YsMHt2bPHzZo1y5WWlrqPPvrIuPLUutg8nDhxwj322GNuy5Ytrqmpyb399tvulltucddff71ra2uzLj2lHn74YReLxVxdXZ07cuRIcjt16lRyn4ceesgNGzbMvfPOO27Hjh1u0qRJbtKkSYZVp95nzcP+/fvds88+63bs2OGamprchg0b3IgRI9yUKVOMKz9fnwgj55z76U9/6oYNG+ays7PdxIkT3datW61LCtw999zjioqKXHZ2tvv85z/v7rnnHrd//37rsgLx7rvvOknnbfPmzXPOnbm8+6mnnnIFBQUuGo26adOmuYaGBtui0+Bi83Dq1Ck3ffp0d80117gBAwa44cOHuwULFmTk/7T1NAeS3KpVq5L7fPTRR+4b3/iG+9znPueuuOIKd9ddd7kjR47YFZ0GnzUPBw4ccFOmTHF5eXkuGo266667zn3729928XjctvAeRJxzLrjzMAAAzhf694wAAJmPMAIAmCOMAADmCCMAgDnCCABgjjACAJjrU2HU3t6uZcuWqb293boUU8zDOczFGczDOczFGX1tHvrU3xklEgnFYjHF43Hl5uZal2OGeTiHuTiDeTiHuTijr81DnzozAgBkJsIIAGAudJ9n1NXVpcOHDysnJ0eRSKTbfYlEott/L1fMwznMxRnMwznMxRlhmAfnnE6cOKHi4mL163fxc5/QvWd06NAhlZSUWJcBAEiRgwcPfubnLIXuzOjsx2ffrq+pvwak/XjxuRPTfgxJiq3d7nmM39qCPFam8TN3UvjnL6g1kYnr3O+a8CPMP48k7/V1drRpz6++l/y5fjGhC6Ozv5rrrwHqH0l/GGVlD0z7MST5eix+awvyWJnG75oL+/wFtSYycZ0H8XPorDD/PJL81/fpt1x6krYLGFasWKFrr71WAwcOVFlZmbZvD+7/LgAAfUtawujVV1/VkiVLtHTpUr3//vsaP368KioqdPTo0XQcDgDQx6UljJ577jktWLBADzzwgL7whS/oxRdf1BVXXKFf/OIX6TgcAKCPS3kYnT59Wjt37lR5efm5g/Trp/Lycm3ZsuW8/dvb25VIJLptAIDLS8rD6MMPP1RnZ6cKCgq63V5QUKDm5ubz9q+pqVEsFktuXNYNAJcf8w4M1dXVisfjye3gwYPWJQEAApbyS7vz8/OVlZWllpaWbre3tLSosLDwvP2j0aii0WiqywAA9CEpPzPKzs7WhAkTVFtbm7ytq6tLtbW1mjRpUqoPBwDIAGn5o9clS5Zo3rx5+tKXvqSJEyfq+eefV2trqx544IF0HA4A0MelJYzuuece/e1vf9PTTz+t5uZm3Xzzzdq4ceN5FzUAACCFsFHq2Q+EmqpZnlpWHP8nf78CHPwf519u3tf5mYsg5yGo+vyuiaCE/TEFVV/YX4NhX0dh1nm6TbvWfrdXH/BnfjUdAACEEQDAHGEEADBHGAEAzBFGAABzhBEAwBxhBAAwRxgBAMwRRgAAc4QRAMAcYQQAMEcYAQDMEUYAAHNp+QiJVIjPnais7IG93j/sXafDLuyPKaj6MnUdhb1DuB9hr4+u59LHrqPX+3JmBAAwRxgBAMwRRgAAc4QRAMAcYQQAMEcYAQDMEUYAAHOEEQDAHGEEADBHGAEAzBFGAABzhBEAwFxoG6XG1m5X/8iAtB8n0xoTSuF/TGFucJmpzUuDWhNhfm6lzFznYZ7zztNt0toNvdqXMyMAgDnCCABgjjACAJgjjAAA5ggjAIA5wggAYI4wAgCYI4wAAOYIIwCAOcIIAGCOMAIAmCOMAADmQtso1asgmwVmYtPJsDdXDbugnl+/cx7mZpphrk3K3Oa5YcOZEQDAHGEEADBHGAEAzBFGAABzhBEAwBxhBAAwRxgBAMwRRgAAc4QRAMAcYQQAMEcYAQDMEUYAAHOhbZQanztRWdkDe71/kA0kaRB6RtgbNAb5PAU1F36Pw5rNXEE2V/V6rI9dR6/35cwIAGCOMAIAmEt5GC1btkyRSKTbNnr06FQfBgCQQdLyntFNN92kt99++9xB+of2rSkAQAikJSX69++vwsLCdHxrAEAGSst7Rvv27VNxcbFGjBih+++/XwcOHLjgvu3t7UokEt02AMDlJeVhVFZWptWrV2vjxo1auXKlmpqadMcdd+jEiRM97l9TU6NYLJbcSkpKUl0SACDkUh5GlZWV+vrXv65x48apoqJCv/71r3X8+HG99tprPe5fXV2teDye3A4ePJjqkgAAIZf2KwsGDx6sG264Qfv37+/x/mg0qmg0mu4yAAAhlva/Mzp58qQaGxtVVFSU7kMBAPqolIfRY489pvr6ev3lL3/R73//e911113KysrSvffem+pDAQAyRMp/TXfo0CHde++9OnbsmK655hrdfvvt2rp1q6655ppUHwoAkCFSHkZr165N9bcEAGS4iHPOWRfxSYlEQrFYTFM1S/0jA9J+vKC6LYe9a3KQ3cvD3Ck9yE7kQXZb9iOo+oJcR35kYvf3oHSebtOutd9VPB5Xbm7uRfelUSoAwBxhBAAwRxgBAMwRRgAAc4QRAMAcYQQAMEcYAQDMEUYAAHOEEQDAHGEEADBHGAEAzBFGAABzaf+kV7/icycqK3ugdRkpE2QDxLxf/cHzmG0/Wul5zO5n2z2PkaSjnX/0POZ//MdNnsccWPYPnsfkNoaqb/B5gmwqGlTTU7+vjbA3moU3nBkBAMwRRgAAc4QRAMAcYQQAMEcYAQDMEUYAAHOEEQDAHGEEADBHGAEAzBFGAABzhBEAwBxhBAAwF9pGqbG129U/MsC6jD7pN4d3ex7z1Q9mpr6QFNp0+E3PY774n6M9j/mvgZ/zPEaS9v2z90az19/0sK9j+eGrvpe817fNx9q7/iV/zUv/98HfeR5zf4mvQwXmcm7+ypkRAMAcYQQAMEcYAQDMEUYAAHOEEQDAHGEEADBHGAEAzBFGAABzhBEAwBxhBAAwRxgBAMwRRgAAc4QRAMBcaLt2x+dOVFb2wLQfJ6guuX6O41fpxn/1PGbUTz/yPGbfkmzPYyRp8nWNnsdMeMZ7B+kTU9o8jym99ZDnMZI0svYBz2MafXTS9stPV/aiLzanoZLzXetzzu/9P3M9j5n7wX96HvO//u0uz2OkcHfgDurn0ceuo9f7cmYEADBHGAEAzBFGAABzhBEAwBxhBAAwRxgBAMwRRgAAc4QRAMAcYQQAMEcYAQDMEUYAAHOEEQDAXGgbpcbWblf/yADrMkz5bZp4w794b4L4dx/Huu6f/DVbbPExJl/ej5X/P30cyKfrdNDzmNJfeG9oe8O/7PA8RpL6+ahvkI/jVOhmz2MO/yrXx5GkP0562fOYf9w3w/MYv01Fg2qoHFRzVT/H6jzdJq3d0Kt9OTMCAJgjjAAA5jyH0ebNmzVz5kwVFxcrEolo/fr13e53zunpp59WUVGRBg0apPLycu3bty9V9QIAMpDnMGptbdX48eO1YsWKHu9fvny5XnjhBb344ovatm2brrzySlVUVKitzfsHnQEALg+eL2CorKxUZWVlj/c55/T888/rySef1KxZsyRJL730kgoKCrR+/XrNnev9kxkBAJkvpe8ZNTU1qbm5WeXl5cnbYrGYysrKtGVLz1eJtLe3K5FIdNsAAJeXlIZRc3OzJKmgoKDb7QUFBcn7Pq2mpkaxWCy5lZSUpLIkAEAfYH41XXV1teLxeHI7eND730MAAPq2lIZRYWGhJKmlpfufNba0tCTv+7RoNKrc3NxuGwDg8pLSMCotLVVhYaFqa2uTtyUSCW3btk2TJgX3V8IAgL7F89V0J0+e1P79+5NfNzU1affu3crLy9OwYcO0ePFiff/739f111+v0tJSPfXUUyouLtbs2bNTWTcAIIN4DqMdO3bozjvvTH69ZMkSSdK8efO0evVqPf7442ptbdXChQt1/Phx3X777dq4caMGDhyYuqoBABnFcxhNnTpVzrkL3h+JRPTss8/q2WefvaTCvPLbLNBvE0RkpiDXkd+mp34E1Uwz7/VdnscUxk6koZKetX+556t6wyITm6v2lvnVdAAAEEYAAHOEEQDAHGEEADBHGAEAzBFGAABzhBEAwBxhBAAwRxgBAMwRRgAAc4QRAMAcYQQAMOe5UWpQ4nMnKis7nJ2+M7G5apDNFjOlseMnZWKDSz/1/ebw7tQXcgGVI27zMarN84hMbMLst7Z0rj/OjAAA5ggjAIA5wggAYI4wAgCYI4wAAOYIIwCAOcIIAGCOMAIAmCOMAADmCCMAgDnCCABgjjACAJgjjAAA5kLbtTu2drv6Rwb0en+/3WSD6rbsR5i7/l6KMM95kF2xw96J/B/+cNrzmK9+MNPzmOZEjucxktS+LOZ5TOl/z7zXVJg7uX/sOnq9L2dGAABzhBEAwBxhBAAwRxgBAMwRRgAAc4QRAMAcYQQAMEcYAQDMEUYAAHOEEQDAHGEEADBHGAEAzIW2UapXmdhUNMwNEIMW5uaqQQryMW26f7TnMe+Ne8PzmIrimz2PkfytibA3p/XDz5oI4zxwZgQAMEcYAQDMEUYAAHOEEQDAHGEEADBHGAEAzBFGAABzhBEAwBxhBAAwRxgBAMwRRgAAc4QRAMBcxDnnrIv4pEQioVgspi/O/YGysgf2epzfBpJhbBh4ViY2+vQrzM+TX0E2uLzibx97HtP/282exxyqL/E85qpD/n4E9evwPibI11QmrlmvOk+3adfa7yoejys3N/ei+3JmBAAwRxgBAMx5DqPNmzdr5syZKi4uViQS0fr167vdP3/+fEUikW7bjBkzUlUvACADeQ6j1tZWjR8/XitWrLjgPjNmzNCRI0eS2yuvvHJJRQIAMpvnT3qtrKxUZWXlRfeJRqMqLCz0XRQA4PKSlveM6urqNGTIEI0aNUoPP/ywjh07dsF929vblUgkum0AgMtLysNoxowZeumll1RbW6sf/ehHqq+vV2VlpTo7O3vcv6amRrFYLLmVlHi/NBQA0Ld5/jXdZ5k7d27y32PHjtW4ceM0cuRI1dXVadq0aeftX11drSVLliS/TiQSBBIAXGbSfmn3iBEjlJ+fr/379/d4fzQaVW5ubrcNAHB5SXsYHTp0SMeOHVNRUVG6DwUA6KM8/5ru5MmT3c5ympqatHv3buXl5SkvL0/PPPOM5syZo8LCQjU2Nurxxx/Xddddp4qKipQWDgDIHJ7DaMeOHbrzzjuTX599v2fevHlauXKl9uzZo1/+8pc6fvy4iouLNX36dH3ve99TNBpNXdUAgIziOYymTp2qi/VW/e1vf3tJBQEALj8pv5ruchBUt2W/XX+Dqi/ITulBdVsOe6dlv/Pw28O7PY+ZvOduz2PaCr13Bx/27HbPY6RgX1NhlimdyGmUCgAwRxgBAMwRRgAAc4QRAMAcYQQAMEcYAQDMEUYAAHOEEQDAHGEEADBHGAEAzBFGAABzhBEAwFzGNEoNsgFiJjZbxBlBNn/1o3PqLT5H7vY84nh9oecxQw5duKN/X5WJDYHDiDMjAIA5wggAYI4wAgCYI4wAAOYIIwCAOcIIAGCOMAIAmCOMAADmCCMAgDnCCABgjjACAJgjjAAA5iLOuVB1NkwkEorFYpqqWeofGZD24wXV4DLIBohhb9AY5jn3W5ufYzWuudnzmKH5xz2PkaS/nxrkecyJ/5frecw1W4P7/9sgn18/gnpNhfn1/rHrUJ02KB6PKzf34uuJMyMAgDnCCABgjjACAJgjjAAA5ggjAIA5wggAYI4wAgCYI4wAAOYIIwCAOcIIAGCOMAIAmCOMAADm+lsXkCpBNrj0c6wgGzT6EfYGkkHNeZANYzvbszyPGZDV6etYb97y757HLJh9u69jeeV37YX9NeVHUGs2yJ+XvcWZEQDAHGEEADBHGAEAzBFGAABzhBEAwBxhBAAwRxgBAMwRRgAAc4QRAMAcYQQAMEcYAQDMEUYAAHOEEQDAXGi7dsfnTlRW9sC0HycTO//6EWS36qDm3M9j+vt8f7Vt/7eVnsd89YMSz2M23fim5zGSVFHsvQN32F8bQXarzjRBvt57izMjAIA5wggAYM5TGNXU1OjWW29VTk6OhgwZotmzZ6uhoaHbPm1tbaqqqtLVV1+tq666SnPmzFFLS0tKiwYAZBZPYVRfX6+qqipt3bpVmzZtUkdHh6ZPn67W1tbkPo8++qjefPNNvf7666qvr9fhw4d19913p7xwAEDm8HQBw8aNG7t9vXr1ag0ZMkQ7d+7UlClTFI/H9fOf/1xr1qzRV77yFUnSqlWrdOONN2rr1q267bbbzvue7e3tam9vT36dSCT8PA4AQB92Se8ZxeNxSVJeXp4kaefOnero6FB5eXlyn9GjR2vYsGHasqXnqzdqamoUi8WSW0mJ9yuMAAB9m+8w6urq0uLFizV58mSNGTNGktTc3Kzs7GwNHjy4274FBQVqbm7u8ftUV1crHo8nt4MHD/otCQDQR/n+O6Oqqirt3btX77333iUVEI1GFY1GL+l7AAD6Nl9nRosWLdJbb72ld999V0OHDk3eXlhYqNOnT+v48ePd9m9paVFhYeElFQoAyFyewsg5p0WLFmndunV65513VFpa2u3+CRMmaMCAAaqtrU3e1tDQoAMHDmjSJP7yGQDQM0+/pquqqtKaNWu0YcMG5eTkJN8HisViGjRokGKxmB588EEtWbJEeXl5ys3N1SOPPKJJkyb1eCUdAACSxzBaufJM/62pU6d2u33VqlWaP3++JOknP/mJ+vXrpzlz5qi9vV0VFRX62c9+lpJiAQCZKeKcc9ZFfFIikVAsFtMX5/7AU6PUMDb++yQ/DRozsXmpFO4Gl37nvKs2mKanFcU3ex7jV9jXrB9hb5Qa5teG5L2+j12H6rRB8Xhcubm5F92X3nQAAHOEEQDAHGEEADBHGAEAzBFGAABzhBEAwBxhBAAwRxgBAMwRRgAAc4QRAMAcYQQAMEcYAQDM+f6k10wR5gacYW6AKPmvL8xzvu+lW3wd6//e+AvPYx44cIfnMcf/eYznMZKkgNoh04j00o4V1HHC+DxxZgQAMEcYAQDMEUYAAHOEEQDAHGEEADBHGAEAzBFGAABzhBEAwBxhBAAwRxgBAMwRRgAAc4QRAMAcYQQAMJcxXbvD2IW2L8nEDuGnZ9zqeczIzzd7HuPX4dtOeB4zWP46QYf59eG3u3WYu79L4Z7zMOLMCABgjjACAJgjjAAA5ggjAIA5wggAYI4wAgCYI4wAAOYIIwCAOcIIAGCOMAIAmCOMAADmCCMAgLmMaZTqV1BNO8PeiNSPIB/T0cmdnsc0/eO/ex7z1Q9meh7j1/7nb/M85rrFW30dK6g14YffdRTUazdIYX/teh3XebpNWruhV/tyZgQAMEcYAQDMEUYAAHOEEQDAHGEEADBHGAEAzBFGAABzhBEAwBxhBAAwRxgBAMwRRgAAc4QRAMBcxDnnrIv4pEQioVgspqmapf6RAb0eRyPSSxPmRpqSNGpH79fCWR/ECz2P2XTjm57HSFLpb/7V85jsI94f07VP+nue/KylsK8JP8LeKNWPIJ8nP41Sd639ruLxuHJzcy+6L2dGAABzhBEAwJynMKqpqdGtt96qnJwcDRkyRLNnz1ZDQ0O3faZOnapIJNJte+ihh1JaNAAgs3gKo/r6elVVVWnr1q3atGmTOjo6NH36dLW2tnbbb8GCBTpy5EhyW758eUqLBgBkFk+f9Lpx48ZuX69evVpDhgzRzp07NWXKlOTtV1xxhQoLvb95DAC4PF3Se0bxeFySlJeX1+32l19+Wfn5+RozZoyqq6t16tSpC36P9vZ2JRKJbhsA4PLi6czok7q6urR48WJNnjxZY8aMSd5+3333afjw4SouLtaePXv0xBNPqKGhQW+88UaP36empkbPPPOM3zIAABnAdxhVVVVp7969eu+997rdvnDhwuS/x44dq6KiIk2bNk2NjY0aOXLked+nurpaS5YsSX6dSCRUUlLitywAQB/kK4wWLVqkt956S5s3b9bQoUMvum9ZWZkkaf/+/T2GUTQaVTQa9VMGACBDeAoj55weeeQRrVu3TnV1dSotLf3MMbt375YkFRUV+SoQAJD5PIVRVVWV1qxZow0bNignJ0fNzc2SpFgspkGDBqmxsVFr1qzR1772NV199dXas2ePHn30UU2ZMkXjxo1LywMAAPR9nsJo5cqVks78YesnrVq1SvPnz1d2drbefvttPf/882ptbVVJSYnmzJmjJ598MmUFAwAyT2gbpX5x7g+UlT3Qupwe+WlMmImNKoNsOhlrvPCfB1yI+95/eR7T+Mdiz2Mk6fr/ts3zmExcE374XUfMxRlhnoePXYfqtIFGqQCAvoEwAgCYI4wAAOYIIwCAOcIIAGCOMAIAmCOMAADmCCMAgDnCCABgjjACAJgjjAAA5ggjAIA5wggAYM73x46HTZCda4PsVo0zIr//g+cx8dXen6frfa6joNZEkB2ug3pMvHbPCfPz5Efn6TZp7YZe7cuZEQDAHGEEADBHGAEAzBFGAABzhBEAwBxhBAAwRxgBAMwRRgAAc4QRAMAcYQQAMEcYAQDMha43nXNOktTZ0eZp3MeuIx3l9KjztLfa/AryMfkR1DxI/ubCT31+5zzIufAjqPnzIxNfu36F+Xny4+zP8bM/1y8m4nqzV4AOHTqkkpIS6zIAACly8OBBDR069KL7hC6Murq6dPjwYeXk5CgSiXS7L5FIqKSkRAcPHlRubq5RhfaYh3OYizOYh3OYizPCMA/OOZ04cULFxcXq1+/i7wqF7td0/fr1+8wEzc3NvawX2VnMwznMxRnMwznMxRnW8xCLxXq1HxcwAADMEUYAAHN9Koyi0aiWLl2qaDRqXYop5uEc5uIM5uEc5uKMvjYPobuAAQBw+elTZ0YAgMxEGAEAzBFGAABzhBEAwBxhBAAwRxgBAMwRRgAAc4QRAMDc/weUsGzR+5OzOQAAAABJRU5ErkJggg==\n"
          },
          "metadata": {}
        }
      ],
      "source": [
        "plt.matshow(x_test_adv[0, :].reshape((28, 28)))\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YL3wLEw8Zqh2"
      },
      "source": [
        "## Carlini&Wagner Infinity-norm attack"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "l38Wnq_RZqh2"
      },
      "source": [
        "Create a ART Carlini&Wagner Infinity-norm attack."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 57,
      "metadata": {
        "id": "WP04Zj2kZqh2"
      },
      "outputs": [],
      "source": [
        "attack_cw = CarliniLInfMethod(classifier=classifier,\n",
        "                              max_iter=100,\n",
        "                              learning_rate=0.01,\n",
        "                              initial_const=1e0,\n",
        "                              largest_const=2e0)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DiWXJGsDZqh3"
      },
      "source": [
        "Generate adversarial test data."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 58,
      "metadata": {
        "id": "-GnzPg79Zqh3"
      },
      "outputs": [],
      "source": [
        "%%capture\n",
        "x_test_adv = attack_cw.generate(x_test);"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5ZcIstoKZqh3"
      },
      "source": [
        "Evaluate accuracy on adversarial test data and calculate average perturbation."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 59,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-oJCiZ2GZqh3",
        "outputId": "43b2beab-fb9f-43ea-a6a1-d7669f82395c"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Accuracy on adversarial test data: 0.00%\n",
            "Average perturbation: 0.009456\n"
          ]
        }
      ],
      "source": [
        "y_test_pred = np.argmax(forward(x_test_adv), axis=1)\n",
        "accuracy_test_adv = np.sum(y_test_pred == y_test) / y_test.shape[0]\n",
        "perturbation = np.mean(np.abs((x_test_adv - x_test)))\n",
        "print('Accuracy on adversarial test data: {:4.2f}%'.format(accuracy_test_adv * 100))\n",
        "print('Average perturbation: {:4.6f}'.format(perturbation))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yegSrKeuZqh3"
      },
      "source": [
        "Visualise the first adversarial test sample."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 60,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 437
        },
        "id": "MUgtIpuOZqh3",
        "outputId": "ae27acfd-3a8b-492a-a4b0-834500964bec"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 480x480 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaMAAAGkCAYAAACckEpMAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAb70lEQVR4nO3df3BV9f3n8dcNJBeQ5KYhJjcpAQMqqEDcUo0pSlEykLjjgrIz+GO/C65fGDH4LaZWJ10VbTubls5S1k4Ks7stqTuilu8IjE4HB6MJpSb4BWFZtm2WpLHAQkKlk9wQICTks3+wXHolgOdyb97JzfMxc8bcc877njcfT3hx7j33c33OOScAAAwlWTcAAABhBAAwRxgBAMwRRgAAc4QRAMAcYQQAMEcYAQDMEUYAAHOEEQDAHGEEADA3ZMKoqqpKN910k0aNGqXCwkJ9+umn1i0NuFdffVU+ny9imTp1qnVbA2Lnzp166KGHlJubK5/Pp61bt0Zsd87plVdeUU5OjkaPHq3i4mIdOnTIptk4utY4LF269LJzpKSkxKbZOKqsrNRdd92l1NRUZWVlaeHChWpsbIzY5+zZsyorK9O4ceM0duxYLVq0SG1tbUYdx8dXGYc5c+Zcdk48/fTTRh1f2ZAIo3feeUfl5eVavXq1PvvsMxUUFGj+/Pk6ceKEdWsD7o477tDx48fDy65du6xbGhBdXV0qKChQVVVVv9vXrFmj119/XRs2bNDu3bt1ww03aP78+Tp79uwAdxpf1xoHSSopKYk4R956660B7HBg1NXVqaysTA0NDdqxY4d6eno0b948dXV1hfd57rnn9N5772nz5s2qq6vTsWPH9Mgjjxh2HXtfZRwkadmyZRHnxJo1a4w6vgo3BNx9992urKws/Pj8+fMuNzfXVVZWGnY18FavXu0KCgqs2zAnyW3ZsiX8uK+vzwWDQffTn/40vK69vd35/X731ltvGXQ4ML48Ds45t2TJErdgwQKTfiydOHHCSXJ1dXXOuQv//5OTk93mzZvD+/zxj390klx9fb1Vm3H35XFwzrlvf/vb7jvf+Y5dU1/RoL8yOnfunPbu3avi4uLwuqSkJBUXF6u+vt6wMxuHDh1Sbm6uJk2apCeeeEKHDx+2bslcS0uLWltbI86RQCCgwsLCYXmO1NbWKisrS1OmTNGKFSt08uRJ65birqOjQ5KUkZEhSdq7d696enoizompU6dqwoQJCX1OfHkcLnrzzTeVmZmpadOmqaKiQqdPn7Zo76pGWjdwLV988YXOnz+v7OzsiPXZ2dn605/+ZNSVjcLCQlVXV2vKlCk6fvy4XnvtNd133306ePCgUlNTrdsz09raKkn9niMXtw0XJSUleuSRR5Sfn6/m5mZ9//vfV2lpqerr6zVixAjr9uKir69Pq1at0qxZszRt2jRJF86JlJQUpaenR+ybyOdEf+MgSY8//rgmTpyo3NxcHThwQC+++KIaGxv17rvvGnZ7uUEfRriktLQ0/POMGTNUWFioiRMn6je/+Y2eeuopw84wWDz66KPhn6dPn64ZM2Zo8uTJqq2t1dy5cw07i5+ysjIdPHhw2Lx/eiVXGofly5eHf54+fbpycnI0d+5cNTc3a/LkyQPd5hUN+pfpMjMzNWLEiMvugmlra1MwGDTqanBIT0/XrbfeqqamJutWTF08DzhHLjdp0iRlZmYm7DmycuVKvf/++/r44481fvz48PpgMKhz586pvb09Yv9EPSeuNA79KSwslKRBd04M+jBKSUnRzJkzVVNTE17X19enmpoaFRUVGXZm79SpU2publZOTo51K6by8/MVDAYjzpFQKKTdu3cP+3Pk6NGjOnnyZMKdI845rVy5Ulu2bNFHH32k/Pz8iO0zZ85UcnJyxDnR2Niow4cPJ9Q5ca1x6M/+/fslafCdE9Z3UHwVb7/9tvP7/a66utr94Q9/cMuXL3fp6emutbXVurUB9d3vftfV1ta6lpYW9/vf/94VFxe7zMxMd+LECevW4q6zs9Pt27fP7du3z0lya9eudfv27XN/+ctfnHPO/fjHP3bp6elu27Zt7sCBA27BggUuPz/fnTlzxrjz2LraOHR2drrnn3/e1dfXu5aWFvfhhx+6b3zjG+6WW25xZ8+etW49plasWOECgYCrra11x48fDy+nT58O7/P000+7CRMmuI8++sjt2bPHFRUVuaKiIsOuY+9a49DU1OR+8IMfuD179riWlha3bds2N2nSJDd79mzjzi83JMLIOed+/vOfuwkTJriUlBR39913u4aGBuuWBtzixYtdTk6OS0lJcV//+tfd4sWLXVNTk3VbA+Ljjz92ki5blixZ4py7cHv3yy+/7LKzs53f73dz5851jY2Ntk3HwdXG4fTp027evHnuxhtvdMnJyW7ixIlu2bJlCfmPtv7GQJLbuHFjeJ8zZ864Z555xn3ta19zY8aMcQ8//LA7fvy4XdNxcK1xOHz4sJs9e7bLyMhwfr/f3Xzzze573/ue6+josG28Hz7nnBu46zAAAC436N8zAgAkPsIIAGCOMAIAmCOMAADmCCMAgDnCCABgbkiFUXd3t1599VV1d3dbt2KKcbiEsbiAcbiEsbhgqI3DkPqcUSgUUiAQUEdHh9LS0qzbMcM4XMJYXMA4XMJYXDDUxmFIXRkBABITYQQAMDfovs+or69Px44dU2pqqnw+X8S2UCgU8d/hinG4hLG4gHG4hLG4YDCMg3NOnZ2dys3NVVLS1a99Bt17RkePHlVeXp51GwCAGDly5Mg1v2dp0F0ZXfz67Hv1oEYq2bgbAEC0etWjXfpt+O/1qxl0YXTxpbmRStZIH2EEAEPW/3/d7ctvufQnbjcwVFVV6aabbtKoUaNUWFioTz/9NF6HAgAMcXEJo3feeUfl5eVavXq1PvvsMxUUFGj+/Pk6ceJEPA4HABji4hJGa9eu1bJly/Tkk0/q9ttv14YNGzRmzBj96le/isfhAABDXMzD6Ny5c9q7d6+Ki4svHSQpScXFxaqvr79s/+7uboVCoYgFADC8xDyMvvjiC50/f17Z2dkR67Ozs9Xa2nrZ/pWVlQoEAuGF27oBYPgxn4GhoqJCHR0d4eXIkSPWLQEABljMb+3OzMzUiBEj1NbWFrG+ra1NwWDwsv39fr/8fn+s2wAADCExvzJKSUnRzJkzVVNTE17X19enmpoaFRUVxfpwAIAEEJcPvZaXl2vJkiX65je/qbvvvlvr1q1TV1eXnnzyyXgcDgAwxMUljBYvXqy//vWveuWVV9Ta2qo777xT27dvv+ymBgAApEE4UerFL4SaowVMBwQAQ1iv61Gttn2lL/gzv5sOAADCCABgjjACAJgjjAAA5ggjAIA5wggAYI4wAgCYI4wAAOYIIwCAOcIIAGCOMAIAmCOMAADmCCMAgDnCCABgjjACAJgjjAAA5ggjAIA5wggAYI4wAgCYI4wAAOYIIwCAOcIIAGCOMAIAmCOMAADmCCMAgDnCCABgjjACAJgjjAAA5ggjAIA5wggAYI4wAgCYI4wAAOYIIwCAOcIIAGCOMAIAmCOMAADmCCMAgDnCCABgjjACAJgjjAAA5ggjAIA5wggAYI4wAgCYI4wAAOYIIwCAOcIIAGCOMAIAmCOMAADmCCMAgDnCCABgjjACAJgjjAAA5ggjAIA5wggAYI4wAgCYi3kYvfrqq/L5fBHL1KlTY30YAEACGRmPJ73jjjv04YcfXjrIyLgcBgCQIOKSEiNHjlQwGIzHUwMAElBc3jM6dOiQcnNzNWnSJD3xxBM6fPjwFfft7u5WKBSKWAAAw0vMw6iwsFDV1dXavn271q9fr5aWFt13333q7Ozsd//KykoFAoHwkpeXF+uWAACDnM855+J5gPb2dk2cOFFr167VU089ddn27u5udXd3hx+HQiHl5eVpjhZopC85nq0BAOKo1/WoVtvU0dGhtLS0q+4b9zsL0tPTdeutt6qpqanf7X6/X36/P95tAAAGsbh/zujUqVNqbm5WTk5OvA8FABiiYh5Gzz//vOrq6vT555/rk08+0cMPP6wRI0bosccei/WhAAAJIuYv0x09elSPPfaYTp48qRtvvFH33nuvGhoadOONN8b6UACABBHzMHr77bdj/ZQAgATH3HQAAHOEEQDAHGEEADBHGAEAzBFGAABzhBEAwBxhBAAwRxgBAMwRRgAAc4QRAMAcYQQAMEcYAQDMEUYAAHNx/6ZXDLzeB2Z6rvmHX7znuaaufYrnGkk62X2D55refxzjuabjTu9fWzJ2827PNQCuH1dGAABzhBEAwBxhBAAwRxgBAMwRRgAAc4QRAMAcYQQAMEcYAQDMEUYAAHOEEQDAHGEEADBHGAEAzDFRagLa9kaV55q/nu/1XPPt0X/2XBOt/Lqxnmt+cvIWzzX/vORfea6RJP9I7+N3psf7r59zPs81krRu2juea1YdXOy55se3v+u55sX/vchzjSTNym3xXNOycJznmt7/e8xzDbzjyggAYI4wAgCYI4wAAOYIIwCAOcIIAGCOMAIAmCOMAADmCCMAgDnCCABgjjACAJgjjAAA5ggjAIA5wggAYI5ZuxNQ0X8p91zj/5vzXHMmK7oZpE/fcs5zja/T+6k6957/5bnmN9N/5blGkn5wvNRzzcYJv4vqWNFo6TnluWbVrTWea+4d1eW55t07/7vnmmj9u1/+e881Y0vi0Aguw5URAMAcYQQAMEcYAQDMEUYAAHOEEQDAHGEEADBHGAEAzBFGAABzhBEAwBxhBAAwRxgBAMwRRgAAc0yUmoByf/qJdQuDwuGkEZ5rnrn9H6M6Vt8fDnmumf7dZzzXjP/gb55rJEl93kt8Z7s912z6/Ijnmm/8i/eJcyXpP2Uf8Fxz7M+Znmtu1Z8918A7rowAAOYIIwCAOc9htHPnTj300EPKzc2Vz+fT1q1bI7Y75/TKK68oJydHo0ePVnFxsQ4d8v4SBgBg+PAcRl1dXSooKFBVVVW/29esWaPXX39dGzZs0O7du3XDDTdo/vz5Onv27HU3CwBITJ5vYCgtLVVpaf/faumc07p16/TSSy9pwYIFkqQ33nhD2dnZ2rp1qx599NHr6xYAkJBi+p5RS0uLWltbVVxcHF4XCARUWFio+vr6fmu6u7sVCoUiFgDA8BLTMGptbZUkZWdnR6zPzs4Ob/uyyspKBQKB8JKXlxfLlgAAQ4D53XQVFRXq6OgIL0eOeP+cAgBgaItpGAWDQUlSW1tbxPq2trbwti/z+/1KS0uLWAAAw0tMwyg/P1/BYFA1NTXhdaFQSLt371ZRUVEsDwUASCCe76Y7deqUmpqawo9bWlq0f/9+ZWRkaMKECVq1apV+9KMf6ZZbblF+fr5efvll5ebmauHChbHsGwCQQDyH0Z49e3T//feHH5eXl0uSlixZourqar3wwgvq6urS8uXL1d7ernvvvVfbt2/XqFGjYtc1ACCh+JxzzrqJvxcKhRQIBDRHCzTSl2zdDgAPeh+Y6bnm5f+2MapjTRzp/WMgKybP8Vzjens91+CCXtejWm1TR0fHNe8HML+bDgAAwggAYI4wAgCYI4wAAOYIIwCAOcIIAGCOMAIAmCOMAADmCCMAgDnCCABgjjACAJgjjAAA5jzP2g1gePCN9P7Xw2//xwbPNf4oJ0QufuKfPNeM6P0sqmMh/rgyAgCYI4wAAOYIIwCAOcIIAGCOMAIAmCOMAADmCCMAgDnCCABgjjACAJgjjAAA5ggjAIA5wggAYI4wAgCYY9ZuAP069s+3eK/prfVc80Z7gecaSeoc7/dckx7VkTAQuDICAJgjjAAA5ggjAIA5wggAYI4wAgCYI4wAAOYIIwCAOcIIAGCOMAIAmCOMAADmCCMAgDnCCABgjolSgWFgxM35nmuen7rDc01+8ljPNQ1PzPBcI0npB+ujqsPgxJURAMAcYQQAMEcYAQDMEUYAAHOEEQDAHGEEADBHGAEAzBFGAABzhBEAwBxhBAAwRxgBAMwRRgAAc0yUCgwDRx7O8Vxz3+jPPdfcuvMZzzWTR/Z6rkHi4coIAGCOMAIAmPMcRjt37tRDDz2k3Nxc+Xw+bd26NWL70qVL5fP5IpaSkpJY9QsASECew6irq0sFBQWqqqq64j4lJSU6fvx4eHnrrbeuq0kAQGLzfANDaWmpSktLr7qP3+9XMBiMuikAwPASl/eMamtrlZWVpSlTpmjFihU6efLkFfft7u5WKBSKWAAAw0vMw6ikpERvvPGGampq9JOf/ER1dXUqLS3V+fPn+92/srJSgUAgvOTl5cW6JQDAIBfzzxk9+uij4Z+nT5+uGTNmaPLkyaqtrdXcuXMv27+iokLl5eXhx6FQiEACgGEm7rd2T5o0SZmZmWpqaup3u9/vV1paWsQCABhe4h5GR48e1cmTJ5WT4/0T4ACA4cHzy3SnTp2KuMppaWnR/v37lZGRoYyMDL322mtatGiRgsGgmpub9cILL+jmm2/W/PnzY9o4ACBxeA6jPXv26P777w8/vvh+z5IlS7R+/XodOHBAv/71r9Xe3q7c3FzNmzdPP/zhD+X3+2PXNQAgoXgOozlz5sg5d8XtH3zwwXU1BAAYfpi1GxhKfL6oyt4u+8+eaz7omuK55obfjfVc07f/E881SDxMlAoAMEcYAQDMEUYAAHOEEQDAHGEEADBHGAEAzBFGAABzhBEAwBxhBAAwRxgBAMwRRgAAc4QRAMAcE6UCQ0jbs0VR1d2Rss9zzb/53TzPNTdXMekposOVEQDAHGEEADBHGAEAzBFGAABzhBEAwBxhBAAwRxgBAMwRRgAAc4QRAMAcYQQAMEcYAQDMEUYAAHNMlAoYOfLStzzX/Md/eCeqY/3oi6mea9IaRkd1LCAaXBkBAMwRRgAAc4QRAMAcYQQAMEcYAQDMEUYAAHOEEQDAHGEEADBHGAEAzBFGAABzhBEAwBxhBAAwx0SpQAz4Rnr/VZpR+ifPNd8a9RfPNZL0L6fyPddkVX0S1bGAaHBlBAAwRxgBAMwRRgAAc4QRAMAcYQQAMEcYAQDMEUYAAHOEEQDAHGEEADBHGAEAzBFGAABzhBEAwBxhBAAwx6zdwN9JKrgtqrrKrdWeawJJPZ5rMkaM8FwjSY2zovlV743qWEA0uDICAJgjjAAA5jyFUWVlpe666y6lpqYqKytLCxcuVGNjY8Q+Z8+eVVlZmcaNG6exY8dq0aJFamtri2nTAIDE4imM6urqVFZWpoaGBu3YsUM9PT2aN2+eurq6wvs899xzeu+997R582bV1dXp2LFjeuSRR2LeOAAgcXh6V3P79u0Rj6urq5WVlaW9e/dq9uzZ6ujo0C9/+Utt2rRJDzzwgCRp48aNuu2229TQ0KB77rnnsufs7u5Wd3d3+HEoFIrmzwEAGMKu6z2jjo4OSVJGRoYkae/everp6VFxcXF4n6lTp2rChAmqr6/v9zkqKysVCATCS15e3vW0BAAYgqIOo76+Pq1atUqzZs3StGnTJEmtra1KSUlRenp6xL7Z2dlqbW3t93kqKirU0dERXo4cORJtSwCAISrqzxmVlZXp4MGD2rVr13U14Pf75ff7r+s5AABDW1RXRitXrtT777+vjz/+WOPHjw+vDwaDOnfunNrb2yP2b2trUzAYvK5GAQCJy1MYOee0cuVKbdmyRR999JHy8/Mjts+cOVPJycmqqakJr2tsbNThw4dVVFQUm44BAAnH08t0ZWVl2rRpk7Zt26bU1NTw+0CBQECjR49WIBDQU089pfLycmVkZCgtLU3PPvusioqK+r2TDgAAyWMYrV+/XpI0Z86ciPUbN27U0qVLJUk/+9nPlJSUpEWLFqm7u1vz58/XL37xi5g0CwBITJ7CyDl3zX1GjRqlqqoqVVVVRd0UYKaxJaqyaCY9zU8e67lm3qIlnmskyXf2f0ZVBwwU5qYDAJgjjAAA5ggjAIA5wggAYI4wAgCYI4wAAOYIIwCAOcIIAGCOMAIAmCOMAADmCCMAgDnCCABgLupvegUGO18U3yAcrE2O6ljRTHo6acd/8Fwz9f987rlGks5HVQUMHK6MAADmCCMAgDnCCABgjjACAJgjjAAA5ggjAIA5wggAYI4wAgCYI4wAAOYIIwCAOcIIAGCOMAIAmCOMAADmmLUbCevoqpmea9bnrInyaN5n7b7t+62ea3pP/s1zDTAUcGUEADBHGAEAzBFGAABzhBEAwBxhBAAwRxgBAMwRRgAAc4QRAMAcYQQAMEcYAQDMEUYAAHOEEQDAHBOlYkho/c63PNfs+6efe6452uu5JGqt/3qi55rM/3osDp0A9rgyAgCYI4wAAOYIIwCAOcIIAGCOMAIAmCOMAADmCCMAgDnCCABgjjACAJgjjAAA5ggjAIA5wggAYI6JUjEknLmny3PN0d4znmvyk8d6rpGkf9tc7Lkm5ZSL6lhAIuLKCABgjjACAJjzFEaVlZW66667lJqaqqysLC1cuFCNjY0R+8yZM0c+ny9iefrpp2PaNAAgsXgKo7q6OpWVlamhoUE7duxQT0+P5s2bp66uyNfzly1bpuPHj4eXNWvWxLRpAEBi8XQDw/bt2yMeV1dXKysrS3v37tXs2bPD68eMGaNgMBibDgEACe+63jPq6OiQJGVkZESsf/PNN5WZmalp06apoqJCp0+fvuJzdHd3KxQKRSwAgOEl6lu7+/r6tGrVKs2aNUvTpk0Lr3/88cc1ceJE5ebm6sCBA3rxxRfV2Niod999t9/nqays1GuvvRZtGwCABBB1GJWVlengwYPatWtXxPrly5eHf54+fbpycnI0d+5cNTc3a/LkyZc9T0VFhcrLy8OPQ6GQ8vLyom0LADAERRVGK1eu1Pvvv6+dO3dq/PjxV923sLBQktTU1NRvGPn9fvn9/mjaAAAkCE9h5JzTs88+qy1btqi2tlb5+fnXrNm/f78kKScnJ6oGAQCJz1MYlZWVadOmTdq2bZtSU1PV2toqSQoEAho9erSam5u1adMmPfjggxo3bpwOHDig5557TrNnz9aMGTPi8gcAAAx9nsJo/fr1ki58sPXvbdy4UUuXLlVKSoo+/PBDrVu3Tl1dXcrLy9OiRYv00ksvxaxhAEDi8fwy3dXk5eWprq7uuhoC+jN25xjvRd/yXjLn4ELvRZJGPXjUc01ab0NUxwISEXPTAQDMEUYAAHOEEQDAHGEEADBHGAEAzBFGAABzhBEAwBxhBAAwRxgBAMwRRgAAc4QRAMAcYQQAMEcYAQDM+dy1puIeYKFQSIFAQHO0QCN9ydbtAACi1Ot6VKtt6ujoUFpa2lX35coIAGCOMAIAmCOMAADmCCMAgDnCCABgjjACAJgjjAAA5ggjAIA5wggAYI4wAgCYI4wAAOZGWjfwZRenyutVjzSoZs0DAHjRqx5Jl/5ev5pBF0adnZ2SpF36rXEnAIBY6OzsVCAQuOo+g27W7r6+Ph07dkypqany+XwR20KhkPLy8nTkyJFrzgCbyBiHSxiLCxiHSxiLCwbDODjn1NnZqdzcXCUlXf1doUF3ZZSUlKTx48dfdZ+0tLRhfZJdxDhcwlhcwDhcwlhcYD0O17oiuogbGAAA5ggjAIC5IRVGfr9fq1evlt/vt27FFONwCWNxAeNwCWNxwVAbh0F3AwMAYPgZUldGAIDERBgBAMwRRgAAc4QRAMAcYQQAMEcYAQDMEUYAAHOEEQDA3P8DvzoXssKEdFMAAAAASUVORK5CYII=\n"
          },
          "metadata": {}
        }
      ],
      "source": [
        "plt.matshow(x_test_adv[0, :].reshape((28, 28)))\n",
        "plt.show()"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "py37_tf220",
      "language": "python",
      "name": "py37_tf220"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.6"
    },
    "colab": {
      "provenance": []
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "0b8242407f8245e197f108e489b709f3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_ee9bf3456da448a9a6fd9c32b4b55d1c",
              "IPY_MODEL_63cc72f10306471bb259f407b86d40b3",
              "IPY_MODEL_571690ae3d4848b2b144e309baa6db62"
            ],
            "layout": "IPY_MODEL_7a3ba25da2ec43c5a02f33256e908a2d"
          }
        },
        "ee9bf3456da448a9a6fd9c32b4b55d1c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_290351d386ad460c89f4a6518fbca91b",
            "placeholder": "​",
            "style": "IPY_MODEL_0a8aa57f7b904d0784b292823e9c2407",
            "value": "C&amp;W L_inf: 100%"
          }
        },
        "63cc72f10306471bb259f407b86d40b3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b0138b0c3dc14596a372a7f7a6353c71",
            "max": 10,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_773990d53c4f498a83c467ea828a6396",
            "value": 10
          }
        },
        "571690ae3d4848b2b144e309baa6db62": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_430139380b54472ca2753c9392e7917a",
            "placeholder": "​",
            "style": "IPY_MODEL_8d62f30dc13341bbb50aa01551102379",
            "value": " 10/10 [02:03&lt;00:00, 14.45s/it]"
          }
        },
        "7a3ba25da2ec43c5a02f33256e908a2d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "290351d386ad460c89f4a6518fbca91b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0a8aa57f7b904d0784b292823e9c2407": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b0138b0c3dc14596a372a7f7a6353c71": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "773990d53c4f498a83c467ea828a6396": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "430139380b54472ca2753c9392e7917a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8d62f30dc13341bbb50aa01551102379": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}