Grant_PPO_KarchowRL.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6l1iUzl6mKVp"
      },
      "source": [
        "# Karchow RL with PPO"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Credits to Grant Cheng (GitHub: CatFish47)."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RdorY8yDmP45"
      },
      "source": [
        "Install anything necessary for stable baselines and open ai gym."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-gaGuFE0jxcb",
        "outputId": "b40258c2-956d-409f-da87-bc5b3e8da79e"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "x11-utils is already the newest version (7.7+3build1).\n",
            "xvfb is already the newest version (2:1.19.6-1ubuntu4.10).\n",
            "0 upgraded, 0 newly installed, 0 to remove and 39 not upgraded.\n",
            "Requirement already satisfied: pyvirtualdisplay==0.2.* in /usr/local/lib/python3.7/dist-packages (0.2.5)\n",
            "Requirement already satisfied: EasyProcess in /usr/local/lib/python3.7/dist-packages (from pyvirtualdisplay==0.2.*) (1.1)\n",
            "Requirement already satisfied: stable-baselines3[extra] in /usr/local/lib/python3.7/dist-packages (1.5.0)\n",
            "Requirement already satisfied: cloudpickle in /usr/local/lib/python3.7/dist-packages (from stable-baselines3[extra]) (1.3.0)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from stable-baselines3[extra]) (1.3.5)\n",
            "Requirement already satisfied: torch>=1.8.1 in /usr/local/lib/python3.7/dist-packages (from stable-baselines3[extra]) (1.10.0+cu111)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from stable-baselines3[extra]) (1.21.5)\n",
            "Collecting gym==0.21\n",
            "  Using cached gym-0.21.0-py3-none-any.whl\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.7/dist-packages (from stable-baselines3[extra]) (3.2.2)\n",
            "Requirement already satisfied: tensorboard>=2.2.0 in /usr/local/lib/python3.7/dist-packages (from stable-baselines3[extra]) (2.8.0)\n",
            "Requirement already satisfied: ale-py~=0.7.4 in /usr/local/lib/python3.7/dist-packages (from stable-baselines3[extra]) (0.7.4)\n",
            "Requirement already satisfied: pillow in /usr/local/lib/python3.7/dist-packages (from stable-baselines3[extra]) (7.1.2)\n",
            "Requirement already satisfied: opencv-python in /usr/local/lib/python3.7/dist-packages (from stable-baselines3[extra]) (4.1.2.30)\n",
            "Requirement already satisfied: autorom[accept-rom-license]~=0.4.2 in /usr/local/lib/python3.7/dist-packages (from stable-baselines3[extra]) (0.4.2)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.7/dist-packages (from stable-baselines3[extra]) (5.4.8)\n",
            "Requirement already satisfied: importlib-metadata>=4.8.1 in /usr/local/lib/python3.7/dist-packages (from gym==0.21->stable-baselines3[extra]) (4.11.3)\n",
            "Requirement already satisfied: importlib-resources in /usr/local/lib/python3.7/dist-packages (from ale-py~=0.7.4->stable-baselines3[extra]) (5.4.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from autorom[accept-rom-license]~=0.4.2->stable-baselines3[extra]) (7.1.2)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from autorom[accept-rom-license]~=0.4.2->stable-baselines3[extra]) (4.63.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from autorom[accept-rom-license]~=0.4.2->stable-baselines3[extra]) (2.23.0)\n",
            "Requirement already satisfied: AutoROM.accept-rom-license in /usr/local/lib/python3.7/dist-packages (from autorom[accept-rom-license]~=0.4.2->stable-baselines3[extra]) (0.4.2)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=4.8.1->gym==0.21->stable-baselines3[extra]) (3.10.0.2)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=4.8.1->gym==0.21->stable-baselines3[extra]) (3.7.0)\n",
            "Requirement already satisfied: absl-py>=0.4 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (1.0.0)\n",
            "Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (57.4.0)\n",
            "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (1.8.1)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (1.0.1)\n",
            "Requirement already satisfied: protobuf>=3.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (3.17.3)\n",
            "Requirement already satisfied: grpcio>=1.24.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (1.44.0)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (3.3.6)\n",
            "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (0.6.1)\n",
            "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (0.4.6)\n",
            "Requirement already satisfied: google-auth<3,>=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (1.35.0)\n",
            "Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (0.37.1)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from absl-py>=0.4->tensorboard>=2.2.0->stable-baselines3[extra]) (1.15.0)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard>=2.2.0->stable-baselines3[extra]) (4.8)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard>=2.2.0->stable-baselines3[extra]) (0.2.8)\n",
            "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard>=2.2.0->stable-baselines3[extra]) (4.2.4)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard>=2.2.0->stable-baselines3[extra]) (1.3.1)\n",
            "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.7/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard>=2.2.0->stable-baselines3[extra]) (0.4.8)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->autorom[accept-rom-license]~=0.4.2->stable-baselines3[extra]) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->autorom[accept-rom-license]~=0.4.2->stable-baselines3[extra]) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->autorom[accept-rom-license]~=0.4.2->stable-baselines3[extra]) (2021.10.8)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->autorom[accept-rom-license]~=0.4.2->stable-baselines3[extra]) (1.24.3)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard>=2.2.0->stable-baselines3[extra]) (3.2.0)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->stable-baselines3[extra]) (2.8.2)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->stable-baselines3[extra]) (3.0.7)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->stable-baselines3[extra]) (1.4.0)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.7/dist-packages (from matplotlib->stable-baselines3[extra]) (0.11.0)\n",
            "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.7/dist-packages (from pandas->stable-baselines3[extra]) (2018.9)\n",
            "Installing collected packages: gym\n",
            "  Attempting uninstall: gym\n",
            "    Found existing installation: gym 0.23.1\n",
            "    Uninstalling gym-0.23.1:\n",
            "      Successfully uninstalled gym-0.23.1\n",
            "Successfully installed gym-0.21.0\n",
            "Requirement already satisfied: gym in /usr/local/lib/python3.7/dist-packages (0.21.0)\n",
            "Collecting gym\n",
            "  Using cached gym-0.23.1-py3-none-any.whl\n",
            "Requirement already satisfied: cloudpickle>=1.2.0 in /usr/local/lib/python3.7/dist-packages (from gym) (1.3.0)\n",
            "Requirement already satisfied: numpy>=1.18.0 in /usr/local/lib/python3.7/dist-packages (from gym) (1.21.5)\n",
            "Requirement already satisfied: importlib-metadata>=4.10.0 in /usr/local/lib/python3.7/dist-packages (from gym) (4.11.3)\n",
            "Requirement already satisfied: gym-notices>=0.0.4 in /usr/local/lib/python3.7/dist-packages (from gym) (0.0.6)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=4.10.0->gym) (3.10.0.2)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=4.10.0->gym) (3.7.0)\n",
            "Installing collected packages: gym\n",
            "  Attempting uninstall: gym\n",
            "    Found existing installation: gym 0.21.0\n",
            "    Uninstalling gym-0.21.0:\n",
            "      Successfully uninstalled gym-0.21.0\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "stable-baselines3 1.5.0 requires gym==0.21, but you have gym 0.23.1 which is incompatible.\u001b[0m\n",
            "Successfully installed gym-0.23.1\n",
            "Requirement already satisfied: gym[box2d] in /usr/local/lib/python3.7/dist-packages (0.23.1)\n",
            "Requirement already satisfied: cloudpickle>=1.2.0 in /usr/local/lib/python3.7/dist-packages (from gym[box2d]) (1.3.0)\n",
            "Requirement already satisfied: numpy>=1.18.0 in /usr/local/lib/python3.7/dist-packages (from gym[box2d]) (1.21.5)\n",
            "Requirement already satisfied: gym-notices>=0.0.4 in /usr/local/lib/python3.7/dist-packages (from gym[box2d]) (0.0.6)\n",
            "Requirement already satisfied: importlib-metadata>=4.10.0 in /usr/local/lib/python3.7/dist-packages (from gym[box2d]) (4.11.3)\n",
            "Requirement already satisfied: box2d-py==2.3.5 in /usr/local/lib/python3.7/dist-packages (from gym[box2d]) (2.3.5)\n",
            "Requirement already satisfied: pygame==2.1.0 in /usr/local/lib/python3.7/dist-packages (from gym[box2d]) (2.1.0)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=4.10.0->gym[box2d]) (3.7.0)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=4.10.0->gym[box2d]) (3.10.0.2)\n"
          ]
        }
      ],
      "source": [
        "!apt-get install -y xvfb x11-utils\n",
        "!pip install pyvirtualdisplay==0.2.*\n",
        "!pip install stable-baselines3[extra]\n",
        "!pip install gym --upgrade\n",
        "!pip install gym[box2d]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4lN5lF4umUpq"
      },
      "source": [
        "Mount notebook to Google drive, where the checkpoint data for the is stored."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iuxVurLU3-r6",
        "outputId": "fe1f5f27-359d-45e9-fec0-abcd3635734c"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pr-BG_6umlfA"
      },
      "source": [
        "Import necessary components for the model and environment. Create the environment."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "O-O22O6Hj1Yq"
      },
      "outputs": [],
      "source": [
        "import gym\n",
        "from stable_baselines3 import PPO\n",
        "from stable_baselines3.common.env_util import make_vec_env\n",
        "import imageio\n",
        "import os\n",
        "import numpy as np\n",
        "\n",
        "# Parallel environments\n",
        "env = make_vec_env(\"CarRacing-v1\", n_envs=4)\n",
        "\n",
        "# model = PPO(\"CnnPolicy\", env, learning_rate=0.0003, n_steps=1024, batch_size=64, n_epochs=10, gamma=0.99, gae_lambda=0.95, clip_range=0.2, seed=8, verbose=1)\n",
        "# model = PPO.load(\"/content/drive/MyDrive/PPO/ppo_racing2\", env=env)\n",
        "# model.learn(total_timesteps=62_500)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gJKk4mU4mr3R"
      },
      "source": [
        "Save the model to drive after finishing training."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "iOL0cOEyj2CT"
      },
      "outputs": [],
      "source": [
        "# model.save(\"/content/drive/MyDrive/PPO/ppo_racing3\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "s07UQxZImvWe"
      },
      "source": [
        "Load a model that's been stored in drive."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0F7BZf9W2BI3",
        "outputId": "7e01dc61-3898-49a5-ea9b-d5a1378ede11"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Wrapping the env in a VecTransposeImage.\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/gym/core.py:173: DeprecationWarning: \u001b[33mWARN: Function `env.seed(seed)` is marked as deprecated and will be removed in the future. Please use `env.reset(seed=seed) instead.\u001b[0m\n",
            "  \"Function `env.seed(seed)` is marked as deprecated and will be removed in the future. \"\n"
          ]
        }
      ],
      "source": [
        "model = PPO.load(\"/content/drive/MyDrive/PPO/ppo_racing1\", env=env)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "z7Yyd9rum0Oc"
      },
      "source": [
        "Create a validation stage."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "joKjnzY_Hv4g"
      },
      "outputs": [],
      "source": [
        "def average_rewards(r):\n",
        "  return np.sum(r)/len(r)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yq97RIhtj3kA",
        "outputId": "088f13af-b71f-4332-b92c-ee81d8b49938"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Track generation: 957..1209 -> 252-tiles track\n",
            "Track generation: 1127..1413 -> 286-tiles track\n",
            "Track generation: 1191..1493 -> 302-tiles track\n",
            "Track generation: 993..1253 -> 260-tiles track\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|█████████▉| 999/1000 [03:52<00:00,  4.40it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Track generation: 1196..1499 -> 303-tiles track\n",
            "Track generation: 1364..1709 -> 345-tiles track\n",
            "Track generation: 1180..1479 -> 299-tiles track\n",
            "Track generation: 988..1242 -> 254-tiles track\n",
            "retry to generate track (normal if there are not manyinstances of this message)\n",
            "Track generation: 1121..1414 -> 293-tiles track\n",
            "Track generation: 1034..1297 -> 263-tiles track\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r100%|█████████▉| 999/1000 [03:52<00:00,  4.29it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Track generation: 1120..1404 -> 284-tiles track\n",
            "Track generation: 1147..1438 -> 291-tiles track\n",
            "Track generation: 1145..1440 -> 295-tiles track\n",
            "retry to generate track (normal if there are not manyinstances of this message)\n",
            "Track generation: 1187..1488 -> 301-tiles track\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n"
          ]
        }
      ],
      "source": [
        "#Validation\n",
        "from tqdm import tqdm\n",
        "obs = env.reset()\n",
        "frames = []\n",
        "avg_reward = 0\n",
        "for i in tqdm(range(1000)):\n",
        "  action, _states = model.predict(obs.copy(), deterministic=True)\n",
        "  obs, reward, done, info = env.step(action)\n",
        "  avg_reward += average_rewards(reward)\n",
        "  frames.append(env.render(\"rgb_array\"))\n",
        "  if done.all():\n",
        "    obs = env.reset()\n",
        "    break\n",
        "\n",
        "env.close()\n",
        "\n",
        "path = os.path.join(f\"eval.mp4\")\n",
        "frames = np.array(frames)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5He9q3dVnAo7"
      },
      "source": [
        "Save the video."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "AHQGCR23j6EQ"
      },
      "outputs": [],
      "source": [
        "# imageio.mimsave(path, frames, fps=60)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FzqDPK8gnDWc"
      },
      "source": [
        "Create a rewards graph using the average reward during the validation stage."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4c_1_ZL1GxFt",
        "outputId": "74ffd4fc-60ae-44d9-846b-03cb72640abb"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "869.8522652685642"
            ]
          },
          "execution_count": 9,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "avg_reward"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1soYdrtTIRQe"
      },
      "outputs": [],
      "source": [
        "#average cumulative rewards over 5 different periods\n",
        "# None: 31.030509762465954\n",
        "# 62500: 327.19548013806343\n",
        "# 125000: 869.8522652685642\n",
        "# 187500: 591.970870397985\n",
        "# 250000: 963.6337747052312"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 510
        },
        "id": "9B6w0n-8Jm__",
        "outputId": "970f5ed3-f645-4ece-bc95-b322ad63038d"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/scipy/fft/__init__.py:97: DeprecationWarning: The module numpy.dual is deprecated.  Instead of using dual, use the functions directly from numpy or scipy.\n",
            "  from numpy.dual import register_func\n",
            "/usr/local/lib/python3.7/dist-packages/scipy/sparse/sputils.py:17: DeprecationWarning: `np.typeDict` is a deprecated alias for `np.sctypeDict`.\n",
            "  supported_dtypes = [np.typeDict[x] for x in supported_dtypes]\n",
            "/usr/local/lib/python3.7/dist-packages/scipy/special/orthogonal.py:81: DeprecationWarning: `np.int` is a deprecated alias for the builtin `int`. To silence this warning, use `int` by itself. Doing this will not modify any behavior and is safe. When replacing `np.int`, you may wish to use e.g. `np.int64` or `np.int32` to specify the precision. If you wish to review your current use, check the release note link for additional information.\n",
            "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
            "  from numpy import (exp, inf, pi, sqrt, floor, sin, cos, around, int,\n"
          ]
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWEAAAFgCAYAAABqo8hyAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3yV5f3/8deHhBB2GGEmbGTviOCqVasoKrYuwMG3X1tH66gdjq/tV9Rfa4fWr6NV22prlSCKVnDjqqPVhLA3BDCLFUYgELKv3x/njo1I4ARyzn3G+/l4nEfuc933Offn5px8uHJd9/25zTmHiIj4o5nfAYiIxDMlYRERHykJi4j4SElYRMRHSsIiIj5SEhYR8VHIkrCZPWNmO8xsZb22jmb2rplt8H528NrNzB41s1wzW25mY+u9Zoa3/QYzmxHMvidNmuQAPfTQQw8/Ho0Syp7w34BJh7TdCbzvnBsIvO89BzgPGOg9rgOegEDSBu4BTgLGA/fUJe4j2blzZxOELyISeiFLws65j4HdhzRPAZ71lp8FLq7X/ncX8DmQYmbdgXOBd51zu51ze4B3+XpiFxGJWuEeE+7qnNvqLW8DunrLPYGCetsVem0NtX+NmV1nZjlmllNcXNy0UYuIhIhvE3MucL10o8dPjvB+f3LOZTjnMlJTU5vqbUVEQircSXi7N8yA93OH114EpNfbLs1ra6hdRCQmhDsJzwdmeMszgHn12q/xzpKYAOz1hi3eAc4xsw7ehNw5XpuISExIDNUbm9ls4Aygs5kVEjjL4dfAi2Z2LZAHXO5t/iZwPpALlAHfBXDO7Taz+4GF3nb3OecOnewTEYlaFoulLDMyMlxOTo7fYYhIfLLGbKwr5kREfKQkLCLiIyVhEREfKQmLiPhISVhE5BDVNbXcNmcpywtLQr4vJWERkUM893ke/1hSxJaS8pDvS0lYRKSeXfsr+P276zltYGfOHdb16C84TkrCIiL1/O6ddRysrOGeC4di1qhTfo+JkrCIiGd5YQlzcgr4r5P7MKBL27DsU0lYRASorXXcM38VnVq34NazB4Ztv0rCIiLAP5YUsSS/hDsmDaJtcvOw7VdJWETiXml5FQ+8tZbR6SlcMjYtrPsOWRU1EZFo8dgHuew6UMHTMzJo1iz0k3H1qScsInEtd8d+nvl0M5ePS2dUekrY968kLCJxyznHva+tomVSAj+bNMiXGJSERSRuvbt6O59s2MltZ59A5zYtfIlBSVhE4lJ5VQ33v7GagV3acPXE3r7FoYk5EYlLf/54EwW7DzLreyfRPMG//qh6wiISd4pKDvKHf+Zy3vBunDKgs6+xKAmLSNz51ZtrALh78hCfI1ESFpE48++NO3lj+VZu/MYA0jq08jscJWERiR/VNbXcO381aR1acv03+vkdDqAkLCJx5PnP81i3vZSfTx5KcvMEv8MBlIRFJE7UFWs/dUB4irUHS0lYROLCgwvWUVZZw8yLwlOsPVhKwiIS85YXlvDCwgJmhLFYe7CUhEUkptXWOmbOX0Wn1klhLdYeLCVhEYlp/1hSxOL8Em6fNJh2YSzWHiwlYRGJWaXlVfz67bWMSk/h0jAXaw+WakeISMx67INciksr+PM14S/WHiz1hEUkJn1ZrD0jjdE+FGsPlpKwiMQc5xz3vb6als0TuH3SYL/DOSIlYZF6sjbt4uP1xX6HIcfpvTU7+Hh9MT/6ln/F2oOlJCziqaqp5abZS/jvvy0ke/Nuv8ORY1ReVcP9rweKtV/jY7H2YCkJi3jeW72d4tIKWiYl8INZi9m+r9zvkOQY/OWTTeTvLmPmRcN8LdYerMiPUCRMMrPz6ZnSkhevn0hZZTU/mLWYyupav8OSRthScpA/fLgxIoq1B0tJWATI23WATzbs5IoT0xnSvR2/vXQki/L28Ms3VvsdmjTCL99cQ61z/M/5/hdrD5aSsAiBXnBCM+OKE9MBuGBkD753al+e/SyPVxYX+hydBOOzjbsCxdrP6E96R/+LtQdLSVjiXmV1LXNzCjlrcBe6tkv+sv3O8wYzoV9H7nplBau27PUxQjma6ppa7n1tFT1TWnLDN/r7HU6jKAlL3Htn1TZ2Hahk+km9vtKemNCMx6ePpUOrJG54fhElZZU+RShHMysrn7XbSvnFBUMiplh7sJSEJe5lZuWT1qElpw9M/dq6zm1a8MerxrJtbzm3vrCUmlrnQ4RyJLv2V/DQgnWcMqAT5w7r5nc4jaYkLHFtU/F+Ptu0i2njezVYW2Bsrw7MvGgYH60v5pH31oc5QjmaBxesDxRrv3BYRBVrD5aSsMS12dn5JDYzLss4coWt6eN7cdm4NB79IJf3Vm8PU3RyNCsK9/LCwnxmnNyHgV0jq1h7sJSEJW6VV9Uwd1Eh5wzrSpe2yUfc1sy4/+LhDO/ZjtvmLGXzzgNhilIa4pzjnvkrI7ZYe7CUhCVuvbNqG3vKqpg+PrhLW5ObJ/DkVeNITDBueG4RZZXVIY5QjiTSi7UHS0lY4tasrHx6d2rFyf07Bf2atA6teHTaGDbsKOWOl1fgnCbq/LC/opoH3orsYu3BUhKWuJS7o5TszbuPOCHXkNMGpvKTcwbx2rItPP3p5hBFKEfy2PsbKC6t4N6LhkVssfZgKQlLXJqVlU/zBOPSccfWi/rBGf05d1hXHnhrLZ9v2tXE0cmRbCzezzP/2sxl4yK7WHuwlIQl7pRX1fDyokLOHdbtmGvNmhkPXjaK3p1acVPmYrbtVcW1cHDOce9rq0lOjPxi7cFSEpa488byrewrr/7aFXKN1Ta5OU9dNY6yyhpunLWIiuqaJopQGlJXrP3WsweS2jayi7UHS0lY4k5mdj79OrdmYr/gJ+QaMrBrWx68bBRL8ku4/3VVXAulumLtA7q0YcbJffwOp8n4koTN7DYzW2VmK81stpklm1lfM8sys1wzm2NmSd62Lbznud76Pn7ELLFh3bZSFuXtYdr4Xk12ddX5I7pz/en9eP7zfF7KKWiS95Sv+7JY+4XRUaw9WGE/EjPrCdwCZDjnhgMJwFTgN8DDzrkBwB7gWu8l1wJ7vPaHve1EjklmVh5Jic245Bgn5Brys3MHMbFfJ+5+dSUri1RxranVFWufNKwbpw6MjmLtwfLrv5NEoKWZJQKtgK3AmcBcb/2zwMXe8hTvOd76sywaLxAX3x2srOGVJUWcP7wbHVsnNel7ByqujaFz6ySuf24Rew6o4lpT+pVXrP3uydFTrD1YYU/Czrki4EEgn0Dy3QssAkqcc3WXIBUCPb3lnkCB99pqb/uvDeaZ2XVmlmNmOcXFuluufN1ry7dQWl7N9JNCc/PHTm1a8MRV4ygureCWF5ao4loT+XzTLl5fvpUbvhFdxdqD5cdwRAcCvdu+QA+gNTDpeN/XOfcn51yGcy4jNfXrJQlFMrPyGdClDSf26RCyfYxKT+HeKcP4ZMNOfv/uupDtJ15U19Qyc36gWPuNZ0RXsfZg+TEccTaw2TlX7JyrAl4BTgFSvOEJgDSgyFsuAtIBvPXtAZ0dL42yastelhaUML0JJ+QaMm18L6aemM4fPtzIglXbQrqvWJeZHSjW/vPJ0VesPVh+JOF8YIKZtfLGds8CVgMfApd628wA5nnL873neOs/cLpgXxopMyufFonNuCRMdQZmXjSMkWnt+fGLy9hYvD8s+4w1uw9U8tCC9ZwyoBOThkdfsfZg+TEmnEVggm0xsMKL4U/AHcCPzSyXwJjv095LngY6ee0/Bu4Md8wS3Q5UVDNv6RYmj+xO+1bhqbaV3DyBJ64aR1JiM254bhEHKlRxrbF+98469ldUc0+UFmsPli9nRzjn7nHODXbODXfOXe2cq3DObXLOjXfODXDOXeacq/C2LfeeD/DWb/IjZole85dtYX9FNVce5xVyjdUzpSWPTRvDxuL93D53uSquNcLKIq9Y+8Q+nBClxdqDFTtnPIs0IDMrn0Fd2zK2V+gm5BpyyoDO3D5pMG+s2MqfP1H/IRiBYu2r6Ngquou1B0tJWGLaisK9rCjay/STQj8h15DrT+/HecO78eu31vLvjTt9iSGavLq0iEV5e7hj0mDat4zeYu3BUhKWmJaZnUfL5gl8e2zPo28cImbG7y4bRd/Orbk5cwlbSg76Fkuk219RzQNvrmVUWvtjLjMabZSEJWaVllcxb+kWLhzV3ffb37RpkchTV2dQUV3LjbMWq+JaAx77YAM7SiuYGQPF2oOlJCwxa97SLZRV1oTsCrnGGtClDQ9eNpJlBSXMnK+Ka4faWLyfZz7dzKXj0hjjw/i9X5SEJSY555iVlc/Q7u0Yldbe73C+NGl4d248oz+zs/OZszDf73AihnOO+7xi7XfESLH2YCkJS0xaWlDCmq37fJ2Qa8hPzxnEqQM684t5q1heWOJ3OBHh/TU7+CjGirUHS0lYYlJmVj6tkhKYMrqH36F8TUIz49FpY0ht04Ibn1/M7jivuFZeVcN9MVisPVhKwhJz9h6s4rXlW5gyugdtfZ6Qa0jH1kk8cdVYivdXcPPsxVTX1Podkm+e/nQz+bvLuOfCoTFVrD1Y8XfEEvNeXVJEeVUt08dHxoRcQ0ampfD/pgznX7m7eHDBer/D8cWWkoM8/kEu5w7rymkD47P6oZKwxBTnHJlZ+Yzo2Z4RETQh15DLT0xn2vhePPnRRt5eudXvcMLugbfWUuscP5881O9QfKMkLDFlcf4e1m0vPe47KYfTzIuGMio9hZ+8uIzcHfFTce3zTbt4bdkWro/RYu3BUhKWmDIrK582LRK5aFTkTcg1pEViAk9eNZbk5glc/1wO++Og4tpXirV/IzaLtQdLSVhixt6yKt5YvpWLx/SgdYvEo78ggnRv35LHpo9h884D/OylZTFfca2uWPvdk4fQMik2i7UHS0lYYsbLiwupqI78CbmGnNy/M3edN4S3Vm7jqY9jt+JaXbH2k/t34rwYLtYeLCVhiQnOOTKz8xmdnsLQHu38DueYfe+0vkwe2Z3fvr2Wf+XGZsW1BxcEirXPvCi2i7UHS0lYYkL25t3k7tgfVRNyh2Nm/PaSkfRPbcPNs5dQFGMV11YW7WV2dj7XTOwd88Xag6UkLDEhMzuftsmJXDgyeibkGtK6RSJPXj2Oqupabnx+EeVVsVFxzTnHTK9Y+4/OPsHvcCKGkrBEvd0HKnlrxTa+M6ZnzEzy9E9tw0OXj2J54V7umbfK73CaxKtLi8jJ28PtkwbFRbH2YCkJS9R7eVEhlTW1EVOysqmcM6wbP/xmf+bkFDA7O7orrtUVax+Z1p7LxqX7HU5EURKWqOacY3Z2PuN6d2BQt9gbY/zxtwZx2sDO3DNvFUsLorfiWl2x9nvjqFh7sJSEJap9tmkXm3YeYPr46J6Qa0hCM+PRqWPo0q4FNz6/iJ37K/wOqdE2xWmx9mApCUtUy8zKp33L5kwe2d3vUEKmQ+sknrxqHLsPVHJz5pKoqrjmnOO+11fTIjGB2ycN8juciKQkLFFr5/4K3lm1jUvGppHcPDYm5BoyvGd7fvntEXy2aRe/e2ed3+EE7YO1O/jnumJ+dPZAurRN9juciKQkLFFr7qJCqmoc00+Kj4meS8elcdWEXjz18SbeWB75FdfqirX3T23NNRP7+B1OxFISlqhUWxuYkBvftyMDusTehFxD/veCYYzplcLP5i5jw/ZSv8M5oqc/3UzerjJmXjSMpESlmoboX0ai0r827iRvVxlXRvkVco2VlNiMJ64cR6ukBK5/bhH7yqv8Dumwtu4NFGs/Z2j8FmsPlpKwRKXMrHw6tGrOpDgsANOtfTKPTx9L3u4yfvriMmprI6/i2gNvBoq1/+KC+C3WHiwlYYk6O0rLeXf1di4dl0aLxNiekGvIhH6duOu8wSxYvZ0nPtrodzhfkbVpF/NVrD1oSsISdV7KKaS61jEtRs8NDta1p/blwlE9eGjBOj5eX+x3OECgWPs9KtbeKErCElXqJuQm9utEv9Q2fofjKzPjN5eMYGCXttzywhIKdpf5HRKzVay90ZSEJap8vKGYwj0Ho75kZVNplRSouFZT47hxlr8V13YfqOTBBeuZ2E/F2htDSViiSmZWPp1aJ3HuMP2S1+nbuTUPXzGalUX7+PmrK327NdJDKtZ+TJSEJWps31fO+2t3cFlGus47PcTZQ7tyy5kDmLuokFlZ4a+4trJoL5nZ+Vw9oXdMFlIKJX2TJWrMWVhATa1j2vj4uEKusW49+wS+cUIq9762isX5e8K237pi7R1aJXHbt1SsvbGUhCUq1NQ6XsjO57SBnendqbXf4USkhGbGI1NH0619Mj94fjHFpeGpuDZv6ZZAsfZzVaz9WCgJS1T457odbNlbHrMlK5tKSqtAxbU9ZZXclLk45BXX9ldU86s31zAyrT2XZ+gvlGOhJCxRITMrn9S2LTh7aFe/Q4l4w3q054HvjCBr825+/dbakO7r8Q9y2VFawUwVaz9mSsIS8baUHOTDdTu4PCON5gn6ygbjO2PTmDGxN3/5dDOvLdsSkn1s3nmApz/dxCVj0xirYu3HTN9oiXgvLCzAAVNP1FBEY9w9eSjjenfg9rnLWbet6Suu3ffaKlokJnDHeSrWfjyUhCWiVdfUMmdhPqcPTFUdgkZKSmzGH68cS5vkRG54vmkrrr2/Zjsfrivm1rNUrP14KQlLRPtg7Q6276vQFXLHqGu7ZP4wfSwFu8v48ZymqbhWUf2fYu0zTu5z/EHGOSVhiWiZ2fl0bdeCswZ38TuUqDW+b0funjyE99Zs5w8f5h73+/3lk0Cx9nsuVLH2pqB/QYlYBbvL+Gh9MVec2ItETcgdl/86uQ9TRvfg9++t55/rdhzz+9Qv1n76CSrW3hT0zZaINWdhAQZMPVHnnx4vM+OB74xgUNe23PrC0mOuuPbAm2upcY6fT1ax9qaiJCwRqaqmljk5BXxzUBd6pLT0O5yY0CopkaeuHodzjuufW8TBysZVXMvevJv5y7Zww+n96NVJk6RNRUlYItJ7q7dTXKoJuabWu1Nr/m/qaFZv3cfdr64IuuJaTa3jnvmr6NE+mRvPGBDiKOOLkrBEpMzsfHq0T+aMQZqQa2pnDu7Kj84eyCuLi3j+87ygXpOZnc+arfu4e/JQFWtvYkrCEnHydh3gkw07ueLEXiToUtiQuOXMgZw5uAv3vraaRXm7j7jtngOVPLRgHRP7deL8Earj3NSUhCXizM4uIKGZcYUm5EKmWTPj4ctH0yOlJTc+v5gdpeUNbvvQu+soLa/mnouGqlh7CCgJS0SprK5l7qICzhzchW7tdSVWKLVv1Zynrh7HvvIqbpq1hKrDVFxbtWUvmVmBYu2Du7XzIcrY50sSNrMUM5trZmvNbI2ZTTSzjmb2rplt8H528LY1M3vUzHLNbLmZjfUjZgmPBau3sXN/pSbkwmRI93b85pKRZH+xm1+9ueYr6+qKtae0SuK2s1WsPVT86gk/ArztnBsMjALWAHcC7zvnBgLve88BzgMGeo/rgCfCH66ES2ZWPmkdWnL6QF0IEC5TRvfku6f04a//+oJ5S4u+bJ+/bAsLv/CKtbdSsfZQCXsSNrP2wOnA0wDOuUrnXAkwBXjW2+xZ4GJveQrwdxfwOZBiZt3DHLaEweadB/j3xl1MG68JuXD7n/OHML5PR+54eTlrtu7jgFesfUTP9lymYu0h5UdPuC9QDPzVzJaY2V/MrDXQ1Tm31dtmG1BXvbsnUFDv9YVe21eY2XVmlmNmOcXFxSEMX0JldnY+ic2MyzLS/A4l7jRPaMbjV46hXXJzbnh+EQ+8tYbt+wLF2vUfYmj5kYQTgbHAE865McAB/jP0AIALnEHeqHJPzrk/OecynHMZqan6UzbalFfV8FJOAd8a2lWlEX3SpW0yT1w1li0lB3n+83y+M7Yn43qrWHuo+ZGEC4FC51yW93wugaS8vW6YwftZV2WkCKj/91Ca1yYx5J1V29hTVqUJOZ+N692R+6cM54Subbhz0mC/w4kLYU/CzrltQIGZ1ZXjPwtYDcwHZnhtM4B53vJ84BrvLIkJwN56wxYSI2Zl5dOrYytO6d/Z71Di3tTxvVhw2zfo0k5/kYRDok/7vRmYZWZJwCbguwT+Q3jRzK4F8oDLvW3fBM4HcoEyb1uJIbk7SsnevJs7Jg3WzSIl7viShJ1zS4GMw6w66zDbOuCHIQ9KfJOZVUDzBE3ISXzSFXPiq/KqGl5eXMg5w7rRuU0Lv8MRCTslYfHVmyu2svdgFVeO14ScxCclYfFVZlY+/Tq3ZmL/Tn6HIuILJWHxzfrtpeTk7WHa+F6qziVxS0lYfJOZlU9SQjMuGacJOYlfSsLii4OVgQm580Z0o2PrJL/DEfGNkrD44rXlWygtr2a6JuQkzgWVhM3sVjNr51219rSZLTazc0IdnMSuzKx8BnRpw/i+Hf0ORcRXwfaE/9s5tw84B+gAXA38OmRRSUxbvWUfSwtKNCEnQvBJuO435XzgOefcqnptIo2SmZ1HUmIzLhn7tYqkInEn2CS8yMwWEEjC75hZW+DrN6QSOYoDFdW8umQLF4zoTkorTciJBFs74lpgNLDJOVdmZp1QIR05Bq8t28L+imqunKAJORE4ShI+zE01+2kMT45HZnY+g7q2ZWwvFQsXgaP3hB/yfiYD44DlBMaCRwI5wMTQhSaxZmXRXpYX7uXei4ZpQk7Ec8QxYefcN51z3wS2AuO82weNA8agu1tII83Kyie5eTMuHqMJOZE6wU7MDXLOrah74pxbCQwJTUgSi0rLq5i3tIgLR/agfUvdPl2kTrATcyvM7C/A897zKwkMTYgEZd7SLZRV1ugeciKHCDYJ/xdwI3Cr9/xj4IlQBCSxxzlHZlY+Q7q3Y3R6it/hiESUoyZhM0sA3vLGhh8OfUgSa5YV7mX11n3cf/FwTciJHOKoY8LOuRqg1szahyEeiUGZWXm0Skrg4tE9/A5FJOIEOxyxn8C48LvAgbpG59wtIYlKYsa+8ipeW7aVKaN70DZZE3Iihwo2Cb/iPUQa5dUlRRysquHKk3r7HYpIRAoqCTvnng11IBJ76ibkRvRsz4g0jWaJHE6w9YQHmtlcM1ttZpvqHqEOTqLb4vwS1m4r1WlpIkcQ7MUafyVwSlo18E3g7/znnGGRw5qVlUebFolcNEoTciINCTYJt3TOvQ+Ycy7POTcTmBy6sCTa7S2r4o3lgQm51i2CnXoQiT/B/nZUmFkzYIOZ3USgbkSb0IUl0e7lxYVUVNdqKELkKILtCd8KtAJuIVBN7SpgRqiCkujmnCMzO59R6SkM66EJOZEjCbYnvNs5t5/A+cIq5i5HtPCLPeTu2M9vLxnpdygiES/YJPyMmaUBC4FPgI/rV1UTqS8zK4+2LRK5YFR3v0MRiXjBnif8DTNLAk4EzgDeMLM2zjndr1y+Ys+BSt5cuY2pJ6bTKkkTciJHE9RviZmdCpzmPVKA1wn0iEW+4uXFhVRqQk4kaMF2Vf4JLAIeAN50zlWGLCKJWnUTcuN6d2Bwt3Z+hyMSFYI9O6IzcB+Be8q9bWbvmdn9oQtLotHnm3azqfgA08erFywSrGDHhEu8y5TTgTTgZEAlseQrZmXl0b5lcyaP1IScSLCCHRPeBKwFPiVw+fJ3NSQh9e3cX8E7q7Zx1YTeJDdP8DsckagR7JjwAOdcbUgjkag2d1EhVTWOKzUhJ9IowY4JDzCz981sJYCZjTSzn4cwLokitbWO2dn5jO/TkQFd2vodjkhUCTYJ/xm4C6gCcM4tB6aGKiiJLv/euIu8XWU6LU3kGASbhFs557IPaatu6mAkOmVm59GhVXMmDe/mdygiUSfYJLzTzPoDDsDMLgW2hiwqiRo7SstZsGo7l4xN04ScyDEIdmLuh8CfgMFmVgRsBq4MWVQSNV7KKaS61jFNQxEixyTY84Q3AWebWWsCvecyAmPCeSGMTSJcba3jhYX5TOzXif6pKi8tciyOOBxhZu3M7C4ze9zMvkUg+c4AcoHLwxGgRK5PcndSsPugJuREjsPResLPAXuAz4DvA3cDBnzbObc0xLFJhJv1eR6dWidx7jBNyIkcq6Ml4X7OuREAZvYXApNxvZxz5SGPTCLa9n3lvL92B987rS9JicHO74rIoY7221NVt+CcqwEKlYAFYM7CAmpqHdNO1FCEyPE4Wk94lJnt85YNaOk9N8A551SvMA7V1DpeyM7n1AGd6dO5td/hiES1IyZh55xO/JSv+Wj9DrbsLefnFwz1OxSRqKfBPGm0zKx8OrdpwbeGdvU7FJGopyQsjbKl5CAfrN3B5RlpNE/Q10fkePn2W2RmCWa2xMxe9573NbMsM8s1sznejUUxsxbe81xvfR+/YpbAhJwDpunuGSJNws+uzK3AmnrPfwM87JwbQODc5Gu99muBPV77w9524oPqmlrmLCzg9IGppHds5Xc4IjHBlyRsZmnAZOAv3nMDzgTmeps8C1zsLU/xnuOtP8vbXsLsw3XFbNtXrivkRJqQXz3h/wNuB+ru1tEJKHHO1ZXHLAR6ess9gQIAb/1eb/uvMLPrzCzHzHKKi4tDGXvcmpWVR9d2LThrcBe/QxGJGWFPwmZ2AbDDObeoKd/XOfcn51yGcy4jNTW1Kd9agILdZXy0vpgrMtJJ1IScSJMJtpRlUzoFuMjMzgeSgXbAI0CKmSV6vd00oMjbvojAXZ4LzSwRaA/sCn/Y8W3OwgIMuEITciJNKuxdGufcXc65NOdcHwLlMD9wzl0JfAhc6m02A5jnLc/3nuOt/8A558IYctyrqqllTk4BZwzqQs+Uln6HIxJTIunvyjuAH5tZLoEx36e99qeBTl77j4E7fYovbr2/ZjvFpRVMVy9YpMn5MRzxJefcP4F/esubgPGH2aYcuCysgclXzMrKp3v7ZM4YpLF2kaYWST1hiUD5u8r4ZMNOrjhRE3IioaDfKjmi2QvzSWhmTFXJSpGQUBKWBlVW1/JSTgFnDu5Ct/bJfocjEpOUhKVB767ezs79lbpCTiSElISlQbOy8uiZ0pLTB2pCTiRUlITlsDbvPMC/N+5i2vh0EpqpVIdIqCgJy2HNzs4nsZlxeUa636GIxLNteBAAABHJSURBVDQlYfmaiuoa5i4q5OwhXenSThNyIqGkJCxf8/bKbew+oAk5kXBQEpavyczKp1fHVpw6oLPfoYjEPCVh+YrcHfvJ2rybqePTaaYJOZGQUxKWr5idnU/zBOOycZqQEwkHJWH5UnlVDS8vLuScYd1IbdvC73BE4oKSsHzpzRVbKSmr4kqVrBQJGyVh+VJmVj59O7dmYv+v3cJPREJESVgAeG3ZFnLy9nDlSb3QzaxFwkdJWMjbdYC7XlnB2F4pzDi5j9/hiMQVJeE4V1ldy82zl9DM4NFpY2iuwu0iYeXr7Y3Ef799ey3LC/fy1NXjSOvQyu9wROKOuj1x7IO12/nLp5uZMbE35w7r5nc4InFJSThObd17kJ+8uIyh3dtx1/lD/A5HJG4pCceh6ppabn1hKRXVtTw+fQzJzRP8DkkkbmlMOA49+kEu2Zt38/AVo+iX2sbvcETimnrCcebfG3fy2AcbuGRsGt8ek+Z3OCJxT0k4juzcX8GPXlhK386tuW/KML/DERGUhONGba3jJy8uo+RgFX+YPpbWLTQSJRIJlITjxJ8/2cRH64v5xQVDGdK9nd/hiIhHSTgOLM7fw+/eWcd5w7txlW5ZJBJRlIRj3N6DVdwyewnd2ifz60tGqjiPSITRwGAMc85x58vL2ba3nJdumEj7ls39DklEDqGecAyblZXPWyu38bNzBzGmVwe/wxGRw1ASjlFrtu7jvtdXc8agVL5/Wj+/wxGRBigJx6CyympuylxMSsvmPHTZKN01WSSCaUw4Bv3vvFVs2nmAWd87iU5tdMNOkUimnnCM+ceSQuYuKuTmbw7g5P6d/Q5HRI5CSTiGbCrez93/WMn4Ph255ayBfocjIkFQEo4R5VU13JS5hBaJzXhk2mgSdZsikaigMeEY8cCba1i9dR9Pz8ige/uWfocjIkFSdykGvL1yG89+lse1p/blrCFd/Q5HRBpBSTjKFe4p4/a5yxiZ1p47Jg32OxwRaSQl4ShWVVPLLbOXUOvgsWljSErUxykSbTQmHMUefnc9i/NLeGzaGHp3au13OCJyDNR1ilKfbCjmiY82Mm18OheO6uF3OCJyjJSEo9CO0nJum7OUgV3a8L8X6DZFItFMwxFRprbW8eM5y9hfUU3m9yfQMkm3qxeJZkrCUeaJjzbyae5Ofv2dEZzQta3f4YjIcdJwRBTJ+WI3v393PReO6sEVJ6b7HY6INAEl4ShRUlbJLbOXkNahJb/69nDdpkgkRmg4Igo45/jpS8sp3l/ByzeeTNtk3aZIJFaoJxwF/vbvL3hvzXbuPG8II9NS/A5HRJpQ2JOwmaWb2YdmttrMVpnZrV57RzN718w2eD87eO1mZo+aWa6ZLTezseGO2U8ri/bywJtrOXtIF/77lD5+hyMiTcyPnnA18BPn3FBgAvBDMxsK3Am875wbCLzvPQc4DxjoPa4Dngh/yP7YXxG4TVGnNkn87tJRGgcWiUFhT8LOua3OucXecimwBugJTAGe9TZ7FrjYW54C/N0FfA6kmFn3MIcdds457v7HCvJ3l/HI1DF0aJ3kd0giEgK+jgmbWR9gDJAFdHXObfVWbQPqajL2BArqvazQazv0va4zsxwzyykuLg5ZzOHy0qJC5i3dwm1nn8D4vh39DkdEQsS3JGxmbYCXgR855/bVX+ecc4BrzPs55/7knMtwzmWkpqY2YaThl7ujlHvmreLk/p34wTcH+B2OiISQL0nYzJoTSMCznHOveM3b64YZvJ87vPYioP6VCWleW0wqr6rhh7OW0Copgf+7YjQJul29SEzz4+wIA54G1jjnfl9v1Xxghrc8A5hXr/0a7yyJCcDeesMWMee+11ezbnspD10+ii7tkv0OR0RCzI+LNU4BrgZWmNlSr+1/gF8DL5rZtUAecLm37k3gfCAXKAO+G95ww+eN5VvJzMrn+m/044xBXfwOR0TCIOxJ2Dn3KdDQ39hnHWZ7B/wwpEFFgPxdZdz58nLG9Erhp+cM8jscEQkTXTEXASqra7l59mLM4NGpY2iu29WLxA3VjogAv3tnLcsK9/LkVWNJ79jK73BEJIzU5fLZB2u38+dPNnP1hN5MGh7z16CIyCGUhH20bW85P3lxGUO6t+PuyUP8DkdEfKAk7JOaWsetLyyhorqWx6ePIbm5blMkEo80JuyTxz7YQNbm3Tx02Sj6p7bxOxwR8Yl6wj74fNMuHn1/A98Z25NLxqX5HY6I+EhJOMx27a/g1heW0KdTa+6fMtzvcETEZxqOCKPaWsdPX1rGnrIqnvmvE2ndQv/8IvFOPeEwevrTzXy4rpifTx7CsB7t/Q5HRCKAknCYLC0o4Tdvr+XcYV25ekJvv8MRkQihJBwG+8qruHn2Yrq2S+a3l+g2RSLyHxqUDDHnHHe9vIItJeW8eP1E2rfS7epF5D/UEw6xzOx83lixlZ+eM4hxvTv4HY6IRBgl4RBau20f9722mtNPSOX60/v5HY6IRCAl4RApq6zmh7MW065lc35/+Sia6TZFInIYGhMOkZnzV7Fp5wGev/YkOrdp4Xc4IhKh1BMOgXlLi3gxp5AfnjGAUwZ09jscEYlgSsJNbPPOA/zPKys4sU8HfnT2QL/DEZEIpyTchCqqa7h59mKaJzbjkaljSNRtikTkKDQm3IQeeHMtK4v28edrMuiR0tLvcEQkCqir1kQWrNrG3/79Bd89pQ/fGtrV73BEJEooCTeBopKD/Gzucob3bMed5w32OxwRiSJKwsepuqaWW2cvoabW8fi0sbRI1G2KRCR4GhM+Tg+/t56cvD08MnU0fTq39jscEYky6gkfh0837OSP/9zIFRnpTBnd0+9wRCQKKQkfo+LSCn40ZykDUtsw86JhfocjIlFKwxHHoLbW8eMXl1JaXsWs751EyySNA4vIsVFP+Bg8+fFGPtmwk3suHMagbm39DkdEopiScCMtytvNQwvWM3lkd6aNT/c7HBGJckrCjVBSVskts5fSM6UlD3xnhG5TJCLHTWPCQXLOcfvc5ewoLWfuDSfTLlm3KRKR46eecJD+/lkeC1Zv545JgxmVnuJ3OCISI5SEg7CyaC+/fGMNZw7uwrWn9vU7HBGJIUrCR7G/opqbZy+hY+skHrxMt6sXkaalMeEjcM7xi1dXkrfrALO/P4GOrZP8DklEYox6wkcwd1Eh/1hSxK1nncBJ/Tr5HY6IxCAl4Qbk7ijlf+etYkK/jtx05gC/wxGRGKUkfBjlVTXclLmElkkJPDJ1DAm6Xb2IhIjGhA/j/tdXs3ZbKX/97ol0bZfsdzgiEsPUEz7Emyu2Misrn+tO78c3B3XxOxwRiXFKwvUU7C7jjpeXMyo9hZ+eM8jvcEQkDigJe6pqarl59hIAHp82hqRE/dOISOhpTNjz4DvrWFpQwh+vHEt6x1Z+hyMicULdPeDDdTt46uNNXHlSL84f0d3vcEQkjigJA93bJzN5RHd+ccFQv0MRkThjzjm/Y2hyGRkZLicnx+8wRCQ+NerCAvWERUR8pCQsIuIjJWERER8pCYuI+ChqkrCZTTKzdWaWa2Z3+h2PiEhTiIokbGYJwB+A84ChwDQz0/lkIhL1oiIJA+OBXOfcJudcJfACMMXnmEREjlu0JOGeQEG954Ve25fM7DozyzGznOLi4rAGJyJyrKIlCR+Vc+5PzrkM51xGamqq3+GIiAQlWpJwEZBe73ma1yYiEtWiJQkvBAaaWV8zSwKmAvN9jklE5LhFRSlL51y1md0EvAMkAM8451b5HJaIyHGLyQI+ZlYM5DXyZZ2BnSEIx0+xdkw6nsgWa8cDx3ZMO51zk4LdOCaT8LEwsxznXIbfcTSlWDsmHU9ki7XjgfAcU7SMCYuIxCQlYRERHykJ/8ef/A4gBGLtmHQ8kS3WjgfCcEwaExYR8ZF6wiIiPlISFhHxkZIwkV+r2My+MLMVZrbUzHK8to5m9q6ZbfB+dvDazcwe9Y5luZmNrfc+M7ztN5jZjHrt47z3z/Ve26gbFQYR/zNmtsPMVtZrC3n8De0jRMcz08yKvM9oqZmdX2/dXV5s68zs3Hrth/3eeVeGZnntc7yrRDGzFt7zXG99nyY6nnQz+9DMVpvZKjO71WuP5s+ooWOKvM/JORfXDwJX4G0E+gFJwDJgqN9xHRLjF0DnQ9p+C9zpLd8J/MZbPh94i8AdXycAWV57R2CT97ODt9zBW5ftbWvea89r4vhPB8YCK8MZf0P7CNHxzAR+ephth3rfqRZAX++7lnCk7x3wIjDVW34SuNFb/gHwpLc8FZjTRMfTHRjrLbcF1ntxR/Nn1NAxRdzn5HuC8fsBTATeqff8LuAuv+M6JMYv+HoSXgd0r/eFW+ctPwVMO3Q7YBrwVL32p7y27sDaeu1f2a4Jj6EPX01aIY+/oX2E6Hga+uX+yveJwKX3Exv63nlJaieQeOj3s+613nKit52F4LOaB3wr2j+jBo4p4j4nDUcEUas4AjhggZktMrPrvLauzrmt3vI2oKu33NDxHKm98DDtoRaO+BvaR6jc5P15/ky9P6sbezydgBLnXPUh7V95L2/9Xm/7JuP96TwGyCJGPqNDjgki7HNSEo4OpzrnxhK4vdMPzez0+itd4L/cqD3XMBzxh2EfTwD9gdHAVuChEO4rJMysDfAy8CPn3L7666L1MzrMMUXc56QkHAW1ip1zRd7PHcA/CNzuabuZdQfwfu7wNm/oeI7UnnaY9lALR/wN7aPJOee2O+dqnHO1wJ8JfEYcJe7Dte8CUsws8ZD2r7yXt769t/1xM7PmBJLVLOfcK15zVH9GhzumSPyclIQjvFaxmbU2s7Z1y8A5wEoCMdbNPs8gMOaF136NN4M9Adjr/bn3DnCOmXXw/gQ7h8AY1lZgn5lN8Gasr6n3XqEUjvgb2keTq0sknm8T+IzqYpjqzZj3BQYSmKQ67PfO6w1+CFx6mLjrH8+lwAfe9scbuwFPA2ucc7+vtypqP6OGjikiP6dQDIJH24PAbO96ArOgd/sdzyGx9SMwI7sMWFUXH4ExpveBDcB7QEev3QjcmXojsALIqPde/w3keo/v1mvP8L6MG4HHaeLJHmA2gT/9qgiMnV0bjvgb2keIjuc5L97l3i9h93rb3+3Fto56Z5409L3zPvNs7zhfAlp47cne81xvfb8mOp5TCQwDLAeWeo/zo/wzauiYIu5z0mXLIiI+0nCEiIiPlIRFRHykJCwi4iMlYRERHykJi4j4SElYooaZdapX/WpbvWpY+83sjyHc7xlmdnKo3l/iW+LRNxGJDM65XQQuN8XMZgL7nXMPhmHXZwD7gX+HYV8SZ9QTlqjn9VRf95ZnmtmzZvaJmeWZ2XfM7LcWqGX7tncpa11924+8okjv1Lt09hYL1KBdbmYveMVfbgBu83rdp5lZqpm9bGYLvccp9fb9nJl9ZoH6uN/32rub2cfe61ea2Wl+/DtJZFJPWGJRf+CbBGrEfgZc4py73cz+AUw2szeAx4ApzrliM7sC+CWBq73uBPo65yrMLMU5V2JmT1Kv121mmcDDzrlPzawXgct1h3j7Hkmgbm5rYIm3r2kELt/9pZklAK3C888g0UBJWGLRW865KjNbQaAo99te+woCdYAHAcOBdwMlBkggcBkyBC5nnWVmrwKvNvD+ZwND7T83IGlngWpdAPOccweBg2b2IYECMQuBZ7xe+KvOuaVNc5gSC5SEJRZVADjnas2syv3n2vxaAt95A1Y55yYe5rWTCdw540LgbjMbcZhtmgETnHPl9Ru9pHxoHQDnnPvYAuVHJwN/M7PfO+f+fozHJjFGY8ISj9YBqWY2EQIlD81smJk1A9Kdcx8CdxAoQdgGKCVwi5w6C4Cb656Y2eh666aYWbKZdSIwobfQzHoD251zfwb+QuDWSCKAkrDEIedcJYESg78xs2UEKmydTGBY4nlvGGMJ8KhzrgR4Dfh23cQccAuQ4U3erSYwcVdnOYESh58D9zvnthBIxsvMbAlwBfBIOI5TooOqqIk0kTCfNicxQj1hEREfqScsIuIj9YRFRHykJCwi4iMlYRERHykJi4j4SElYRMRH/x8aU9++ntzw8wAAAABJRU5ErkJggg==",
            "text/plain": [
              "<Figure size 360x360 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          },
          "output_type": "display_data"
        }
      ],
      "source": [
        "import seaborn as sns\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "rewards_data = [31.030509762465954, 327.19548013806343, 869.8522652685642, 591.970870397985, 963.6337747052312]\n",
        "data = pd.DataFrame(data={\"Timesteps\": [0, 62_500, 125_000, 187_500, 250_000], \"Rewards\": rewards_data})\n",
        "sns.relplot(data=data, x=\"Timesteps\", y=\"Rewards\", kind=\"line\")\n",
        "plt.savefig('rewards_graph.png')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xgTYODv-Qjv_"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "colab": {
      "name": "Untitled0.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}