Danny_A2C_KarchowRL.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8uZ0wB5qJXda",
        "outputId": "a42b51a5-6829-402a-bb11-259d7cbe28ef"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "x11-utils is already the newest version (7.7+3build1).\n",
            "xvfb is already the newest version (2:1.19.6-1ubuntu4.10).\n",
            "0 upgraded, 0 newly installed, 0 to remove and 39 not upgraded.\n",
            "Requirement already satisfied: pyvirtualdisplay==0.2.* in /usr/local/lib/python3.7/dist-packages (0.2.5)\n",
            "Requirement already satisfied: EasyProcess in /usr/local/lib/python3.7/dist-packages (from pyvirtualdisplay==0.2.*) (1.1)\n",
            "Requirement already satisfied: stable-baselines3[extra] in /usr/local/lib/python3.7/dist-packages (1.5.0)\n",
            "Requirement already satisfied: cloudpickle in /usr/local/lib/python3.7/dist-packages (from stable-baselines3[extra]) (1.3.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from stable-baselines3[extra]) (1.21.5)\n",
            "Collecting gym==0.21\n",
            "  Using cached gym-0.21.0-py3-none-any.whl\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.7/dist-packages (from stable-baselines3[extra]) (3.2.2)\n",
            "Requirement already satisfied: torch>=1.8.1 in /usr/local/lib/python3.7/dist-packages (from stable-baselines3[extra]) (1.10.0+cu111)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from stable-baselines3[extra]) (1.3.5)\n",
            "Requirement already satisfied: autorom[accept-rom-license]~=0.4.2 in /usr/local/lib/python3.7/dist-packages (from stable-baselines3[extra]) (0.4.2)\n",
            "Requirement already satisfied: tensorboard>=2.2.0 in /usr/local/lib/python3.7/dist-packages (from stable-baselines3[extra]) (2.8.0)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.7/dist-packages (from stable-baselines3[extra]) (5.4.8)\n",
            "Requirement already satisfied: opencv-python in /usr/local/lib/python3.7/dist-packages (from stable-baselines3[extra]) (4.1.2.30)\n",
            "Requirement already satisfied: ale-py~=0.7.4 in /usr/local/lib/python3.7/dist-packages (from stable-baselines3[extra]) (0.7.4)\n",
            "Requirement already satisfied: pillow in /usr/local/lib/python3.7/dist-packages (from stable-baselines3[extra]) (7.1.2)\n",
            "Requirement already satisfied: importlib-metadata>=4.8.1 in /usr/local/lib/python3.7/dist-packages (from gym==0.21->stable-baselines3[extra]) (4.11.3)\n",
            "Requirement already satisfied: importlib-resources in /usr/local/lib/python3.7/dist-packages (from ale-py~=0.7.4->stable-baselines3[extra]) (5.4.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from autorom[accept-rom-license]~=0.4.2->stable-baselines3[extra]) (2.23.0)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from autorom[accept-rom-license]~=0.4.2->stable-baselines3[extra]) (4.63.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from autorom[accept-rom-license]~=0.4.2->stable-baselines3[extra]) (7.1.2)\n",
            "Requirement already satisfied: AutoROM.accept-rom-license in /usr/local/lib/python3.7/dist-packages (from autorom[accept-rom-license]~=0.4.2->stable-baselines3[extra]) (0.4.2)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=4.8.1->gym==0.21->stable-baselines3[extra]) (3.10.0.2)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=4.8.1->gym==0.21->stable-baselines3[extra]) (3.7.0)\n",
            "Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (57.4.0)\n",
            "Requirement already satisfied: google-auth<3,>=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (1.35.0)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (1.0.1)\n",
            "Requirement already satisfied: absl-py>=0.4 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (1.0.0)\n",
            "Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (0.37.1)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (3.3.6)\n",
            "Requirement already satisfied: protobuf>=3.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (3.17.3)\n",
            "Requirement already satisfied: grpcio>=1.24.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (1.44.0)\n",
            "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (0.6.1)\n",
            "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (0.4.6)\n",
            "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (1.8.1)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from absl-py>=0.4->tensorboard>=2.2.0->stable-baselines3[extra]) (1.15.0)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard>=2.2.0->stable-baselines3[extra]) (4.8)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard>=2.2.0->stable-baselines3[extra]) (0.2.8)\n",
            "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard>=2.2.0->stable-baselines3[extra]) (4.2.4)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard>=2.2.0->stable-baselines3[extra]) (1.3.1)\n",
            "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.7/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard>=2.2.0->stable-baselines3[extra]) (0.4.8)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->autorom[accept-rom-license]~=0.4.2->stable-baselines3[extra]) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->autorom[accept-rom-license]~=0.4.2->stable-baselines3[extra]) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->autorom[accept-rom-license]~=0.4.2->stable-baselines3[extra]) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->autorom[accept-rom-license]~=0.4.2->stable-baselines3[extra]) (2021.10.8)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard>=2.2.0->stable-baselines3[extra]) (3.2.0)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->stable-baselines3[extra]) (2.8.2)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->stable-baselines3[extra]) (3.0.7)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->stable-baselines3[extra]) (1.4.0)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.7/dist-packages (from matplotlib->stable-baselines3[extra]) (0.11.0)\n",
            "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.7/dist-packages (from pandas->stable-baselines3[extra]) (2018.9)\n",
            "Installing collected packages: gym\n",
            "  Attempting uninstall: gym\n",
            "    Found existing installation: gym 0.23.1\n",
            "    Uninstalling gym-0.23.1:\n",
            "      Successfully uninstalled gym-0.23.1\n",
            "Successfully installed gym-0.21.0\n",
            "Requirement already satisfied: gym in /usr/local/lib/python3.7/dist-packages (0.21.0)\n",
            "Collecting gym\n",
            "  Using cached gym-0.23.1-py3-none-any.whl\n",
            "Requirement already satisfied: numpy>=1.18.0 in /usr/local/lib/python3.7/dist-packages (from gym) (1.21.5)\n",
            "Requirement already satisfied: gym-notices>=0.0.4 in /usr/local/lib/python3.7/dist-packages (from gym) (0.0.6)\n",
            "Requirement already satisfied: cloudpickle>=1.2.0 in /usr/local/lib/python3.7/dist-packages (from gym) (1.3.0)\n",
            "Requirement already satisfied: importlib-metadata>=4.10.0 in /usr/local/lib/python3.7/dist-packages (from gym) (4.11.3)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=4.10.0->gym) (3.10.0.2)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=4.10.0->gym) (3.7.0)\n",
            "Installing collected packages: gym\n",
            "  Attempting uninstall: gym\n",
            "    Found existing installation: gym 0.21.0\n",
            "    Uninstalling gym-0.21.0:\n",
            "      Successfully uninstalled gym-0.21.0\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "stable-baselines3 1.5.0 requires gym==0.21, but you have gym 0.23.1 which is incompatible.\u001b[0m\n",
            "Successfully installed gym-0.23.1\n",
            "Requirement already satisfied: gym[box2d] in /usr/local/lib/python3.7/dist-packages (0.23.1)\n",
            "Requirement already satisfied: gym-notices>=0.0.4 in /usr/local/lib/python3.7/dist-packages (from gym[box2d]) (0.0.6)\n",
            "Requirement already satisfied: importlib-metadata>=4.10.0 in /usr/local/lib/python3.7/dist-packages (from gym[box2d]) (4.11.3)\n",
            "Requirement already satisfied: cloudpickle>=1.2.0 in /usr/local/lib/python3.7/dist-packages (from gym[box2d]) (1.3.0)\n",
            "Requirement already satisfied: numpy>=1.18.0 in /usr/local/lib/python3.7/dist-packages (from gym[box2d]) (1.21.5)\n",
            "Requirement already satisfied: pygame==2.1.0 in /usr/local/lib/python3.7/dist-packages (from gym[box2d]) (2.1.0)\n",
            "Requirement already satisfied: box2d-py==2.3.5 in /usr/local/lib/python3.7/dist-packages (from gym[box2d]) (2.3.5)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=4.10.0->gym[box2d]) (3.7.0)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=4.10.0->gym[box2d]) (3.10.0.2)\n"
          ]
        }
      ],
      "source": [
        "!apt-get install -y xvfb x11-utils\n",
        "!pip install pyvirtualdisplay==0.2.*\n",
        "!pip install stable-baselines3[extra]\n",
        "#!pip install stable-baselines[mpi]\n",
        "!pip install gym --upgrade\n",
        "!pip install gym[box2d]"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HKLJMhGVpcAA",
        "outputId": "14c97a45-d1fe-4605-fdfd-20d700151663"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "of5iXeUHJmUg",
        "outputId": "d5038bbc-53bf-4e80-d2e5-f94613e332d5"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Using cpu device\n",
            "Wrapping the env in a VecTransposeImage.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/gym/core.py:173: DeprecationWarning: \u001b[33mWARN: Function `env.seed(seed)` is marked as deprecated and will be removed in the future. Please use `env.reset(seed=seed) instead.\u001b[0m\n",
            "  \"Function `env.seed(seed)` is marked as deprecated and will be removed in the future. \"\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Track generation: 957..1209 -> 252-tiles track\n",
            "Track generation: 1127..1413 -> 286-tiles track\n",
            "Track generation: 1191..1493 -> 302-tiles track\n",
            "Track generation: 993..1253 -> 260-tiles track\n",
            "Track generation: 1196..1499 -> 303-tiles track\n",
            "Track generation: 1364..1709 -> 345-tiles track\n",
            "Track generation: 1180..1479 -> 299-tiles track\n",
            "Track generation: 988..1242 -> 254-tiles track\n",
            "retry to generate track (normal if there are not manyinstances of this message)\n",
            "Track generation: 1121..1414 -> 293-tiles track\n",
            "Track generation: 1034..1297 -> 263-tiles track\n",
            "Track generation: 1120..1404 -> 284-tiles track\n",
            "Track generation: 1147..1438 -> 291-tiles track\n",
            "Track generation: 1145..1440 -> 295-tiles track\n",
            "retry to generate track (normal if there are not manyinstances of this message)\n",
            "Track generation: 1187..1488 -> 301-tiles track\n",
            "Track generation: 1165..1461 -> 296-tiles track\n",
            "Track generation: 1069..1340 -> 271-tiles track\n",
            "Track generation: 1023..1283 -> 260-tiles track\n",
            "Track generation: 1159..1453 -> 294-tiles track\n",
            "Track generation: 1268..1589 -> 321-tiles track\n",
            "Track generation: 1056..1324 -> 268-tiles track\n",
            "Track generation: 1303..1633 -> 330-tiles track\n",
            "Track generation: 1077..1351 -> 274-tiles track\n",
            "Track generation: 962..1211 -> 249-tiles track\n",
            "Track generation: 1189..1491 -> 302-tiles track\n",
            "Track generation: 1288..1625 -> 337-tiles track\n",
            "Track generation: 1066..1341 -> 275-tiles track\n",
            "Track generation: 993..1245 -> 252-tiles track\n",
            "Track generation: 1172..1469 -> 297-tiles track\n",
            "Track generation: 1045..1310 -> 265-tiles track\n",
            "Track generation: 1168..1464 -> 296-tiles track\n",
            "Track generation: 1095..1376 -> 281-tiles track\n",
            "retry to generate track (normal if there are not manyinstances of this message)\n",
            "Track generation: 1087..1363 -> 276-tiles track\n",
            "Track generation: 1342..1680 -> 338-tiles track\n",
            "Track generation: 1265..1586 -> 321-tiles track\n",
            "Track generation: 1252..1569 -> 317-tiles track\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<stable_baselines3.a2c.a2c.A2C at 0x7f67e0b524d0>"
            ]
          },
          "metadata": {},
          "execution_count": 3
        }
      ],
      "source": [
        "import gym\n",
        "from stable_baselines3 import PPO\n",
        "from stable_baselines3 import A2C\n",
        "#from stable_baselines import TRPO\n",
        "\n",
        "from stable_baselines3.common.env_util import make_vec_env\n",
        "import imageio\n",
        "import os\n",
        "import numpy as np\n",
        "\n",
        "# Parallel environments\n",
        "env = make_vec_env(\"CarRacing-v1\", n_envs=4)\n",
        "#del model  # delete trained model to demonstrate loading\n",
        "#model = A2C.load(\"/content/drive/MyDrive/A2C\",env=env)\n",
        "model = A2C(\"CnnPolicy\", env, learning_rate=0.0005, n_steps=1024, gamma=0.9, gae_lambda=0.999,use_rms_prop =False, seed=8,verbose=1)#42.003257751464844\n",
        "#model = A2C(\"CnnPolicy\", env, learning_rate=0.0003, n_steps=1024, gamma=0.99, gae_lambda=0.95, seed=8,verbose=1)#13.41351318359375\n",
        "#model = A2C(\"CnnPolicy\", env,seed=8,verbose=1)#12.216592788696289\n",
        "#model = A2C(\"CnnPolicy\", env, learning_rate=0.0003, n_steps=1024, gamma=0.99, gae_lambda=0.95,use_rms_prop =False, seed=8,verbose=1)#34.76980209350586\n",
        "\n",
        "#model.set_env(env)\n",
        "model.learn(total_timesteps=25000)\n",
        "#model.save('/content/drive/MyDrive/A2C')\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WmV0XmBQJrj1",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "247509e8-45c1-45d6-9336-c019a3022d2e"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Track generation: 1263..1584 -> 321-tiles track\n",
            "Track generation: 1184..1484 -> 300-tiles track\n",
            "Track generation: 1146..1436 -> 290-tiles track\n",
            "Track generation: 1123..1408 -> 285-tiles track\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 27%|██▋       | 274/1000 [01:04<02:50,  4.26it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Track generation: 1260..1579 -> 319-tiles track\n",
            "Track generation: 1183..1483 -> 300-tiles track\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 28%|██▊       | 275/1000 [01:04<03:02,  3.98it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Track generation: 1017..1284 -> 267-tiles track\n",
            "Track generation: 1176..1474 -> 298-tiles track\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 55%|█████▍    | 549/1000 [02:09<01:53,  3.97it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Track generation: 1116..1399 -> 283-tiles track\n",
            "Track generation: 1087..1365 -> 278-tiles track\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 55%|█████▌    | 552/1000 [02:10<01:51,  4.02it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Track generation: 1156..1455 -> 299-tiles track\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 82%|████████▏ | 823/1000 [03:12<00:38,  4.58it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Track generation: 1022..1290 -> 268-tiles track\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 82%|████████▎ | 825/1000 [03:12<00:37,  4.64it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Track generation: 1109..1392 -> 283-tiles track\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 1000/1000 [03:49<00:00,  4.35it/s]\n",
            "/usr/local/lib/python3.7/dist-packages/imageio/plugins/ffmpeg.py:727: DeprecationWarning: tostring() is deprecated. Use tobytes() instead.\n",
            "  self._proc.stdin.write(im.tostring())\n"
          ]
        }
      ],
      "source": [
        "#Validation\n",
        "from tqdm import tqdm\n",
        "obs = env.reset()\n",
        "frames = []\n",
        "rewards=[]\n",
        "for i in tqdm(range(1000)):\n",
        "  action, _states = model.predict(obs.copy(), deterministic=True)\n",
        "  obs, reward, done, info = env.step(action)\n",
        "  rewards.append(reward)\n",
        "  frames.append(env.render(\"rgb_array\"))\n",
        "  if done.all():\n",
        "    obs = env.reset()\n",
        "    break\n",
        "\n",
        "env.close()\n",
        "\n",
        "path = os.path.join(f\"eval.mp4\")\n",
        "frames = np.array(frames)\n",
        "imageio.mimsave(path, frames, fps=30)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "rewardLS=[]\n",
        "for i,r in enumerate(rewards):\n",
        "  rewardLS.append(r.mean())"
      ],
      "metadata": {
        "id": "S8wLNj3ZzgRA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "rewards[:10]"
      ],
      "metadata": {
        "id": "mZHwizmDHafF",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "634a7587-3b7e-4e8a-b308-e377222bf228"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[array([6.15     , 6.588963 , 6.820415 , 6.9422536], dtype=float32),\n",
              " array([-0.1, -0.1, -0.1, -0.1], dtype=float32),\n",
              " array([-0.1, -0.1, -0.1, -0.1], dtype=float32),\n",
              " array([-0.1, -0.1, -0.1, -0.1], dtype=float32),\n",
              " array([-0.1, -0.1, -0.1, -0.1], dtype=float32),\n",
              " array([-0.1, -0.1, -0.1, -0.1], dtype=float32),\n",
              " array([-0.1, -0.1, -0.1, -0.1], dtype=float32),\n",
              " array([-0.1, -0.1, -0.1, -0.1], dtype=float32),\n",
              " array([-0.1, -0.1, -0.1, -0.1], dtype=float32),\n",
              " array([-0.1, -0.1, -0.1, -0.1], dtype=float32)]"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "rewardLS[:10]"
      ],
      "metadata": {
        "id": "U_RSko5bhxz9",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "48f324e1-7ea5-47a4-dc16-783d4384e028"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[6.6254077, -0.1, -0.1, -0.1, -0.1, -0.1, -0.1, -0.1, -0.1, -0.1]"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import seaborn as sns\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt"
      ],
      "metadata": {
        "id": "VTy2GHabiNlu",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "aaac3550-568a-414b-fb0f-752c1e8cc45e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/scipy/fft/__init__.py:97: DeprecationWarning: The module numpy.dual is deprecated.  Instead of using dual, use the functions directly from numpy or scipy.\n",
            "  from numpy.dual import register_func\n",
            "/usr/local/lib/python3.7/dist-packages/scipy/sparse/sputils.py:17: DeprecationWarning: `np.typeDict` is a deprecated alias for `np.sctypeDict`.\n",
            "  supported_dtypes = [np.typeDict[x] for x in supported_dtypes]\n",
            "/usr/local/lib/python3.7/dist-packages/scipy/special/orthogonal.py:81: DeprecationWarning: `np.int` is a deprecated alias for the builtin `int`. To silence this warning, use `int` by itself. Doing this will not modify any behavior and is safe. When replacing `np.int`, you may wish to use e.g. `np.int64` or `np.int32` to specify the precision. If you wish to review your current use, check the release note link for additional information.\n",
            "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
            "  from numpy import (exp, inf, pi, sqrt, floor, sin, cos, around, int,\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ql__PtCbJsS2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 369
        },
        "outputId": "c30f3351-41fa-418b-fc8a-430a10c2388a"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 360x360 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWAAAAFgCAYAAACFYaNMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd5xU5b3H8c9vly3AAkvvvSMgyIqg0gQFS1RssWGNSBJrIl5JTK43iVcTjSXXWLBrjCWKvaAozQK4CNKl9+LS6/bn/jFnlwUXmIEzc2Z2v+/Xa17MnDPld5jly7PPeZ7nmHMOERGJvaSgCxARqawUwCIiAVEAi4gERAEsIhIQBbCISEAUwCIiAakSdAGHYmYrgV1AEVDonMsyszrA60ArYCVwiXNuW1A1iogcC4vXccBeAGc55zaX2fY3YKtz7n4zuwuo7Zz7r8O9z7Bhw9wnn3wS3WJFRA7PytuYaF0Q5wEvevdfBM4/0gs2b958pKeIiAQingPYAZ+a2UwzG+lta+ic2+Dd3wg0LO+FZjbSzLLNLDsnJycWtYqIRCxu+4CBU51z68ysAfCZmS0qu9M558ys3P4T59xYYCxAVlZWfPaxiEilF7ctYOfcOu/PH4G3gd7AJjNrDOD9+WNwFYqIHJu4DGAzq25mNUruA2cA84D3gKu9p10NvBtMhSIixy5euyAaAm+bGYRq/Ldz7hMz+xZ4w8yuB1YBlwRYo4jIMYnLAHbOLQeOL2f7FmBw7CsSEfFfXHZBiIhUBgpgEZGAKIBFRAKiABYRCYgCWEQkIArgOOGc47Kx07jttVms2bo36HJEJAbichhaZbQnv4hvlm8B4KO5GxnRtyU3DWpH7eqpAVcmItGiFnCc2J1bCMAtg9tzfs8mPP/VCvo/MJEnJi0jt6Ao4OpEJBoUwHFiV24BAO0bZPC3i47n41v7c2KrOvz1k0UMenAS/8leQ1Gx1hUSqUgUwHHg5WmrGPHsDAAy0kO9Qh0b1eC5a07ktZF9aFAjjdFvzuHsf0xl4g8/Eq+L6ItIZBTAceC7VdvYuDMXgCa1qh6wr0+burzz61N47PKe7Cso4trnv+Xyp6czZ+32IEoVER8pgOPArtxCOjWqwdQ7B9GxUY2f7DczzunehM9uH8A9P+vCD5t2ce5jX3Hzq7NYvUUjJkQSlQI4DuzOK6BmegrN61Q77PNSqyRxzSmtmTx6IDcNasdnCzYy+KFJ/M/789m6Jz9G1YqIXxTAAdqyO4+T7/ucWau3UyM9/BGBNdJTuGNoRyaPHsRFvZrx4tcrGfC3ifxz4lL25WvEhEiiUAAHaNXWvazfkUteYTHNalc98gsO0rBmOvdd0J3xt/XnpDZ1eWD8Dwx6cBJvfKsREyKJQAEcoJKxv89dk8WYszof9fu0b1iDZ67O4o0b+9KoVjp3vjWHMx+dwheLNmnEhEgcUwAHaJcXwE0zq5GeknzM79e7dR3e/tXJPH7FCeQXFnPdC9lcOnYas9doxIRIPNJU5ICMGTeH92avB4io//dIzIyzujXm9C4NeXXGah6dsITz//kVZ3dvzJ1DO9KybnXfPktEjo1awAGZs3YHe/KLyEirQu1q/q/3kJKcxFV9WzH5zkHcMrg9Xyz8kcF/n8w9781ny+483z9PRCKnAA7I7rxCzunemK/HnEbV1GPvfjiUjLQq/Ob0DkwePZBLTmzOy9NWMeCBSTz2xRL25hdG7XNF5MgUwAHZlVtIZrUUaqanxOTzGtRM53+Hd2P8bf3o27YuD366mIEPTOK1GaspLCqOSQ0iciAFcIx9t3obHe/+mG1788lIi034ltWuQQ2eviqL/4zqS9PaVblr3FyGPTqVzxZoxIRIrCmAY2zJpl3kFYZanK3rHX7mWzSd2KoO4355Mk9eeQLFxY4bXsrm509NY9bqbYHVJFLZaBREjJUMPZs+ZjD1MtICrcXMGNa1MYM7N+S1b9fw6ITFDH/8a87q1ojRQzvRup5GTIhEkwI4htZs3cuOfaF1f+tmpJGUZAFXFJKSnMSIPi0Z3rMpT09ZztNTl/Pp/E1cflILbhncPvD/KEQqKgVwjKzasoeBD04iyYzqqckkx0n4lpWRVoXbT+/AFX1a8OiEJbwyfTVvzVzLyP5t+UW/1lRP04+LiJ/UBxwjG3bk4hxUT02mbYOMoMs5rAY10rl3eDc+vb0/p7avx8MTFjPwwUm8Mn2VRkyI+CguA9jMmpvZRDNbYGbzzexWb/s9ZrbOzGZ7t7OCrjVcJes+vHT9SYz75ckBVxOetvUzeGpEFm/9si8t61Tj92/PY+gjU5i3bkfQpYlUCHEZwEAh8FvnXBegD/BrM+vi7XvYOdfDu30UXInh27I7jxWb9wBQq2oKVZLj9a+9fL1a1uE/o/ry1IhebNqZx7Nfrgi6JJEKIS479ZxzG4AN3v1dZrYQaBpsVUfv9je+Z8riHCDUz5qIzIyhxzXi/hqLKNRSlyK+iPummJm1AnoC071NN5nZHDN7zsxqH+I1I80s28yyc3JyYlTpoW3csY9aVVPo3aoOtavFfvKFnww0YUPEJ3EdwGaWAbwF3Oac2wk8AbQFehBqIf+9vNc558Y657Kcc1n169ePWb2Hsiu3kDO6NOSNUX0TrvvhJwwUvyL+iNvfh80shVD4vuKcGwfgnNtUZv/TwAcBlReWNVv3svTH3ezKLSy93HyiM1ACi/gkLlPBzAx4FljonHuozPbGXv8wwHBgXhD1heuxL5byevYaAGokaN/vwcwMpwQW8UW8psIpwAhgrpnN9rb9DrjMzHoQaoOtBG4MprzwbN2bT53qqTSvXZUTW9cJuhxfhPqAg65CpGKIywB2zn2J99vuQRJi2FmJ3bmFtK1fnf+MSoxxv+EwUwCL+CUuAzjRrdm6lzdnrmXjztwKt6CNoS4IEb8k+Cn5+PTOrHU8+vkSVmzek7Djfg9FLWAR/1SsdIgTu/IKSU1O4tLezRnUqUHQ5fhO+SviDwVwFOzKLaRm1RT+dF7XoEvxnZmpBSziEwWwj96auZYXv1lJfmGxr5eajyehM6NKYBE/VMyUCMj0FVuYsza0UtjxzWoFXE10qA9YxD8KYB/tyi2kTf3q3Dq4PfVrVNyrSCh/RfyhAPbB92u2U6tqCrtyC6lVNYXzeiTswm1HFGoBK4JF/KAA9sGIZ6ezN7+I1CpJZLWqGDPeDiU0DlhE/KBxwMeooKiYnbmFtKhTjfzCYprUSg+6pKhSH7CIf9QCPkZ78kKXGhrRtyVnd29MtdSK/VdqqA9YxC8VOy2iaNWWPYz+zxxObV8PCF3pokGNit36BcBMfcAiPlEAH6XvVm9jxsqtzFi5FaDCjvs9WHkrJInI0akcqREFJVc5/sv5XZm6JIdeLSv2ybcS6gMW8Y8COAL5hcU8PmkpQzo3ZJfX93tRr2Zc2adlwJXFTqgPWAks4gcFcAS+W72NRyYs4ZEJS2hUM52UZCOtSuUaSKK1IET8U7nS4xjt8rodzunemG1782lQI53Q1ZMqD10RQ8Q/agEfxsdzN3D7G7O5+uRW/GpgO3bnFQDw2zM68sdzurA3vyjgCmPPTF0QIn5RAB/G4k27yS0oZuyU5bw2Yw1dm9YEQkPOKvJaD4djGgch4ht1QRxk6Y+7yS8sBmBXbgHVUpP54OZT6d6sFl8t3QJUniFn5dIoCBHfVOIk+am8wiKufm4GVZKN0UM7sjO3gIy0KhzXpBYvX38SXy7ZzIYd+0hPSQ661MBoJpyIfxTAZaQmJ3Hv8K7c//Eibvr3LJKTjJZ1q5XuL5n1VpmZgSsOugqRikEBXIaZMbBjA/q1r8+479by0GeLaVs/I+iy4kpoNTQlsIgfFMDlSE4yLs5qzoUnNAu6lLijmXAi/lEAH0ZSks74Hyw0DE1E/KBREBIRQ6uhifhFASwRUQtYxD8JF8BmNszMfjCzpWZ2V9D1VEZqAIv4I6EC2MySgX8CZwJdgMvMrEuwVVUuZromnIhfEiqAgd7AUufccudcPvAacF7ANVUqBmoCi/gk0QK4KbCmzOO13rYDmNlIM8s2s+ycnJyYFVcZqA9YxD+JFsBhcc6Ndc5lOeey6tevH3Q5FYqWoxTxT6IF8DqgeZnHzbxtEiOhPmAlsIgfEi2AvwXam1lrM0sFLgXeC7imSkUtYBH/JNRMOOdcoZndBIwHkoHnnHPzAy6rUtFUZBH/JFQAAzjnPgI+CrqOykvD0ET8kmhdEBKwUAtYESziBwWwRETLE4n4RwEsEVEfsIh/FMASkdCC7EpgET8ogCUiagGL+EcBLBHRVGQR/yiAJSJakF3EPwpgiZjiV8QfCmCJjKEEFvGJAlgiovwV8Y8CWCJipj5gEb8ogCUiagGL+EcBLBHROGAR/yiAJSKhFrASWMQPCmCJSKgPOOgqRCoGBbBERKuhifhHASyRUR+wiG8UwBIRUxtYxDcKYImIrogh4h8FsERE44BF/KMAlohoHLCIfxTAEhFdEUPEPwpgiYhawCL+UQBLRHRFDBH/KIAlQpoJJ+IXBbBExAzUBhbxhwJYImKoD1jEL3EXwGb2gJktMrM5Zva2mWV621uZ2T4zm+3dngy61spIfcAi/om7AAY+A7o657oDi4ExZfYtc8718G6jgimvctNVkUX8E3cB7Jz71DlX6D2cBjQLsh45kFrAIv6JuwA+yHXAx2UetzazWWY22cz6HepFZjbSzLLNLDsnJyf6VVYi6gMW8U+VID7UzCYAjcrZ9Xvn3Lvec34PFAKvePs2AC2cc1vMrBfwjpkd55zbefCbOOfGAmMBsrKyFBc+0kU5RfwTSAA754Ycbr+ZXQOcAwx23r9251wekOfdn2lmy4AOQHZ0q5WDKX5F/BF3XRBmNgy4EzjXObe3zPb6Zpbs3W8DtAeWB1Nl5WVaDk3EN4G0gI/gMSAN+MxCo/6neSMe+gN/MrMCoBgY5ZzbGlyZlVNoMR4R8UPcBbBzrt0htr8FvBXjcuQgWpBdxD9x1wUh8S1eeiAWbdzJ/PU7gi5D5JjEXQtY4lvQy1Huyy/irnFzeHf2egDO7t6YO4d2pGXd6sEVJXKUFMASEbPgFmRft30fp9z/RenjzGopfLHwRz6dv5ErTmrJzae1o25GWiC1iRwNBbBEJMiJGJN/CE2qqV0thbFXZdGjeSbb9uTzyOdLeHnaKt6cuZZRA9pw/altqJqaHEyRIhFQH7BEJsCpyMtzdpOeksR3fzidE1vVISU5iQY10/nf4d0Yf1t/Tm5blwc/XczAByfy2ozVFBYVB1SpSHgUwBIRCzCB123fR9PMqnjDEw/QrkEGY6/K4s1RfWmaWZW7xs3lzEenMmHBJo3akLilAJaIhBbjiX2gjRk3h4/nbaR1vYzDPi+rVR3e+uXJPHllL4qKHb94KZufPzWNWau3xahSkfApgCUiQfQBFxYV8+qMNQD87PjGR3y+mTGsayPG396fv5zfleWb9zD88a/51SszWbF5T7TLFQmbTsJJxGLd/t26Jx+AP5/flfN6NA37dSnJSVzZpyXDezbl6anLGTtlOZ/O38TlJ7XglsHtqacRExIwtYAlIkHMhNu2twAIjX44GtXTqnDbkA5MGj2QS3s355Xpqxnwt4n84/Ml7M0vPPIbiESJAlgiEuu1IGat3sbQR6YAULta6jG9V4Ma6fzl/G58ent/+rWvz0OfLWbAA5P493SNmJBgKIAlIrGeCffhnA2l91vWrebLe7atn8GTI3rx1i/70rJONX739lyGPjKFT+dv1IgJiSkFsETkpwPAomvNtr10aJjBoj8Po1ltfwK4RK+WdfjPqL6MHdELB4x8eSYXP/kNM1dpxITEhgJYIlPOGNxo2ra3gMxqqaSnRGdmm5lxxnGN+PS2/vzv8G6s2rqXC5/4mlEvz2R5zu6ofKZICQWwRKQkfqP9q7pzjhteymbGiq1Uj8G04irJSVx+Ugsmjx7Ib07vwNQlOZz+8BTufmcuObvyov75UjkpgCUiJQ3gaHeVrt22j88WbAKgU+Oa0f2wMqqlVuGWwe2ZfOcgrjipBa/NWMOABybyyITF7MnTiAnxlwJYImJeGzjap6rWbA1djer5a07kN6d3iPKn/VS9jDT+dF5XPvvNAAZ2rM8jE5Yw4IFJ/GvaKgo0YkJ8ogCWiOxvAUc3gkvG/jbOTCclObgf09b1qvP4Fb0Y96uTaVOvOne/M4+hD0/hk3kaMSHHTgEsESntA47iZ3wybwO//vd3wLGP/fXLCS1q8/qNfXjmqiySkoxR/5rJRU9+Q/ZKXZZQjl5YAWxmt5pZTQt51sy+M7Mzol2cxJ9Y9AGXrPuQkVYlbgIYQiMmhnRpyCe39uP+C7qxZuteLnryG0a+lM3SHzViQiIXbgv4OufcTuAMoDYwArg/alVJ3CpZCjKaK6Kt3baX07s05Kv/Oo3UKvH3S1qV5CQu7d2CSaMHcscZHfh62RaGPjKF3709lx935QZdniSQcH+6S37zPAt42Tk3n9iPyZc4Es0W8Pa9BdSvkUato1z7IVaqpVbhptPaM3n0QEb0ackb365h4AOTeOyLJeoflrCEG8AzzexTQgE83sxqADoVXAlFcx7GN8u20PveCWzZk3/UC+8EoW5GGvecexwTfjOAk1rX4cFPF7NEXRIShnAD+HrgLuBE59xeIBW4NmpVSdwqHYYWhQbex/M28KM36aFlncS7ynGretW5/KSWAOQVqH0iR3bY9YDN7ISDNrUp73IwUnmUnoSLQh/w+u376Ny4Ji9ceyJ1q8fPybdI7B8loi4IObIjLcj+d+/PdKAXMIfQz1h3IBvoG73SJB7tn4rs/3tv21tA7WopNKyZ7v+bx0isZgpKxXDYLgjn3CDn3CBgA9DLOZflnOsF9ATWxaJAiS/7W8D+cc5xyVOhVciqpyX2RVqi8fcjFVe4fcAdnXNzSx445+YBnaNRkJndY2brzGy2dzurzL4xZrbUzH4ws6HR+Hw5vP19wP5FzPLNe5ixIjShoWuTWr69b5A0CkLCEW5zY66ZPQP8y3t8BaHuiGh52Dn3YNkNZtYFuBQ4DmgCTDCzDs65oijWIQeJRgtv3bZ9ALw+sg+9W9fx8Z1jL1ZrZUjFEG4L+BpgPnCrd1tA7EdBnAe85pzLc86tAJYCvWNcg3j8auAVFzu27Q1ddLNuRioJf5JXfcASgSO2gM0sGfjY6wt+OPolAXCTmV1F6ETfb51z24CmwLQyz1nrbfsJMxsJjARo0aJFlEutXMznJvA1L3zLlMU5AGTG0bTjo7X/vw8lsBzZEVvA3q/4xWbmW+ecmU0ws3nl3M4DngDaAj0Infz7+2HfrPyax3onDLPq16/vV9mCv8Os9uUXlYZvvYxUMqsmzuSLQymdqq38lTCE2we8m1A/8GfAnpKNzrlbjuZDnXNDwnmemT0NfOA9XAc0L7O7GRqJEXN+DrMqudLE3y7szrk9mlAlwGUn/ZLgHSgSY+EG8DjvFnVm1tg5V3Ip3OHAPO/+e8C/zewhQifh2gMzYlGT7OfXcpS5BUWs2hr6v7x29ehd8y3WNAxNIhFWADvnXox2IWX8zcx6EPoZXgnc6NUw38zeIHQCsBD4tUZAxN7+X7GPLWLufmceb85cC5BQ6z4cSTSnakvFE1YAm1l74D6gC6FZcQA459r4XZBzbsRh9t0L3Ov3Z0r4/Grhfb4wdL23RjXTaV0v8dZ9OJRYXTFEKoZwuyCeB/6b0CiIQYSGoCV+h51EzI+pyM45duUW8utBbRk9tJMvdcWLWFwxRCqOcEO0qnPuc8Ccc6ucc/cAZ0evLIlbx7gg+/z1O/hswSYKi11cXe3CNxoHLBEItwWcZ2ZJwBIzu4nQ6IOM6JUl8ar0LP9RBszIl2aybnto5ltFGPd7sP0z4ZTAcmThtoBvBaoBtxBaFe1K4OpoFSXx61j6gPflF7Fu+z7qVk+lW9NanNAi09fa4oGpD0IiEG4LeKtzbjeh8cBaiL0SO5az/Fu9Kcd3DO3IZb0r5gxF5a9EItwAfs7MmgHfAlOBKWVXR5PK42gWZN+bX8iNL8+kkbfOb0UadnYwzYSTSIQ7DniAmaUCJwIDgQ/NLMM5l9hLV0nEjmYUxOJNu5m6ZHPp44rY91simlcMkYon3HHApwL9vFsmoenBU6NYl8Spo+kDXu+ddPvL+V1Zt30fPStg3+/B1AKWcITbBTEJmEloMsZHzrn8qFUkcS2SBdlzC4rYuCO3NIDP6d64Qrd+QX3AEplwA7gecArQH7jFzIqBb5xzf4haZRLXwmnh/XPiUv7vi6XUrZ5K1ZRkalWA1c6ORDPhJBLh9gFvN7PlhFYjawacDFT8f03yUxEs9zVt+RYA8gqLad8wI/EXWw9LZThG8Uu4fcDLgUXAl4TW671W3RCVUyTxsmV3Pud0b8y9w7tRVFw5WoRaDU0iEW4XRDvnXHFUK5GEcKRhVl8t3cwf3pnHVX1bsmVPPrWrpVaKrocSxzpTUCqXcGfCtTOzz81sHoCZdTezu6NYl8SpI10R4/3v17N88x7ueX8BO/YVVOgxv+WxY1wrQyqXcAP4aWAMUADgnJtD6ArFUskcfEWMnF15zF27o3T/uu376NE8k+evPZH+HerTv0PluiSUH6vFSeURbhdENefcjINOohRGoR6Jcwf3cd738ULGfbeOgR3rc9eZndi6J58GNdIY1LEBgzo2CKzOoPh5ySap+MIN4M1m1hbv352ZXUTogplSyRw8DnjVlr00qJHGd6u2ceajU0k2o2OjGkGWGKj9q6GJHFm4XRC/Bp4COpnZOuA2YFTUqpK4VdLC276vAIAN2/dxart6TLlzEDf0a0OSGW3rV96VSjUOWCIR7jjg5cAQM6tOKLT3EuoDXhXF2iQO7ckLXYbvgse/5q4zO7FpVx5NMquSWS2V353VmduHdCAlWWNhFb8SjsO2gM2sppmNMbPHzOx0QsF7NbAUuCQWBUp82eG1fAHu/3gRRcWOxpmllwmkampyhbi8/NFSH7BE4kgt4JeBbcA3wA3A7wmd6B3unJsd5dokDu3ND517vX1IB05sXZs3vl3DaZ0q38m2QzGtBiEROFIAt3HOdQMws2cInXhr4ZzLjXplEpd254UCuHpaMie3rcfJbesFXFF8UQtYInGk3xVLf990zhUBaxW+ldseL4Az0sIdQFO5aCqyROJIAXy8me30bruA7iX3zWxnLAqU+FIyseKElrUDriS+qQUs4ThsM8Y5lxyrQiQxnNO9CUM6NyQ9RT8a5dFVkSUSlfd0tRw1he+hVYoVN8U3CmARH2ktCIlE3J1JMbPXgY7ew0xgu3Ouh5m1AhYCP3j7pjnnNBtP4opOwkkk4i6AnXM/L7lvZn8HdpTZvcw51yP2VYmEK/xr5onEXQCXsNDSa5cApwVdi0i41AcskYjnPuB+wCbn3JIy21qb2Swzm2xm/Q71QjMbaWbZZpadk5MT/UpFPOoDlkgE0gI2swlAo3J2/d459653/zLg1TL7SmbhbTGzXsA7Znacc+4n45Gdc2OBsQBZWVn6pyAxoytiSCQCCWDn3JDD7TezKsAFQK8yr8kD8rz7M81sGdAByI5iqSIRUQtYIhGvXRBDgEXOubUlG8ysvpkle/fbAO2B5QHVJ1IurQUhkYjXk3CXcmD3A0B/4E9mVgAUA6Occ1tjXpnIYeiKGBKJuAxg59w15Wx7C3gr9tWIhE9XxJBIxGsXhEhCU/xKOBTAIj4yrccuEVAAi/hIw9AkEgpgkShQF7CEQwEs4iPNRJZIKIBFfKTV0CQSCmARH5WOA1YCSxgUwCI+2t8CVgLLkSmARXyktSAkEgpgET+pD1gioAAW8ZGh1XgkfApgER9pFIREQgEs4iP1AUsk4nI1NJFEVToVOcAEzi8s5rMFm1i7bS/rtu/jVwPb0ahWemD1yKEpgEV8FPRaPE9OXsb9Hy86YNsb2Wu47pTWjBrYlprpKQFVJuVRAIv4KOgrYvxr2qrS+2d3a8xpnRoweXEOj09axqszVnPzae25sk9LUquo9zEeKIBFfFQyCmLe+h0x/+ziYsfGHbncOKANtw/pQHpKMgAX9mrGDf3acN/HC/nTBwt44euV3DG0I+d0a0xSklavCJL+GxTxk5dn475bF9OPXZ6zmza/+4jCYkezzKql4VuiW7NavPKLk3jh2hOplprMLa/O4vzHv+LrZZtjWqccSAEs4iMLqEH5xaIfS+8f3zyz3OeYGQM7NuDDW/rx4MXHk7Mrj8ufns41z89g0cadsSpVylAAiyS4gqJiVmzeQ62qKay8/2y6Nys/gEskJxkX9WrGxDsGcteZnZi5ahtnPjqV0f/5ng079sWoagEFsIivgmgAj3h2Oq9MX03Vg7odjiQ9JZlRA9oy9c5B/OLU1rw7ez0DH5jEXz9ZxI59BVGqVspSAIv4KNaDH3bnFTJt+VYAzuvZ5KjeI7NaKr8/uwuf/3YAZ3VrzBOTljHggYk8++UK8gqL/CxXDqIAFvFRpK3QY+GcY9bqbQD832U9GXNm52N6v+Z1qvHwz3vwwc2n0q1pLf78wQIG/30y785eR3GxpvZFgwJYxEcpyUmc0q4uWS1rR/2z3vpuHSOenQFAk0z/Zrp1bVqLl68/iZeu602N9BRufW025/7zS75aqhETflMAi/gsyYziGMzE+HJJDgB929SlY6Oavr9//w71+fDmU3n458ezbU8BVzwznaufm8HCDRox4RcFsIjPzIxo/8a+asseFmzYSZ82dXh1ZB8y0qIzpyopyRjesxmf/3YAvz+rM7PXbOesf0zlt298z7rtGjFxrBTAIj5LsuguxrMvv4ghD01m8abd1MtIi9rnlJWekswN/dswZfQgRvZrw/tz1jPowUnc9/FCduzViImjFVgAm9nFZjbfzIrNLOugfWPMbKmZ/WBmQ8tsH+ZtW2pmd8W+apEjM4hqC3juuh0UFDlqVU3hwl7NovdB5ahVLYUxZ3Vm4h0DOad7Y8ZOWU7/Byby9JTl5BZoxESkgmwBzwMuAKaU3WhmXYBLgeOAYcDjZpZsZsnAP4EzgS7AZd5zReJKNPuAt+7J593ZoWnO4351MoM6NjhxPZ4AABKRSURBVIjK5xxJ08yqPHRJDz68uR/HN8/k3o8WMvjvk3l71lqNmIhAYAHsnFvonPuhnF3nAa855/KccyuApUBv77bUObfcOZcPvOY9VySumFnUVkMb9sgUXpm+GoBGNYNf47dLk5q8dF1v/nX9SWRWS+H217/nZ499yVTvBKEcXjz2ATcF1pR5vNbbdqjtP2FmI80s28yyc3L0gyCxlWREpQW8Ycc+ftyVB8DtQzpQPUon3o7Gqe3r8f5Np/LopT3Ysa+AEc/OYMSz05kfwKpwiSSqAWxmE8xsXjm3qLZcnXNjnXNZzrms+vXrR/OjRH4iKQot4I07crnzzTkAPHllL24d0t7fD/BBUpJxXo+mfP7bAdx9dmfmrtvBOf/3Jbe/Ppu12/YGXV5ciup/oc65IUfxsnVA8zKPm3nbOMx2kbiRlOR/C/jFb1YydUloIkTb+tV9fW+/pVVJ5hf92nBxVnOemLSM579awYdzNnD1yS359aB2ZFZLDbrEuBGPXRDvAZeaWZqZtQbaAzOAb4H2ZtbazFIJnah7L8A6RcplUTgJt3DDTjKrpfDY5T1p1yDD1/eOllpVU7jrzE5MvGMg5/ZowjNfrqD/30JrTEhIkMPQhpvZWqAv8KGZjQdwzs0H3gAWAJ8Av3bOFTnnCoGbgPHAQuAN77kiccXvLohHJixm0g859GpRm3O6Nym98GeiaJJZlQcvPp6Pb+1HV2+NCa0/HBJYL75z7m3g7UPsuxe4t5ztHwEfRbk0kWMSGgfsTwJ/MGc9j0xYAkCLutV8ec+gdGpUkxsHtOXrZVvYk6cxw6Brwon4Lsn8W5by62VbAHj00h6c2q6eT+8anKTSi5ZqrDAogEV858dEjK178hn890nsziukU6ManNej3BGXCafkoqWK3xAFsIjPzIzi4mN7jxkrtrDNW2Ohc2P/VzoLSkkLWLPlQhTAIj471sV4Vm/Zy5y1oQkM2XcPoWZ6il+lBa7kBKLyN0QBLOKzpGNYjnLbnnz6PzARgOqpyTFb7SxW1Ad8oHgcByyS0I5lIkbJGNnUKkmc1a2xn2XFhaQktYDLUgtYxHeRt4BXbdnDf783n6178gH44c/DEm68bzhK+4DVAgYUwCK+C4VMZAFz2dhprN+RC0D3ZrUqZPhC2T5gBTAogEV8F0kf8I+7ctmVW1gavo9e2iMulpmMlpL/VhS/IQpgEZ+Fuxylc47e935e+nj00I4VZrzvoSR5LWCdhAtRAIv4LDQO+NABU1hUTJFzzF27f63cWlVT6NkiMxblBaokgI91nHRFoQAW8dmRFuO58805jJu1rnRq8Ue39KNLk4oz2eJwTCfhDqAAFvHZwV0Q475by459BVzWuwU7cwsYNyu0jPWXS0Pr+7ZM8EV2IpGkiRgHUACL+Mxsf8Cs276P37zxPQDPTF3BsK6NALjvgm50alSDDTty4+rSQtGW5M08UB9wSOX55kViZFduIfsKivhswSY27gyNbrjipBbMXrO9dKJFVsvatG9Yg55BFhqAksV41AIOUQCL+GzGiq0A3PBSNjXSqpCcZPzpvK4Y8P6c9cxavZ3W9eL7skLRUjoVWQPRAE1FFvFdSav3v4Z1Ii0lmfYNMkhOstKLVt5z7nFUSa6c//S0GM+B1AIW8dktg9tz/8eLGDWgDdee0oqCIo25KqHFeA6kABbx2agBbRk1oC0A6SnJpKckB1xR/EjSVOQDVM7fg0QkEJqIcSAFsIjEjCZiHEgBLCIxY6V9wMHWES8UwCISM6WL8WgYGqAAFpEY0lTkAymARSRmdEWMAymARSRmNBHjQApgEYkZTcQ4UCABbGYXm9l8Mys2s6wy2083s5lmNtf787Qy+yaZ2Q9mNtu7NQiidhE5evvHASuAIbiZcPOAC4CnDtq+GfiZc269mXUFxgNlr9FyhXMuO0Y1iojP9o8DDraOeBFIADvnFgI/ufKrc25WmYfzgapmluacy4theSISJVY6DE0gvvuALwS+Oyh8n/e6H/5gh7lut5mNNLNsM8vOycmJfqUiEhb1AR8oagFsZhPMbF45t/PCeO1xwF+BG8tsvsI51w3o591GHOr1zrmxzrks51xW/fr1j/VQRMQnJX3Af/1kUcCVxIeodUE454YczevMrBnwNnCVc25Zmfdb5/25y8z+DfQGXvKjVhGJjZIALihSCxjirAvCzDKBD4G7nHNfldlexczqefdTgHMIncgTkQRy6I7DyimoYWjDzWwt0Bf40MzGe7tuAtoBfzxouFkaMN7M5gCzgXXA00HULiJHr0qSErisoEZBvE2om+Hg7X8B/nKIl/WKalEiEnVVkpPo0TwT5XBIXHVBiEjFVyNdF+IpoQAWkZgyM03E8CiARSSmkkzjgEsogEUkppLUAi6lABaRmEqy+FsPuLjYcfc7c3n2yxXkFRbF7HPVGy4iMRVPfcB5hUV8uWQz17+4f42vF75ewR1ndORn3ZuQFOXhGmoBi0hMxVMf8CvTVh8Qvg9efDw10lK49bXZnPvPL/lq6eaofr5awCISU4YF2gWxJ6+QfQVFPDFpGd+t3gbA5Se14I/ndCE9JZkLejbl3e/X8eD4xVzxzHQGdKjPXWd2onPjmr7XogAWkZhKSgr2svQXPvE1izbuKn38s+Ob8L/Du5U+TkoyhvdsxpldG/PyN6t4bOJSzvrHVC7o2YzfnNGBpplVfatFASwiMRXqAw4mgZ+YtKw0fBvXSufG/m04tX29cp+bnpLMDf3bcElWcx6ftJTnv17J+3PW89rIPpzQorYv9SiARSSmkswCaQG///360mUwZ949hLoZaWG9rla1FMac1ZmrTm7FS1+vpHvTWr7VpAAWkZgKYhja9OVbuPnV0AV3/nJ+17DDt6ymmVUZc1ZnX+vSKAgRialYT8RYvGkXPx87DYBrT2nFlX1axu7Dj0ABLCIxZTFsAa/bvo8zHp4CwIUnNOOP53SJyeeGSwEsIjEVqz7gfflFnHL/FwAM7tSAv17Y7ScXAg6a+oBFJKaM6LeAi4sdfe77HICGNdN49poTo/p5R0stYBGJqaQoD0PLLyzmgie+Zse+AgAmjx4Utc86VmoBi0hMRXMixvrt+/jFi9ks2LATgBm/G0x6SnJ0PswHCmARialoLsYz9OEp7MorBGDSHQNpUDM9Oh/kE3VBiEhMRWMxnrzCIq56bkZp+H58az9a1avu62dEg1rAIhJTfvcBFxQVc+YjU1m+eQ8An9zWj06N/F84JxoUwCISU35OxPhxVy697/289PHnvx1A2/oZ/rx5DCiARSSm/JqI8e3KrVz85DeljyePHkjLuvHf7VCWAlhEYsqPiRgzVmzlkqdC4VstNZlZfzydtCrxO9rhUBTAIhJTxzIRI7egiPMe+4ofNoWWlDyzayPuu6BbQoYvKIBFJMaSkiJvAW/encdDny1m3HdryS0oBuC6U1rzh3M6x9304kgogEUkpiLpA96wYx9bdudz9zvzmL1me+n27//7DGpVTYlWiTETSACb2cXAPUBnoLdzLtvb3gpYCPzgPXWac26Ut68X8AJQFfgIuNXFy5X9RCRsR+oD3rG3gLXb99KlcU1O/etEisoMmbjvgm70a1+vQoQvBNcCngdcADxVzr5lzrke5Wx/ArgBmE4ogIcBH0etQhGJiiSD/KJiioodyUnGza/OYsXm3fzXsE6c0KI2J/zlM4qKHdVSk0vDt1HNdB68+PhDXj4oUQUSwM65hUDYfTdm1hio6Zyb5j1+CTgfBbBIwlm4IXQC7ex/TOWMLg15//v1AIx4dsYBz9ubXwTAtDGDaVQrvqcUH6147ANubWazgJ3A3c65qUBTYG2Z56z1tpXLzEYCIwFatGgRxVJFJFIrvBlrm3fn848vlgLw0S39+Gb5Fv78wQJSk5OY/rvBJJmxaVduhQ1fiGIAm9kEoFE5u37vnHv3EC/bALRwzm3x+nzfMbPjIv1s59xYYCxAVlaW+olF4khJt8Knt/fnwznrKSx2dGlSky5NanLpic0pKComs1oqELogZkUWtQB2zg05itfkAXne/ZlmtgzoAKwDmpV5ajNvm4gkmOeuOZHJi3OoUz2VEX1bHbCvelo8/lIePXF1tGZWH9jqnCsyszZAe2C5c26rme00sz6ETsJdBfxfkLWKyNFp1yCDdg0SZ72GaApkOUozG25ma4G+wIdmNt7b1R+YY2azgTeBUc65rd6+XwHPAEuBZegEnIgkOKvoQ2mzsrJcdnZ20GWISOVW7pAvLcguIhIQBbCISEAUwCIiAVEAi4gERAEsIhIQBbCISEAUwCIiAVEAi4gERAEsIhIQBbCISEAq/FRkM8sBVkX4snrA5iiUE6SKdkwV7XhAx5QojuaYNjvnhh28scIH8NEws2znXFbQdfipoh1TRTse0DElCj+PSV0QIiIBUQCLiAREAVy+sUEXEAUV7Zgq2vGAjilR+HZM6gMWEQmIWsAiIgFRAIuIBEQBXIaZDTOzH8xsqZndFXQ94TKz5mY20cwWmNl8M7vV217HzD4zsyXen7W97WZm//COc46ZnRDsEZTPzJLNbJaZfeA9bm1m0726XzezVG97mvd4qbe/VZB1H4qZZZrZm2a2yMwWmlnfCvAd3e79zM0zs1fNLD3Rvicze87MfjSzeWW2Rfy9mNnV3vOXmNnVYX24c063UD94MqGLfbYBUoHvgS5B1xVm7Y2BE7z7NYDFQBfgb8Bd3va7gL96988idFFTA/oA04M+hkMc12+AfwMfeI/fAC717j8J/NK7/yvgSe/+pcDrQdd+iON5EfiFdz8VyEzk7whoCqwAqpb5fq5JtO+J0MWATwDmldkW0fcC1AGWe3/W9u7XPuJnB33w8XIjdIXm8WUejwHGBF3XUR7Lu8DpwA9AY29bY+AH7/5TwGVlnl/6vHi5Ac2Az4HTgA+8H/jNQJWDvy9gPNDXu1/Fe54FfQwHHU8tL6zsoO2J/B01BdZ4oVPF+56GJuL3BLQ6KIAj+l6Ay4Cnymw/4HmHuqkLYr+SH6YSa71tCcX7ta4nMB1o6Jzb4O3aCDT07ifCsT4C3AkUe4/rAtudc4Xe47I1lx6Pt3+H9/x40hrIAZ73ulWeMbPqJPB35JxbBzwIrAY2EPp7n0lif08lIv1ejur7UgBXIGaWAbwF3Oac21l2nwv9t5wQYw7N7BzgR+fczKBr8VEVQr/mPuGc6wnsIfSrbalE+o4AvH7R8wj959IEqA78ZL2DRBfN70UBvN86oHmZx828bQnBzFIIhe8rzrlx3uZNZtbY298Y+NHbHu/HegpwrpmtBF4j1A3xKJBpZlW855StufR4vP21gC2xLDgMa4G1zrnp3uM3CQVyon5HAEOAFc65HOdcATCO0HeXyN9TiUi/l6P6vhTA+30LtPfO4KYSOknwXsA1hcXMDHgWWOice6jMrveAkrOxVxPqGy7ZfpV3RrcPsKPMr1uBc86Ncc41c861IvQ9fOGcuwKYCFzkPe3g4yk5zou858dVS9I5txFYY2YdvU2DgQUk6HfkWQ30MbNq3s9gyTEl7PdURqTfy3jgDDOr7f1mcIa37fCC7vyOpxuhM5yLCY2G+H3Q9URQ96mEfkWaA8z2bmcR6l/7HFgCTADqeM834J/ecc4FsoI+hsMc20D2j4JoA8wAlgL/AdK87ene46Xe/jZB132IY+kBZHvf0zuEzpYn9HcE/A+wCJgHvAykJdr3BLxKqA+7gNBvKtcfzfcCXOcd21Lg2nA+W1ORRUQCoi4IEZGAKIBFRAKiABYRCYgCWEQkIApgEZGAVDnyU0QSn5kVERo2VOJ859zKgMoRAXRFDKkkzGy3cy7jEPuM0L+F4vL2i0SLuiCkUjKzVhZa+/klQpMImpvZE2aW7a1v+z9lnrvSzO4zs9ne/hPMbLyZLTOzUWWeN9rMvvXWif2f8j5XpCwFsFQWVb0AnW1mb3vb2gOPO+eOc86tIjT7MQvoDgwws+5lXr/aOdcDmAq8QGgqbR9CM8EwszO89+tNaMZbLzPrH4sDk8SlPmCpLPZ5AQqULtu5yjk3rcxzLjGzkYT+XTQmtKj9HG9fybogc4EM59wuYJeZ5ZlZJqG5/2cAs7znZRAK5CnRORypCBTAUpntKbljZq2BO4ATnXPbzOwFQmsXlMjz/iwuc7/kcRVCawTc55x7KqoVS4WiLgiRkJqEAnmHmTUEzozw9eOB67w1mTGzpmbWwOcapYJRC1gEcM59b2azCK3stQb4KsLXf2pmnYFvQoMq2A1cyf51ZEV+QsPQREQCoi4IEZGAKIBFRAKiABYRCYgCWEQkIApgEZGAKIBFRAKiABYRCcj/Aw1WyB+/mHDBAAAAAElFTkSuQmCC\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "rewards_data = {'Frame': np.arange(len(rewardLS)), 'Rewards':np.array(rewardLS).cumsum()}\n",
        "rewards_data = pd.DataFrame(data=rewards_data)\n",
        "sns.relplot(data=rewards_data, x=\"Frame\", y=\"Rewards\", kind=\"line\")\n",
        "plt.savefig('rewards_graph.png')"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "max(rewards_data.Rewards)#24.752187728881836 with 250k timesteps"
      ],
      "metadata": {
        "id": "sUhcqRmEsUxk",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "10e7dfc6-dac7-42bb-846d-832b1a3ecf10"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "42.003257751464844"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "FOuwDHL50GpZ"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "colab": {
      "name": "CarRacing A2C.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}