breakout.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DQN with CartPole-v1 Environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 5006
    },
    "colab_type": "code",
    "id": "RkTeQM5oQpMF",
    "outputId": "ba6df977-abc2-4f73-ed4e-72be4eab3ed4"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading package lists... Done\n",
      "Building dependency tree       \n",
      "Reading state information... Done\n",
      "zlib1g-dev is already the newest version (1:1.2.11.dfsg-0ubuntu2).\n",
      "zlib1g-dev set to manually installed.\n",
      "The following additional packages will be installed:\n",
      "  autotools-dev cmake-data file ibverbs-providers libarchive13 libfabric1\n",
      "  libhwloc-dev libhwloc-plugins libhwloc5 libibverbs-dev libibverbs1\n",
      "  libjsoncpp1 libltdl-dev libltdl7 liblzo2-2 libmagic-mgc libmagic1\n",
      "  libnl-3-200 libnl-route-3-200 libnuma-dev libnuma1 libopenmpi2 libpciaccess0\n",
      "  libpsm-infinipath1 librdmacm1 librhash0 libtool libuv1 ocl-icd-libopencl1\n",
      "  openmpi-bin openmpi-common\n",
      "Suggested packages:\n",
      "  cmake-doc ninja-build lrzip libhwloc-contrib-plugins libtool-doc openmpi-doc\n",
      "  pciutils autoconf automaken gcj-jdk\n",
      "The following NEW packages will be installed:\n",
      "  autotools-dev cmake cmake-data file ibverbs-providers libarchive13\n",
      "  libfabric1 libhwloc-dev libhwloc-plugins libhwloc5 libibverbs-dev\n",
      "  libibverbs1 libjsoncpp1 libltdl-dev libltdl7 liblzo2-2 libmagic-mgc\n",
      "  libmagic1 libnl-3-200 libnl-route-3-200 libnuma-dev libnuma1 libopenmpi-dev\n",
      "  libopenmpi2 libpciaccess0 libpsm-infinipath1 librdmacm1 librhash0 libtool\n",
      "  libuv1 ocl-icd-libopencl1 openmpi-bin openmpi-common\n",
      "0 upgraded, 33 newly installed, 0 to remove and 8 not upgraded.\n",
      "Need to get 10.4 MB of archives.\n",
      "After this operation, 52.1 MB of additional disk space will be used.\n",
      "Get:1 http://archive.ubuntu.com/ubuntu bionic/main amd64 liblzo2-2 amd64 2.08-1.2 [48.7 kB]\n",
      "Get:2 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 libmagic-mgc amd64 1:5.32-2ubuntu0.1 [184 kB]\n",
      "Get:3 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 libmagic1 amd64 1:5.32-2ubuntu0.1 [68.4 kB]\n",
      "Get:4 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 file amd64 1:5.32-2ubuntu0.1 [22.1 kB]\n",
      "Get:5 http://archive.ubuntu.com/ubuntu bionic/main amd64 libnuma1 amd64 2.0.11-2.1 [21.6 kB]\n",
      "Get:6 http://archive.ubuntu.com/ubuntu bionic/main amd64 autotools-dev all 20180224.1 [39.6 kB]\n",
      "Get:7 http://archive.ubuntu.com/ubuntu bionic/main amd64 cmake-data all 3.10.2-1ubuntu2 [1,331 kB]\n",
      "Get:8 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 libarchive13 amd64 3.2.2-3.1ubuntu0.1 [289 kB]\n",
      "Get:9 http://archive.ubuntu.com/ubuntu bionic/main amd64 libjsoncpp1 amd64 1.7.4-3 [73.6 kB]\n",
      "Get:10 http://archive.ubuntu.com/ubuntu bionic/main amd64 librhash0 amd64 1.3.6-2 [78.1 kB]\n",
      "Get:11 http://archive.ubuntu.com/ubuntu bionic/main amd64 libuv1 amd64 1.18.0-3 [64.4 kB]\n",
      "Get:12 http://archive.ubuntu.com/ubuntu bionic/main amd64 cmake amd64 3.10.2-1ubuntu2 [3,138 kB]\n",
      "Get:13 http://archive.ubuntu.com/ubuntu bionic/main amd64 libnl-3-200 amd64 3.2.29-0ubuntu3 [52.8 kB]\n",
      "Get:14 http://archive.ubuntu.com/ubuntu bionic/main amd64 libnl-route-3-200 amd64 3.2.29-0ubuntu3 [146 kB]\n",
      "Get:15 http://archive.ubuntu.com/ubuntu bionic/main amd64 libibverbs1 amd64 17.1-1 [44.4 kB]\n",
      "Get:16 http://archive.ubuntu.com/ubuntu bionic/main amd64 ibverbs-providers amd64 17.1-1 [160 kB]\n",
      "Get:17 http://archive.ubuntu.com/ubuntu bionic/universe amd64 libpsm-infinipath1 amd64 3.3+20.604758e7-5 [174 kB]\n",
      "Get:18 http://archive.ubuntu.com/ubuntu bionic/main amd64 librdmacm1 amd64 17.1-1 [56.1 kB]\n",
      "Get:19 http://archive.ubuntu.com/ubuntu bionic/universe amd64 libfabric1 amd64 1.5.3-1 [302 kB]\n",
      "Get:20 http://archive.ubuntu.com/ubuntu bionic/main amd64 libltdl7 amd64 2.4.6-2 [38.8 kB]\n",
      "Get:21 http://archive.ubuntu.com/ubuntu bionic/main amd64 libltdl-dev amd64 2.4.6-2 [162 kB]\n",
      "Get:22 http://archive.ubuntu.com/ubuntu bionic/main amd64 libpciaccess0 amd64 0.14-1 [17.9 kB]\n",
      "Get:23 http://archive.ubuntu.com/ubuntu bionic/main amd64 libtool all 2.4.6-2 [194 kB]\n",
      "Get:24 http://archive.ubuntu.com/ubuntu bionic/universe amd64 libhwloc5 amd64 1.11.9-1 [105 kB]\n",
      "Get:25 http://archive.ubuntu.com/ubuntu bionic/main amd64 ocl-icd-libopencl1 amd64 2.2.11-1ubuntu1 [30.3 kB]\n",
      "Get:26 http://archive.ubuntu.com/ubuntu bionic/universe amd64 libhwloc-plugins amd64 1.11.9-1 [12.5 kB]\n",
      "Get:27 http://archive.ubuntu.com/ubuntu bionic/universe amd64 libopenmpi2 amd64 2.1.1-8 [2,056 kB]\n",
      "Get:28 http://archive.ubuntu.com/ubuntu bionic/universe amd64 openmpi-common all 2.1.1-8 [140 kB]\n",
      "Get:29 http://archive.ubuntu.com/ubuntu bionic/universe amd64 openmpi-bin amd64 2.1.1-8 [88.2 kB]\n",
      "Get:30 http://archive.ubuntu.com/ubuntu bionic/main amd64 libnuma-dev amd64 2.0.11-2.1 [32.2 kB]\n",
      "Get:31 http://archive.ubuntu.com/ubuntu bionic/universe amd64 libhwloc-dev amd64 1.11.9-1 [167 kB]\n",
      "Get:32 http://archive.ubuntu.com/ubuntu bionic/main amd64 libibverbs-dev amd64 17.1-1 [103 kB]\n",
      "Get:33 http://archive.ubuntu.com/ubuntu bionic/universe amd64 libopenmpi-dev amd64 2.1.1-8 [925 kB]\n",
      "Fetched 10.4 MB in 4s (2,504 kB/s)\n",
      "Extracting templates from packages: 100%\n",
      "Selecting previously unselected package liblzo2-2:amd64.\n",
      "(Reading database ... 26397 files and directories currently installed.)\n",
      "Preparing to unpack .../00-liblzo2-2_2.08-1.2_amd64.deb ...\n",
      "Unpacking liblzo2-2:amd64 (2.08-1.2) ...\n",
      "Selecting previously unselected package libmagic-mgc.\n",
      "Preparing to unpack .../01-libmagic-mgc_1%3a5.32-2ubuntu0.1_amd64.deb ...\n",
      "Unpacking libmagic-mgc (1:5.32-2ubuntu0.1) ...\n",
      "Selecting previously unselected package libmagic1:amd64.\n",
      "Preparing to unpack .../02-libmagic1_1%3a5.32-2ubuntu0.1_amd64.deb ...\n",
      "Unpacking libmagic1:amd64 (1:5.32-2ubuntu0.1) ...\n",
      "Selecting previously unselected package file.\n",
      "Preparing to unpack .../03-file_1%3a5.32-2ubuntu0.1_amd64.deb ...\n",
      "Unpacking file (1:5.32-2ubuntu0.1) ...\n",
      "Selecting previously unselected package libnuma1:amd64.\n",
      "Preparing to unpack .../04-libnuma1_2.0.11-2.1_amd64.deb ...\n",
      "Unpacking libnuma1:amd64 (2.0.11-2.1) ...\n",
      "Selecting previously unselected package autotools-dev.\n",
      "Preparing to unpack .../05-autotools-dev_20180224.1_all.deb ...\n",
      "Unpacking autotools-dev (20180224.1) ...\n",
      "Selecting previously unselected package cmake-data.\n",
      "Preparing to unpack .../06-cmake-data_3.10.2-1ubuntu2_all.deb ...\n",
      "Unpacking cmake-data (3.10.2-1ubuntu2) ...\n",
      "Selecting previously unselected package libarchive13:amd64.\n",
      "Preparing to unpack .../07-libarchive13_3.2.2-3.1ubuntu0.1_amd64.deb ...\n",
      "Unpacking libarchive13:amd64 (3.2.2-3.1ubuntu0.1) ...\n",
      "Selecting previously unselected package libjsoncpp1:amd64.\n",
      "Preparing to unpack .../08-libjsoncpp1_1.7.4-3_amd64.deb ...\n",
      "Unpacking libjsoncpp1:amd64 (1.7.4-3) ...\n",
      "Selecting previously unselected package librhash0:amd64.\n",
      "Preparing to unpack .../09-librhash0_1.3.6-2_amd64.deb ...\n",
      "Unpacking librhash0:amd64 (1.3.6-2) ...\n",
      "Selecting previously unselected package libuv1:amd64.\n",
      "Preparing to unpack .../10-libuv1_1.18.0-3_amd64.deb ...\n",
      "Unpacking libuv1:amd64 (1.18.0-3) ...\n",
      "Selecting previously unselected package cmake.\n",
      "Preparing to unpack .../11-cmake_3.10.2-1ubuntu2_amd64.deb ...\n",
      "Unpacking cmake (3.10.2-1ubuntu2) ...\n",
      "Selecting previously unselected package libnl-3-200:amd64.\n",
      "Preparing to unpack .../12-libnl-3-200_3.2.29-0ubuntu3_amd64.deb ...\n",
      "Unpacking libnl-3-200:amd64 (3.2.29-0ubuntu3) ...\n",
      "Selecting previously unselected package libnl-route-3-200:amd64.\n",
      "Preparing to unpack .../13-libnl-route-3-200_3.2.29-0ubuntu3_amd64.deb ...\n",
      "Unpacking libnl-route-3-200:amd64 (3.2.29-0ubuntu3) ...\n",
      "Selecting previously unselected package libibverbs1:amd64.\n",
      "Preparing to unpack .../14-libibverbs1_17.1-1_amd64.deb ...\n",
      "Unpacking libibverbs1:amd64 (17.1-1) ...\n",
      "Selecting previously unselected package ibverbs-providers:amd64.\n",
      "Preparing to unpack .../15-ibverbs-providers_17.1-1_amd64.deb ...\n",
      "Unpacking ibverbs-providers:amd64 (17.1-1) ...\n",
      "Selecting previously unselected package libpsm-infinipath1.\n",
      "Preparing to unpack .../16-libpsm-infinipath1_3.3+20.604758e7-5_amd64.deb ...\n",
      "Unpacking libpsm-infinipath1 (3.3+20.604758e7-5) ...\n",
      "Selecting previously unselected package librdmacm1:amd64.\n",
      "Preparing to unpack .../17-librdmacm1_17.1-1_amd64.deb ...\n",
      "Unpacking librdmacm1:amd64 (17.1-1) ...\n",
      "Selecting previously unselected package libfabric1.\n",
      "Preparing to unpack .../18-libfabric1_1.5.3-1_amd64.deb ...\n",
      "Unpacking libfabric1 (1.5.3-1) ...\n",
      "Selecting previously unselected package libltdl7:amd64.\n",
      "Preparing to unpack .../19-libltdl7_2.4.6-2_amd64.deb ...\n",
      "Unpacking libltdl7:amd64 (2.4.6-2) ...\n",
      "Selecting previously unselected package libltdl-dev:amd64.\n",
      "Preparing to unpack .../20-libltdl-dev_2.4.6-2_amd64.deb ...\n",
      "Unpacking libltdl-dev:amd64 (2.4.6-2) ...\n",
      "Selecting previously unselected package libpciaccess0:amd64.\n",
      "Preparing to unpack .../21-libpciaccess0_0.14-1_amd64.deb ...\n",
      "Unpacking libpciaccess0:amd64 (0.14-1) ...\n",
      "Selecting previously unselected package libtool.\n",
      "Preparing to unpack .../22-libtool_2.4.6-2_all.deb ...\n",
      "Unpacking libtool (2.4.6-2) ...\n",
      "Selecting previously unselected package libhwloc5:amd64.\n",
      "Preparing to unpack .../23-libhwloc5_1.11.9-1_amd64.deb ...\n",
      "Unpacking libhwloc5:amd64 (1.11.9-1) ...\n",
      "Selecting previously unselected package ocl-icd-libopencl1:amd64.\n",
      "Preparing to unpack .../24-ocl-icd-libopencl1_2.2.11-1ubuntu1_amd64.deb ...\n",
      "Unpacking ocl-icd-libopencl1:amd64 (2.2.11-1ubuntu1) ...\n",
      "Selecting previously unselected package libhwloc-plugins.\n",
      "Preparing to unpack .../25-libhwloc-plugins_1.11.9-1_amd64.deb ...\n",
      "Unpacking libhwloc-plugins (1.11.9-1) ...\n",
      "Selecting previously unselected package libopenmpi2:amd64.\n",
      "Preparing to unpack .../26-libopenmpi2_2.1.1-8_amd64.deb ...\n",
      "Unpacking libopenmpi2:amd64 (2.1.1-8) ...\n",
      "Selecting previously unselected package openmpi-common.\n",
      "Preparing to unpack .../27-openmpi-common_2.1.1-8_all.deb ...\n",
      "Unpacking openmpi-common (2.1.1-8) ...\n",
      "Selecting previously unselected package openmpi-bin.\n",
      "Preparing to unpack .../28-openmpi-bin_2.1.1-8_amd64.deb ...\n",
      "Unpacking openmpi-bin (2.1.1-8) ...\n",
      "Selecting previously unselected package libnuma-dev:amd64.\n",
      "Preparing to unpack .../29-libnuma-dev_2.0.11-2.1_amd64.deb ...\n",
      "Unpacking libnuma-dev:amd64 (2.0.11-2.1) ...\n",
      "Selecting previously unselected package libhwloc-dev:amd64.\n",
      "Preparing to unpack .../30-libhwloc-dev_1.11.9-1_amd64.deb ...\n",
      "Unpacking libhwloc-dev:amd64 (1.11.9-1) ...\n",
      "Selecting previously unselected package libibverbs-dev:amd64.\n",
      "Preparing to unpack .../31-libibverbs-dev_17.1-1_amd64.deb ...\n",
      "Unpacking libibverbs-dev:amd64 (17.1-1) ...\n",
      "Selecting previously unselected package libopenmpi-dev.\n",
      "Preparing to unpack .../32-libopenmpi-dev_2.1.1-8_amd64.deb ...\n",
      "Unpacking libopenmpi-dev (2.1.1-8) ...\n",
      "Setting up libuv1:amd64 (1.18.0-3) ...\n",
      "Setting up libnuma1:amd64 (2.0.11-2.1) ...\n",
      "Setting up cmake-data (3.10.2-1ubuntu2) ...\n",
      "Setting up libmagic-mgc (1:5.32-2ubuntu0.1) ...\n",
      "Setting up libmagic1:amd64 (1:5.32-2ubuntu0.1) ...\n",
      "Setting up librhash0:amd64 (1.3.6-2) ...\n",
      "Setting up libpsm-infinipath1 (3.3+20.604758e7-5) ...\n",
      "update-alternatives: using /usr/lib/libpsm1/libpsm_infinipath.so.1.16 to provide /usr/lib/x86_64-linux-gnu/libpsm_infinipath.so.1 (libpsm_infinipath.so.1) in auto mode\n",
      "Setting up openmpi-common (2.1.1-8) ...\n",
      "Processing triggers for libc-bin (2.27-3ubuntu1) ...\n",
      "Setting up autotools-dev (20180224.1) ...\n",
      "Setting up libltdl7:amd64 (2.4.6-2) ...\n",
      "Setting up libpciaccess0:amd64 (0.14-1) ...\n",
      "Processing triggers for man-db (2.8.3-2ubuntu0.1) ...\n",
      "Setting up libnuma-dev:amd64 (2.0.11-2.1) ...\n",
      "Setting up ocl-icd-libopencl1:amd64 (2.2.11-1ubuntu1) ...\n",
      "Setting up libnl-3-200:amd64 (3.2.29-0ubuntu3) ...\n",
      "Setting up liblzo2-2:amd64 (2.08-1.2) ...\n",
      "Setting up libjsoncpp1:amd64 (1.7.4-3) ...\n",
      "Setting up libltdl-dev:amd64 (2.4.6-2) ...\n",
      "Setting up libarchive13:amd64 (3.2.2-3.1ubuntu0.1) ...\n",
      "Setting up libnl-route-3-200:amd64 (3.2.29-0ubuntu3) ...\n",
      "Setting up file (1:5.32-2ubuntu0.1) ...\n",
      "Setting up libhwloc5:amd64 (1.11.9-1) ...\n",
      "Setting up libhwloc-plugins (1.11.9-1) ...\n",
      "Setting up cmake (3.10.2-1ubuntu2) ...\n",
      "Setting up libtool (2.4.6-2) ...\n",
      "Setting up libibverbs1:amd64 (17.1-1) ...\n",
      "Setting up libhwloc-dev:amd64 (1.11.9-1) ...\n",
      "Setting up librdmacm1:amd64 (17.1-1) ...\n",
      "Setting up ibverbs-providers:amd64 (17.1-1) ...\n",
      "Setting up libibverbs-dev:amd64 (17.1-1) ...\n",
      "Setting up libfabric1 (1.5.3-1) ...\n",
      "Setting up libopenmpi2:amd64 (2.1.1-8) ...\n",
      "Setting up libopenmpi-dev (2.1.1-8) ...\n",
      "update-alternatives: using /usr/lib/x86_64-linux-gnu/openmpi/include to provide /usr/include/mpi (mpi) in auto mode\n",
      "Setting up openmpi-bin (2.1.1-8) ...\n",
      "update-alternatives: using /usr/bin/mpirun.openmpi to provide /usr/bin/mpirun (mpirun) in auto mode\n",
      "Processing triggers for libc-bin (2.27-3ubuntu1) ...\n",
      "Collecting stable-baselines\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ee/e9/9d5693d05073a6630c7227a70a654a5ee8edd804bc23670718350de39062/stable_baselines-2.3.0-py3-none-any.whl (226kB)\n",
      "\u001b[K    100% |████████████████████████████████| 235kB 12.2MB/s \n",
      "\u001b[?25hRequirement already satisfied: opencv-python in /usr/local/lib/python3.6/dist-packages (from stable-baselines) (3.4.4.19)\n",
      "Collecting progressbar2 (from stable-baselines)\n",
      "  Downloading https://files.pythonhosted.org/packages/4f/6f/acb2dd76f2c77527584bd3a4c2509782bb35c481c610521fc3656de5a9e0/progressbar2-3.38.0-py2.py3-none-any.whl\n",
      "Collecting zmq (from stable-baselines)\n",
      "  Downloading https://files.pythonhosted.org/packages/6e/78/833b2808793c1619835edb1a4e17a023d5d625f4f97ff25ffff986d1f472/zmq-0.0.0.tar.gz\n",
      "Requirement already satisfied: dill in /usr/local/lib/python3.6/dist-packages (from stable-baselines) (0.2.8.2)\n",
      "Collecting click (from stable-baselines)\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/fa/37/45185cb5abbc30d7257104c434fe0b07e5a195a6847506c074527aa599ec/Click-7.0-py2.py3-none-any.whl (81kB)\n",
      "\u001b[K    100% |████████████████████████████████| 81kB 22.1MB/s \n",
      "\u001b[?25hRequirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from stable-baselines) (0.13.0)\n",
      "Requirement already satisfied: scipy in /usr/local/lib/python3.6/dist-packages (from stable-baselines) (1.1.0)\n",
      "Requirement already satisfied: matplotlib in /usr/local/lib/python3.6/dist-packages (from stable-baselines) (2.1.2)\n",
      "Requirement already satisfied: gym[atari,classic_control]>=0.10.9 in /usr/local/lib/python3.6/dist-packages (from stable-baselines) (0.10.9)\n",
      "Requirement already satisfied: tensorflow>=1.5.0 in /usr/local/lib/python3.6/dist-packages (from stable-baselines) (1.12.0)\n",
      "Requirement already satisfied: pandas in /usr/local/lib/python3.6/dist-packages (from stable-baselines) (0.22.0)\n",
      "Requirement already satisfied: seaborn in /usr/local/lib/python3.6/dist-packages (from stable-baselines) (0.7.1)\n",
      "Collecting glob2 (from stable-baselines)\n",
      "  Downloading https://files.pythonhosted.org/packages/f0/e8/970c7a031b2d7f9a21fefaa8c9d5c38001f8f25055f4ffcb32b3dbecd1ea/glob2-0.6.tar.gz\n",
      "Collecting cloudpickle>=0.5.5 (from stable-baselines)\n",
      "  Downloading https://files.pythonhosted.org/packages/fc/87/7b7ef3038b4783911e3fdecb5c566e3a817ce3e890e164fc174c088edb1e/cloudpickle-0.6.1-py2.py3-none-any.whl\n",
      "Collecting mpi4py (from stable-baselines)\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/31/27/1288918ac230cc9abc0da17d84d66f3db477757d90b3d6b070d709391a15/mpi4py-3.0.0.tar.gz (1.4MB)\n",
      "\u001b[K    100% |████████████████████████████████| 1.4MB 13.2MB/s \n",
      "\u001b[?25hRequirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (from stable-baselines) (4.28.1)\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from stable-baselines) (1.14.6)\n",
      "Collecting python-utils>=2.3.0 (from progressbar2->stable-baselines)\n",
      "  Downloading https://files.pythonhosted.org/packages/eb/a0/19119d8b7c05be49baf6c593f11c432d571b70d805f2fe94c0585e55e4c8/python_utils-2.3.0-py2.py3-none-any.whl\n",
      "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from progressbar2->stable-baselines) (1.11.0)\n",
      "Requirement already satisfied: pyzmq in /usr/local/lib/python3.6/dist-packages (from zmq->stable-baselines) (17.0.0)\n",
      "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib->stable-baselines) (0.10.0)\n",
      "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->stable-baselines) (2.5.3)\n",
      "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->stable-baselines) (2.3.0)\n",
      "Requirement already satisfied: pytz in /usr/local/lib/python3.6/dist-packages (from matplotlib->stable-baselines) (2018.7)\n",
      "Requirement already satisfied: pyglet>=1.2.0 in /usr/local/lib/python3.6/dist-packages (from gym[atari,classic_control]>=0.10.9->stable-baselines) (1.3.2)\n",
      "Requirement already satisfied: requests>=2.0 in /usr/local/lib/python3.6/dist-packages (from gym[atari,classic_control]>=0.10.9->stable-baselines) (2.18.4)\n",
      "Collecting PyOpenGL; extra == \"atari\" (from gym[atari,classic_control]>=0.10.9->stable-baselines)\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/9c/1d/4544708aaa89f26c97cc09450bb333a23724a320923e74d73e028b3560f9/PyOpenGL-3.1.0.tar.gz (1.2MB)\n",
      "\u001b[K    100% |████████████████████████████████| 1.2MB 16.3MB/s \n",
      "\u001b[?25hCollecting atari-py>=0.1.4; extra == \"atari\" (from gym[atari,classic_control]>=0.10.9->stable-baselines)\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/12/23/96ce488af867646277d91c3c9a1f328a5452240fe51df0a5fdf9211d76e0/atari_py-0.1.7-cp36-cp36m-manylinux1_x86_64.whl (2.6MB)\n",
      "\u001b[K    100% |████████████████████████████████| 2.6MB 10.4MB/s \n",
      "\u001b[?25hRequirement already satisfied: Pillow; extra == \"atari\" in /usr/local/lib/python3.6/dist-packages (from gym[atari,classic_control]>=0.10.9->stable-baselines) (4.0.0)\n",
      "Requirement already satisfied: absl-py>=0.1.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow>=1.5.0->stable-baselines) (0.6.1)\n",
      "Requirement already satisfied: gast>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow>=1.5.0->stable-baselines) (0.2.0)\n",
      "Requirement already satisfied: protobuf>=3.6.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow>=1.5.0->stable-baselines) (3.6.1)\n",
      "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow>=1.5.0->stable-baselines) (1.1.0)\n",
      "Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow>=1.5.0->stable-baselines) (1.15.0)\n",
      "Requirement already satisfied: astor>=0.6.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow>=1.5.0->stable-baselines) (0.7.1)\n",
      "Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.6/dist-packages (from tensorflow>=1.5.0->stable-baselines) (0.32.3)\n",
      "Requirement already satisfied: tensorboard<1.13.0,>=1.12.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow>=1.5.0->stable-baselines) (1.12.0)\n",
      "Requirement already satisfied: keras-applications>=1.0.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow>=1.5.0->stable-baselines) (1.0.6)\n",
      "Requirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.6/dist-packages (from tensorflow>=1.5.0->stable-baselines) (1.0.5)\n",
      "Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from pyglet>=1.2.0->gym[atari,classic_control]>=0.10.9->stable-baselines) (0.16.0)\n",
      "Requirement already satisfied: urllib3<1.23,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests>=2.0->gym[atari,classic_control]>=0.10.9->stable-baselines) (1.22)\n",
      "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests>=2.0->gym[atari,classic_control]>=0.10.9->stable-baselines) (3.0.4)\n",
      "Requirement already satisfied: idna<2.7,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests>=2.0->gym[atari,classic_control]>=0.10.9->stable-baselines) (2.6)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests>=2.0->gym[atari,classic_control]>=0.10.9->stable-baselines) (2018.11.29)\n",
      "Requirement already satisfied: olefile in /usr/local/lib/python3.6/dist-packages (from Pillow; extra == \"atari\"->gym[atari,classic_control]>=0.10.9->stable-baselines) (0.46)\n",
      "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf>=3.6.1->tensorflow>=1.5.0->stable-baselines) (40.6.2)\n",
      "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.13.0,>=1.12.0->tensorflow>=1.5.0->stable-baselines) (3.0.1)\n",
      "Requirement already satisfied: werkzeug>=0.11.10 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.13.0,>=1.12.0->tensorflow>=1.5.0->stable-baselines) (0.14.1)\n",
      "Requirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from keras-applications>=1.0.6->tensorflow>=1.5.0->stable-baselines) (2.8.0)\n",
      "Building wheels for collected packages: zmq, glob2, mpi4py, PyOpenGL\n",
      "  Running setup.py bdist_wheel for zmq ... \u001b[?25l-\b \bdone\n",
      "\u001b[?25h  Stored in directory: /root/.cache/pip/wheels/44/7a/7d/ac1d865766b06f9769ac1154bf31dbb5abb3b52ecfe278247a\n",
      "  Running setup.py bdist_wheel for glob2 ... \u001b[?25l-\b \bdone\n",
      "\u001b[?25h  Stored in directory: /root/.cache/pip/wheels/1b/31/14/5a25a14e933674d8636bcc42af811c7eb87ea8f6e947c17273\n",
      "  Running setup.py bdist_wheel for mpi4py ... \u001b[?25l-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \bdone\n",
      "\u001b[?25h  Stored in directory: /root/.cache/pip/wheels/23/03/c5/30289417f1428c692651e046174b64ac871a377df227802bf6\n",
      "  Running setup.py bdist_wheel for PyOpenGL ... \u001b[?25l-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \bdone\n",
      "\u001b[?25h  Stored in directory: /root/.cache/pip/wheels/6c/00/7f/1dd736f380848720ad79a1a1de5272e0d3f79c15a42968fb58\n",
      "Successfully built zmq glob2 mpi4py PyOpenGL\n",
      "Installing collected packages: python-utils, progressbar2, zmq, click, glob2, cloudpickle, mpi4py, stable-baselines, PyOpenGL, atari-py\n",
      "Successfully installed PyOpenGL-3.1.0 atari-py-0.1.7 click-7.0 cloudpickle-0.6.1 glob2-0.6 mpi4py-3.0.0 progressbar2-3.38.0 python-utils-2.3.0 stable-baselines-2.3.0 zmq-0.0.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/gym/envs/registration.py:14: PkgResourcesDeprecationWarning: Parameters to load are deprecated.  Call .resolve and .require separately.\n",
      "  result = entry_point.load(False)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------------\n",
      "| % time spent exploring  | 34       |\n",
      "| episodes                | 100      |\n",
      "| mean 100 episode reward | 16.9     |\n",
      "| steps                   | 1668     |\n",
      "--------------------------------------\n",
      "--------------------------------------\n",
      "| % time spent exploring  | 2        |\n",
      "| episodes                | 200      |\n",
      "| mean 100 episode reward | 30.2     |\n",
      "| steps                   | 4692     |\n",
      "--------------------------------------\n",
      "--------------------------------------\n",
      "| % time spent exploring  | 2        |\n",
      "| episodes                | 300      |\n",
      "| mean 100 episode reward | 131      |\n",
      "| steps                   | 17820    |\n",
      "--------------------------------------\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<stable_baselines.deepq.dqn.DQN at 0x7f3bc27d6c50>"
      ]
     },
     "execution_count": 1,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Installing the gym packgae\n",
    "!pip -q install gym\n",
    "# Installing Open MPI required to run stable-baselines\n",
    "!apt install cmake libopenmpi-dev zlib1g-dev \n",
    "# Installing stable-baselines package for DQN implementation\n",
    "!pip install stable-baselines\n",
    "\n",
    "# This is an openAI package that provides several environments that can be user to train our reinforcement learning agent\n",
    "import gym\n",
    "# Package to plot figures/graphs\n",
    "import matplotlib as plt\n",
    "\n",
    "# Importing the DummyVecEnv to initialize a vectorized environment that acts as a wrapper\n",
    "from stable_baselines.common.vec_env import DummyVecEnv\n",
    "\n",
    "# Importing the method that will decide how the policy will be calculated (MLP or CNN)\n",
    "from stable_baselines.deepq.policies import MlpPolicy, CnnPolicy\n",
    "\n",
    "# Importing the DQN model library\n",
    "from stable_baselines import DQN\n",
    "\n",
    "# The following methods create an environment that is provided by the OpenAI gym library\n",
    "env = gym.make('CartPole-v1')\n",
    "env = DummyVecEnv([lambda: env])\n",
    "\n",
    "# Model is initialized here and the method used to calculate policy is given along with the environment setup\n",
    "model = DQN(MlpPolicy, env, verbose=1)\n",
    "\n",
    "# Actual training of the DQN netwrok happens here, timesteps denote the number of steps the agent will learn for before terminating.\n",
    "model.learn(total_timesteps=25000)\n",
    "\n",
    "# Code to render the agent interacting with environment based on what it has learned\n",
    "# Unable to render on colad\n",
    "# obs = env.reset()\n",
    "# while True:\n",
    "\n",
    "# Predict action and next state from observations\n",
    "#   action, _states = model.predict(obs)\n",
    "\n",
    "# Get observation, reward and if the step is done by performing an action on the environment\n",
    "#   obs, rewards, dones, info = env.step(action)\n",
    "#   #     plt.imshow(env.render(mode='rgb_array'))\n",
    "#   #     plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DQN with Breakout-v0 Environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1942
    },
    "colab_type": "code",
    "id": "ZZ5LfDi7Qs11",
    "outputId": "1e26ca51-9c73-4481-bfa3-ea310d2969d2"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/gym/envs/registration.py:14: PkgResourcesDeprecationWarning: Parameters to load are deprecated.  Call .resolve and .require separately.\n",
      "  result = entry_point.load(False)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------------\n",
      "| % time spent exploring  | 81       |\n",
      "| episodes                | 100      |\n",
      "| mean 100 episode reward | 1.4      |\n",
      "| steps                   | 18803    |\n",
      "--------------------------------------\n",
      "--------------------------------------\n",
      "| % time spent exploring  | 63       |\n",
      "| episodes                | 200      |\n",
      "| mean 100 episode reward | 1.2      |\n",
      "| steps                   | 37230    |\n",
      "--------------------------------------\n",
      "--------------------------------------\n",
      "| % time spent exploring  | 43       |\n",
      "| episodes                | 300      |\n",
      "| mean 100 episode reward | 1.4      |\n",
      "| steps                   | 57202    |\n",
      "--------------------------------------\n",
      "--------------------------------------\n",
      "| % time spent exploring  | 22       |\n",
      "| episodes                | 400      |\n",
      "| mean 100 episode reward | 1.6      |\n",
      "| steps                   | 78851    |\n",
      "--------------------------------------\n",
      "--------------------------------------\n",
      "| % time spent exploring  | 2        |\n",
      "| episodes                | 500      |\n",
      "| mean 100 episode reward | 1.4      |\n",
      "| steps                   | 110808   |\n",
      "--------------------------------------\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "ignored",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-2-990380400d0d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     14\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     15\u001b[0m \u001b[0;31m# Actual training of the DQN netwrok happens here, timesteps denote the number of steps the agent will learn for before terminating.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 16\u001b[0;31m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlearn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtotal_timesteps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1000000\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     18\u001b[0m \u001b[0;31m# Code to render the agent interacting with environment based on what it has learned\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/stable_baselines/deepq/dqn.py\u001b[0m in \u001b[0;36mlearn\u001b[0;34m(self, total_timesteps, callback, seed, log_interval, tb_log_name)\u001b[0m\n\u001b[1;32m    182\u001b[0m                     \u001b[0mkwargs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'update_param_noise_scale'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    183\u001b[0m                 \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_default\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 184\u001b[0;31m                     \u001b[0maction\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mact\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mupdate_eps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mupdate_eps\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    185\u001b[0m                 \u001b[0menv_action\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0maction\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    186\u001b[0m                 \u001b[0mreset\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/stable_baselines/deepq/build_graph.py\u001b[0m in \u001b[0;36mact\u001b[0;34m(obs, stochastic, update_eps)\u001b[0m\n\u001b[1;32m    157\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    158\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mact\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstochastic\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mupdate_eps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 159\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0m_act\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstochastic\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mupdate_eps\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    160\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    161\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mact\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobs_phs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/stable_baselines/common/tf_util.py\u001b[0m in \u001b[0;36m<lambda>\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    243\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    244\u001b[0m         \u001b[0mfunc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_Function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mupdates\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgivens\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mgivens\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 245\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0;32mlambda\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    246\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    247\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/stable_baselines/common/tf_util.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, sess, *args, **kwargs)\u001b[0m\n\u001b[1;32m    284\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0minpt\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgivens\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    285\u001b[0m             \u001b[0mfeed_dict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0minpt\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minpt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgivens\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0minpt\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 286\u001b[0;31m         \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs_update\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    287\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mresults\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    288\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    927\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    928\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 929\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    930\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    931\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1150\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1151\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m-> 1152\u001b[0;31m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[1;32m   1153\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1154\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1326\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1327\u001b[0m       return self._do_call(_run_fn, feeds, fetches, targets, options,\n\u001b[0;32m-> 1328\u001b[0;31m                            run_metadata)\n\u001b[0m\u001b[1;32m   1329\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1330\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1332\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1333\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1334\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1335\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1336\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1317\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_extend_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1318\u001b[0m       return self._call_tf_sessionrun(\n\u001b[0;32m-> 1319\u001b[0;31m           options, feed_dict, fetch_list, target_list, run_metadata)\n\u001b[0m\u001b[1;32m   1320\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1321\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_call_tf_sessionrun\u001b[0;34m(self, options, feed_dict, fetch_list, target_list, run_metadata)\u001b[0m\n\u001b[1;32m   1405\u001b[0m     return tf_session.TF_SessionRun_wrapper(\n\u001b[1;32m   1406\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptions\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1407\u001b[0;31m         run_metadata)\n\u001b[0m\u001b[1;32m   1408\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1409\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_call_tf_sessionprun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# # Installing the gym packgae\n",
    "# !pip -q install gym\n",
    "# # Installing Open MPI required to run stable-baselines\n",
    "# !apt install cmake libopenmpi-dev zlib1g-dev \n",
    "# # Installing stable-baselines package for DQN implementation\n",
    "# !pip install stable-baselines\n",
    "\n",
    "# Import package to create atari environment\n",
    "from stable_baselines.common.cmd_util import make_atari\n",
    "\n",
    "# Importing the method that will decide how the policy will be calculated (MLP or CNN)\n",
    "from stable_baselines.deepq.policies import MlpPolicy, CnnPolicy\n",
    "\n",
    "# Importing the DQN model library\n",
    "from stable_baselines import DQN\n",
    "\n",
    "# Initialize and create the environment model\n",
    "env = make_atari('BreakoutNoFrameskip-v0')\n",
    "\n",
    "# Create the DQN model network by setting up the agent's policy and the environment\n",
    "model =DQN(MlpPolicy, env, verbose=1)\n",
    "\n",
    "# Actual training of the DQN netwrok happens here, timesteps denote the number of steps the agent will learn for before terminating.\n",
    "model.learn(total_timesteps=1000000)\n",
    "\n",
    "# Code to render the agent interacting with environment based on what it has learned\n",
    "# Unable to render on colad\n",
    "# obs = env.reset()\n",
    "# while True:\n",
    "\n",
    "# Predict action and next state from observations\n",
    "#   action, _states = model.predict(obs)\n",
    "\n",
    "# Get observation, reward and if the step is done by performing an action on the environment\n",
    "#   obs, rewards, dones, info = env.step(action)\n",
    "#   #     plt.imshow(env.render(mode='rgb_array'))\n",
    "#   #     plt.show()"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "bonus.ipynb",
   "provenance": [],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}