Q_Learning_IceLake.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import gym\n",
    "import random\n",
    "import time\n",
    "from IPython.display import clear_output\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "outputs": [
    {
     "name": "stdout",
     "text": [
      "[[0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]]\n"
     ],
     "output_type": "stream"
    }
   ],
   "source": [
    "env = gym.make(\"FrozenLake-v0\")\n",
    "action_space_size = env.action_space.n\n",
    "state_space_size = env.observation_space.n\n",
    "\n",
    "q_table = np.zeros((state_space_size, action_space_size))\n",
    "print(q_table)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "outputs": [],
   "source": [
    "num_episodes = 20000\n",
    "max_steps_per_episode = 100\n",
    "\n",
    "learning_rate = 0.1\n",
    "discount_rate = 0.99\n",
    "\n",
    "exploration_rate = 1\n",
    "max_exploration_rate = 1\n",
    "min_exploration_rate = 0.01\n",
    "exploration_decay_rate = 0.01"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% fff\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Q-learning algorithm\n",
    "```\n",
    "for episode in range(num_episodes):\n",
    "    # initialize new episode params\n",
    "    \n",
    "    for step in range(max_steps_per_episode): \n",
    "        # Exploration-exploitation trade-off\n",
    "        # Take new action\n",
    "        # Update Q-table\n",
    "        # Set new state\n",
    "        # Add new reward        \n",
    "\n",
    "    # Exploration rate decay   \n",
    "    # Add current episode reward to total rewards list\n",
    "```"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "outputs": [],
   "source": [
    "rewards_all_episodes = []\n",
    "for episode in range(num_episodes):\n",
    "    state = env.reset()\n",
    "    done = False\n",
    "    rewards_current_episode = 0\n",
    "\n",
    "    for step in range(max_steps_per_episode): \n",
    "        # Exploration-exploitation trade-off\n",
    "        exploration_rate_threshold = random.uniform(0, 1)\n",
    "        if exploration_rate_threshold > exploration_rate:\n",
    "            action = np.argmax(q_table[state,:]) \n",
    "        else:\n",
    "            action = env.action_space.sample()\n",
    "        #take new action\n",
    "        new_state, reward, done, info = env.step(action)\n",
    "        # Update Q-table for Q(s,a)\n",
    "        q_table[state, action] = q_table[state, action] * (1 - learning_rate) + \\\n",
    "            learning_rate * (reward + discount_rate * np.max(q_table[new_state, :]))\n",
    "        \n",
    "        state = new_state\n",
    "        rewards_current_episode += reward \n",
    "        \n",
    "        if done == True: \n",
    "            break\n",
    "            \n",
    "    # Exploration rate decay\n",
    "    exploration_rate = min_exploration_rate + \\\n",
    "        (max_exploration_rate - min_exploration_rate) * np.exp(-exploration_decay_rate*episode)\n",
    "    rewards_all_episodes.append(rewards_current_episode)\n",
    "    "
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "outputs": [
    {
     "name": "stdout",
     "text": [
      "********Average reward per thousand episodes********\n",
      "\n",
      "1000 :  0.04100000000000003\n",
      "2000 :  0.05500000000000004\n",
      "3000 :  0.08400000000000006\n",
      "4000 :  0.10800000000000008\n",
      "5000 :  0.10900000000000008\n",
      "6000 :  0.1320000000000001\n",
      "7000 :  0.1460000000000001\n",
      "8000 :  0.17500000000000013\n",
      "9000 :  0.24000000000000019\n",
      "10000 :  0.2620000000000002\n",
      "11000 :  0.45800000000000035\n",
      "12000 :  0.6610000000000005\n",
      "13000 :  0.6670000000000005\n",
      "14000 :  0.6930000000000005\n",
      "15000 :  0.6980000000000005\n",
      "16000 :  0.6840000000000005\n",
      "17000 :  0.6650000000000005\n",
      "18000 :  0.7030000000000005\n",
      "19000 :  0.6820000000000005\n",
      "20000 :  0.6890000000000005\n"
     ],
     "output_type": "stream"
    }
   ],
   "source": [
    "# Calculate and print the average reward per thousand episodes\n",
    "rewards_per_thosand_episodes = np.split(np.array(rewards_all_episodes),num_episodes/1000)\n",
    "count = 1000\n",
    "\n",
    "print(\"********Average reward per thousand episodes********\\n\")\n",
    "for r in rewards_per_thosand_episodes:\n",
    "    print(count, \": \", str(sum(r/1000)))\n",
    "    count += 1000\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "outputs": [
    {
     "name": "stdout",
     "text": [
      "\n",
      "\n",
      "********Q-table********\n",
      "\n",
      "[[0.56155668 0.50676359 0.51113332 0.51633663]\n",
      " [0.31394613 0.31218741 0.24458661 0.49321051]\n",
      " [0.41637602 0.37275899 0.41689343 0.45542588]\n",
      " [0.20679537 0.29910297 0.36394872 0.42988882]\n",
      " [0.58086203 0.41773475 0.44126002 0.42910216]\n",
      " [0.         0.         0.         0.        ]\n",
      " [0.33825872 0.14911566 0.18237918 0.0651859 ]\n",
      " [0.         0.         0.         0.        ]\n",
      " [0.33209066 0.48196698 0.49282896 0.64355292]\n",
      " [0.2918537  0.70621805 0.53639371 0.44925847]\n",
      " [0.65793949 0.41190176 0.40708876 0.33627327]\n",
      " [0.         0.         0.         0.        ]\n",
      " [0.         0.         0.         0.        ]\n",
      " [0.4509215  0.62594686 0.76507311 0.51494189]\n",
      " [0.74501853 0.84665135 0.77416626 0.77714155]\n",
      " [0.         0.         0.         0.        ]]\n"
     ],
     "output_type": "stream"
    }
   ],
   "source": [
    "# Print updated Q-table\n",
    "print(\"\\n\\n********Q-table********\\n\")\n",
    "print(q_table)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### The code to watch the agent play the game \n",
    "```\n",
    "# Watch our agent play Frozen Lake by playing the best action \n",
    "# from each state according to the Q-table\n",
    "\n",
    "for episode in range(3):\n",
    "    # initialize new episode params\n",
    "\n",
    "    for step in range(max_steps_per_episode):        \n",
    "        # Show current state of environment on screen\n",
    "        # Choose action with highest Q-value for current state       \n",
    "        # Take new action\n",
    "        \n",
    "        if done:\n",
    "            if reward == 1:\n",
    "                # Agent reached the goal and won episode\n",
    "            else:\n",
    "                # Agent stepped in a hole and lost episode            \n",
    "            \n",
    "        # Set new state\n",
    "        \n",
    "env.close()\n",
    "```\n",
    "\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "outputs": [
    {
     "name": "stdout",
     "text": [
      "  (Down)\n",
      "SFFF\n",
      "FHFH\n",
      "FFFH\n",
      "HFF\u001b[41mG\u001b[0m\n",
      "****You reached the goal!****\n"
     ],
     "output_type": "stream"
    }
   ],
   "source": [
    "for episode in range(3):\n",
    "    state = env.reset()\n",
    "    done = False\n",
    "    print(\"*****EPISODE \", episode+1, \"*****\\n\\n\\n\\n\")\n",
    "    time.sleep(1)\n",
    "    \n",
    "    for step in range(max_steps_per_episode):\n",
    "        clear_output(wait=True)\n",
    "        env.render()\n",
    "        time.sleep(0.3)\n",
    "        \n",
    "        action = np.argmax(q_table[state,:])        \n",
    "        new_state, reward, done, info = env.step(action)\n",
    "        \n",
    "        if done:\n",
    "            clear_output(wait=True)\n",
    "            env.render()\n",
    "            if reward == 1:\n",
    "                print(\"****You reached the goal!****\")\n",
    "                time.sleep(3)\n",
    "            else:\n",
    "                print(\"****You fell through a hole!****\")\n",
    "                time.sleep(3)\n",
    "                clear_output(wait=True)\n",
    "            break\n",
    "        state = new_state\n",
    "\n",
    "env.close()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "outputs": [],
   "source": [
    "\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  },
  "pycharm": {
   "stem_cell": {
    "cell_type": "raw",
    "source": [],
    "metadata": {
     "collapsed": false
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}