Muesli_LunarLander.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "3w04_bU_4H9g"
      },
      "outputs": [],
      "source": [
        "!pip install tensorboardX\n",
        "from tensorboardX import SummaryWriter\n",
        "import gym\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "from torch.distributions import Categorical\n",
        "\n",
        "\n",
        "!pip install gym[classic_control]\n",
        "!pip install gym[box2d] #for lunarlander\n",
        "!apt update\n",
        "!apt install xvfb\n",
        "!pip install pyvirtualdisplay\n",
        "!pip install gym-notebook-wrapper\n",
        "import gnwrapper\n",
        "!nvidia-smi\n",
        "print(torch.cuda.is_available())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WgMBnokKfDY7"
      },
      "outputs": [],
      "source": [
        "## For linear lr decay\n",
        "## https://github.com/cmpark0126/pytorch-polynomial-lr-decay\n",
        "! pip install git+https://github.com/cmpark0126/pytorch-polynomial-lr-decay.git\n",
        "from torch_poly_lr_decay import PolynomialLRDecay"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "n_n1jjuq_TCF"
      },
      "outputs": [],
      "source": [
        "class Representation(nn.Module): \n",
        "    \"\"\"Representation Network\n",
        "\n",
        "    Representation network produces hidden state from observations.\n",
        "    Hidden state scaled within the bounds of [-1,1]. \n",
        "    Simple mlp network used with 1 skip connection.\n",
        "\n",
        "    input : raw input\n",
        "    output : hs(hidden state) \n",
        "    \"\"\"\n",
        "    def __init__(self, input_dim, output_dim, width):\n",
        "        super().__init__()\n",
        "        self.skip = torch.nn.Linear(input_dim, output_dim)  \n",
        "        self.layer1 = torch.nn.Linear(input_dim, width)\n",
        "        self.layer2 = torch.nn.Linear(width, width)\n",
        "        self.layer3 = torch.nn.Linear(width, width) \n",
        "        self.layer4 = torch.nn.Linear(width, width)  \n",
        "        self.layer5 = torch.nn.Linear(width, output_dim)     \n",
        "        \n",
        "    def forward(self, x):\n",
        "        s = self.skip(x)\n",
        "        x = self.layer1(x)\n",
        "        x = torch.nn.functional.relu(x)\n",
        "        x = self.layer2(x)\n",
        "        x = torch.nn.functional.relu(x)\n",
        "        x = self.layer3(x)\n",
        "        x = torch.nn.functional.relu(x)\n",
        "        x = self.layer4(x)\n",
        "        x = torch.nn.functional.relu(x)\n",
        "        x = self.layer5(x)    \n",
        "        x = torch.nn.functional.relu(x+s)\n",
        "        x = 2*(x - x.min(-1,keepdim=True)[0])/(x.max(-1,keepdim=True)[0] - x.min(-1,keepdim=True)[0])-1 \n",
        "        return x\n",
        "\n",
        "\n",
        "class Dynamics(nn.Module): \n",
        "    \"\"\"Dynamics Network\n",
        "\n",
        "    Dynamics network transits (hidden state + action) to next hidden state and inferences reward model.\n",
        "    Hidden state scaled within the bounds of [-1,1]. Action encoded to one-hot representation. \n",
        "    Zeros tensor is used for action -1.\n",
        "    \n",
        "    Output of the reward head is categorical representation, instaed of scalar value.\n",
        "    Categorical output will be converted to scalar value with 'to_scalar()',and when \n",
        "    traning target value will be converted to categorical target with 'to_cr()'.\n",
        "    \n",
        "    input : hs, action\n",
        "    output : next_hs, reward \n",
        "    \"\"\"\n",
        "    def __init__(self, input_dim, output_dim, width, action_space):\n",
        "        super().__init__()\n",
        "        self.layer1 = torch.nn.Linear(input_dim + action_space, width)\n",
        "        self.layer2 = torch.nn.Linear(width, width) \n",
        "        self.hs_head = torch.nn.Linear(width, output_dim)\n",
        "        self.reward_head = nn.Sequential(\n",
        "            nn.Linear(width,width),\n",
        "            nn.ReLU(),\n",
        "            nn.Linear(width,width),\n",
        "            nn.ReLU(),\n",
        "            nn.Linear(width,support_size*2+1)           \n",
        "        ) \n",
        "        self.one_hot_act = torch.cat((F.one_hot(torch.arange(0, action_space) % action_space, num_classes=action_space),\n",
        "                                      torch.zeros(action_space).unsqueeze(0)),\n",
        "                                      dim=0).to(device)        \n",
        "       \n",
        "    def forward(self, x, action):\n",
        "        action = self.one_hot_act[action.squeeze(1)]\n",
        "        x = torch.cat((x,action.to(device)), dim=1)\n",
        "        x = self.layer1(x)\n",
        "        x = torch.nn.functional.relu(x)\n",
        "        x = self.layer2(x)\n",
        "        x = torch.nn.functional.relu(x)\n",
        "        hs = self.hs_head(x)\n",
        "        hs = torch.nn.functional.relu(hs)\n",
        "        reward = self.reward_head(x)    \n",
        "        hs = 2*(hs - hs.min(-1,keepdim=True)[0])/(hs.max(-1,keepdim=True)[0] - hs.min(-1,keepdim=True)[0])-1\n",
        "        return hs, reward\n",
        "\n",
        "\n",
        "class Prediction(nn.Module): \n",
        "    \"\"\"Prediction Network\n",
        "\n",
        "    Prediction network inferences probability distribution of policy and value model from hidden state. \n",
        "\n",
        "    Output of the value head is categorical representation, instaed of scalar value.\n",
        "    Categorical output will be converted to scalar value with 'to_scalar()',and when \n",
        "    traning target value will be converted to categorical target with 'to_cr()'.\n",
        "        \n",
        "    input : hs\n",
        "    output : P, V \n",
        "    \"\"\"\n",
        "    def __init__(self, input_dim, output_dim, width):\n",
        "        super().__init__()\n",
        "        self.layer1 = torch.nn.Linear(input_dim, width)\n",
        "        self.layer2 = torch.nn.Linear(width, width) \n",
        "        self.policy_head = nn.Sequential(\n",
        "            nn.Linear(width,width),\n",
        "            nn.ReLU(),\n",
        "            nn.Linear(width,width),\n",
        "            nn.ReLU(),\n",
        "            nn.Linear(width,output_dim)           \n",
        "        ) \n",
        "        self.value_head = nn.Sequential(\n",
        "            nn.Linear(width,width),\n",
        "            nn.ReLU(),\n",
        "            nn.Linear(width,width),\n",
        "            nn.ReLU(),\n",
        "            nn.Linear(width,support_size*2+1)           \n",
        "        ) \n",
        "   \n",
        "    def forward(self, x):\n",
        "        x = self.layer1(x)\n",
        "        x = torch.nn.functional.relu(x)\n",
        "        x = self.layer2(x)\n",
        "        x = torch.nn.functional.relu(x)\n",
        "        P = self.policy_head(x)\n",
        "        P = torch.nn.functional.softmax(P, dim=-1) \n",
        "        V = self.value_head(x)      \n",
        "        return P, V\n",
        "\n",
        "\n",
        "\"\"\"\n",
        "For categorical representation\n",
        "reference : https://github.com/werner-duvaud/muzero-general\n",
        "In my opinion, support size have to cover the range of maximum absolute value of \n",
        "reward and value of entire trajectories. Support_size 30 can cover almost [-900,900].\n",
        "\"\"\"\n",
        "support_size = 30\n",
        "eps = 0.001\n",
        "\n",
        "def to_scalar(x):\n",
        "    x = torch.softmax(x, dim=-1)\n",
        "    probabilities = x\n",
        "    support = (torch.tensor([x for x in range(-support_size, support_size + 1)]).expand(probabilities.shape).float().to(device))\n",
        "    x = torch.sum(support * probabilities, dim=1, keepdim=True)\n",
        "    scalar = torch.sign(x) * (((torch.sqrt(1 + 4 * eps * (torch.abs(x) + 1 + eps)) - 1) / (2 * eps))** 2 - 1)\n",
        "    return scalar\n",
        "\n",
        "def to_scalar_no_soft(x): ## test purpose\n",
        "    probabilities = x \n",
        "    support = (torch.tensor([x for x in range(-support_size, support_size + 1)]).expand(probabilities.shape).float().to(device))\n",
        "    x = torch.sum(support * probabilities, dim=1, keepdim=True)\n",
        "    scalar = torch.sign(x) * (((torch.sqrt(1 + 4 * eps * (torch.abs(x) + 1 + eps)) - 1) / (2 * eps))** 2 - 1)\n",
        "    return scalar\n",
        "\n",
        "def to_cr(x):\n",
        "    x = x.squeeze(-1).unsqueeze(0)\n",
        "    x = torch.sign(x) * (torch.sqrt(torch.abs(x) + 1) - 1) + eps * x\n",
        "    x = torch.clip(x, -support_size, support_size)\n",
        "    floor = x.floor()\n",
        "    under = x - floor\n",
        "    floor_prob = (1 - under)\n",
        "    under_prob = under\n",
        "    floor_index = floor + support_size\n",
        "    under_index = floor + support_size + 1\n",
        "    logits = torch.zeros(x.shape[0], x.shape[1], 2 * support_size + 1).type(torch.float32).to(device)\n",
        "    logits.scatter_(2, floor_index.long().unsqueeze(-1), floor_prob.unsqueeze(-1))\n",
        "    under_prob = under_prob.masked_fill_(2 * support_size < under_index, 0.0)\n",
        "    under_index = under_index.masked_fill_(2 * support_size < under_index, 0.0)\n",
        "    logits.scatter_(2, under_index.long().unsqueeze(-1), under_prob.unsqueeze(-1))\n",
        "    return logits.squeeze(0)\n",
        "\n",
        "\n",
        "##Target network\n",
        "class Target(nn.Module):\n",
        "    \"\"\"Target Network\n",
        "    \n",
        "    Target network is used to approximate v_pi_prior, q_pi_prior, pi_prior.\n",
        "    It contains older network parameters. (exponential moving average update)\n",
        "    \"\"\"\n",
        "    def __init__(self, state_dim, action_dim, width):\n",
        "        super().__init__()\n",
        "        self.representation_network = Representation(state_dim*8, state_dim*4, width) \n",
        "        self.dynamics_network = Dynamics(state_dim*4, state_dim*4, width, action_dim)\n",
        "        self.prediction_network = Prediction(state_dim*4, action_dim, width) \n",
        "        self.to(device)\n",
        "\n",
        "##Muesli agent\n",
        "class Agent(nn.Module):\n",
        "    \"\"\"Agent Class\"\"\"\n",
        "    def __init__(self, state_dim, action_dim, width):\n",
        "        super().__init__()\n",
        "        self.representation_network = Representation(state_dim*8, state_dim*4, width) \n",
        "        self.dynamics_network = Dynamics(state_dim*4, state_dim*4, width, action_dim)\n",
        "        self.prediction_network = Prediction(state_dim*4, action_dim, width) \n",
        "        self.optimizer = torch.optim.AdamW(self.parameters(), lr=0.00032, weight_decay=0)\n",
        "        self.scheduler = PolynomialLRDecay(self.optimizer, max_decay_steps=4000, end_learning_rate=0.0000)   \n",
        "        self.to(device)\n",
        "\n",
        "        self.state_traj = []\n",
        "        self.action_traj = []\n",
        "        self.P_traj = []\n",
        "        self.r_traj = []      \n",
        "\n",
        "        self.state_replay = []\n",
        "        self.action_replay = []\n",
        "        self.P_replay = []\n",
        "        self.r_replay = []   \n",
        "\n",
        "        self.action_space = action_dim\n",
        "        self.env = gym.make(game_name)     \n",
        "\n",
        "        self.var = 0\n",
        "        self.beta_product = 1.0\n",
        "\n",
        "        self.var_m = [0 for _ in range(5)]\n",
        "        self.beta_product_m = [1.0 for _ in range(5)] \n",
        "\n",
        "\n",
        "    def self_play_mu(self, max_timestep=10000):        \n",
        "        \"\"\"Self-play and save trajectory to replay buffer\n",
        "\n",
        "        (Originally target network used to inference policy, but i used agent network instead)\n",
        "\n",
        "        Eight previous observations stacked -> representation network -> prediction network \n",
        "        -> sampling action follow policy -> next env step\n",
        "        \"\"\"      \n",
        "        game_score = 0\n",
        "        state = self.env.reset()\n",
        "        state_dim = len(state)\n",
        "        for i in range(max_timestep):   \n",
        "            start_state = state\n",
        "            if i == 0:\n",
        "                stacked_state = np.concatenate((state, state, state, state, state, state, state, state), axis=0)\n",
        "            else:\n",
        "                stacked_state = np.roll(stacked_state,-state_dim,axis=0)                \n",
        "                stacked_state[-state_dim:]=state\n",
        "\n",
        "            with torch.no_grad():\n",
        "                hs = self.representation_network(torch.from_numpy(stacked_state).float().to(device))\n",
        "                P, v = self.prediction_network(hs)    \n",
        "            action = np.random.choice(np.arange(self.action_space), p=P.detach().cpu().numpy())   \n",
        "            state, r, done, info = self.env.step(action)                    \n",
        "            \n",
        "            if i == 0:\n",
        "                for _ in range(8):\n",
        "                    self.state_traj.append(start_state)\n",
        "            else:\n",
        "                self.state_traj.append(start_state)\n",
        "            self.action_traj.append(action)\n",
        "            self.P_traj.append(P.cpu().numpy())\n",
        "            self.r_traj.append(r)\n",
        "\n",
        "            game_score += r\n",
        "\n",
        "            ## For fix lunarlander-v2 env does not return reward -100 when 'TimeLimit.truncated'\n",
        "            if done:\n",
        "                if (info['TimeLimit.truncated'] == True) and abs(r)!=100:\n",
        "                    game_score -= 100\n",
        "                    self.r_traj[-1] = -100\n",
        "                    r = -100\n",
        "                last_frame = i\n",
        "                break\n",
        "\n",
        "        #print('self_play: score, r, done, info, lastframe', int(game_score), r, done, info, i)\n",
        "\n",
        "\n",
        "        # for update inference over trajectory length\n",
        "        for _ in range(5):\n",
        "            self.state_traj.append(np.zeros_like(state))\n",
        "\n",
        "        for _ in range(6):\n",
        "            self.r_traj.append(0.0)\n",
        "            self.action_traj.append(-1)  \n",
        "\n",
        "        # traj append to replay\n",
        "        self.state_replay.append(self.state_traj)\n",
        "        self.action_replay.append(self.action_traj)\n",
        "        self.P_replay.append(self.P_traj)\n",
        "        self.r_replay.append(self.r_traj)  \n",
        "\n",
        "        writer.add_scalars('Selfplay',\n",
        "                           {'lastreward': r,\n",
        "                            'lastframe': last_frame+1\n",
        "                           },global_i)\n",
        "\n",
        "        return game_score , r, last_frame\n",
        "\n",
        "\n",
        "    def update_weights_mu(self, target):\n",
        "        \"\"\"Optimize network weights.\n",
        "\n",
        "        Iteration: 20\n",
        "        Mini-batch size: 16 (4 replay, 4 seqeuences in 1 replay)\n",
        "        Replay: Uniform replay without on-policy data\n",
        "        Discount: 0.997\n",
        "        Unroll: 5 step\n",
        "        L_m: 5 step(Muesli)\n",
        "        Observations: Stack 8 frame\n",
        "        regularizer_multiplier: 5 \n",
        "        Loss: L_pg_cmpo + L_v/6/4 + L_r/5/1 + L_m\n",
        "        \"\"\"\n",
        "\n",
        "        for _ in range(20): \n",
        "            state_traj = []\n",
        "            action_traj = []\n",
        "            P_traj = []\n",
        "            r_traj = []      \n",
        "            G_arr_mb = []\n",
        "\n",
        "            for epi_sel in range(4):\n",
        "                if(epi_sel>-1):## replay proportion\n",
        "                    sel = np.random.randint(0,len(self.state_replay)) \n",
        "                else:\n",
        "                    sel = -1\n",
        "\n",
        "                ## multi step return G (orignally retrace used)\n",
        "                G = 0\n",
        "                G_arr = []\n",
        "                for r in self.r_replay[sel][::-1]:\n",
        "                    G = 0.997 * G + r\n",
        "                    G_arr.append(G)\n",
        "                G_arr.reverse()\n",
        "                \n",
        "                for i in np.random.randint(len(self.state_replay[sel])-5-7,size=4):\n",
        "                    state_traj.append(self.state_replay[sel][i:i+13])\n",
        "                    action_traj.append(self.action_replay[sel][i:i+5])\n",
        "                    r_traj.append(self.r_replay[sel][i:i+5])\n",
        "                    G_arr_mb.append(G_arr[i:i+6])                        \n",
        "                    P_traj.append(self.P_replay[sel][i])\n",
        "\n",
        "\n",
        "            state_traj = torch.from_numpy(np.array(state_traj)).to(device)\n",
        "            action_traj = torch.from_numpy(np.array(action_traj)).unsqueeze(2).to(device)\n",
        "            P_traj = torch.from_numpy(np.array(P_traj)).to(device)\n",
        "            G_arr_mb = torch.from_numpy(np.array(G_arr_mb)).unsqueeze(2).float().to(device)\n",
        "            r_traj = torch.from_numpy(np.array(r_traj)).unsqueeze(2).float().to(device)\n",
        "            inferenced_P_arr = []\n",
        "\n",
        "            ## stacking 8 frame\n",
        "            stacked_state_0 = torch.cat((state_traj[:,0], state_traj[:,1], state_traj[:,2], state_traj[:,3],\n",
        "                                         state_traj[:,4], state_traj[:,5], state_traj[:,6], state_traj[:,7]), dim=1)\n",
        "\n",
        "\n",
        "            ## agent network inference (5 step unroll)\n",
        "            first_hs = self.representation_network(stacked_state_0)\n",
        "            first_P, first_v_logits = self.prediction_network(first_hs)      \n",
        "            inferenced_P_arr.append(first_P)\n",
        "\n",
        "            second_hs, r_logits = self.dynamics_network(first_hs, action_traj[:,0])    \n",
        "            second_P, second_v_logits = self.prediction_network(second_hs)\n",
        "            inferenced_P_arr.append(second_P)\n",
        "\n",
        "            third_hs, r2_logits = self.dynamics_network(second_hs, action_traj[:,1])    \n",
        "            third_P, third_v_logits = self.prediction_network(third_hs)\n",
        "            inferenced_P_arr.append(third_P)\n",
        "\n",
        "            fourth_hs, r3_logits = self.dynamics_network(third_hs, action_traj[:,2])    \n",
        "            fourth_P, fourth_v_logits = self.prediction_network(fourth_hs)\n",
        "            inferenced_P_arr.append(fourth_P)\n",
        "\n",
        "            fifth_hs, r4_logits = self.dynamics_network(fourth_hs, action_traj[:,3])    \n",
        "            fifth_P, fifth_v_logits = self.prediction_network(fifth_hs)\n",
        "            inferenced_P_arr.append(fifth_P)\n",
        "\n",
        "            sixth_hs, r5_logits = self.dynamics_network(fifth_hs, action_traj[:,4])    \n",
        "            sixth_P, sixth_v_logits = self.prediction_network(sixth_hs)\n",
        "            inferenced_P_arr.append(sixth_P)\n",
        "\n",
        "\n",
        "            ## target network inference\n",
        "            with torch.no_grad():\n",
        "                t_first_hs = target.representation_network(stacked_state_0)\n",
        "                t_first_P, t_first_v_logits = target.prediction_network(t_first_hs)  \n",
        "\n",
        "\n",
        "            ## normalized advantage\n",
        "            beta_var = 0.99\n",
        "            self.var = beta_var*self.var + (1-beta_var)*(torch.sum((G_arr_mb[:,0] - to_scalar(t_first_v_logits))**2)/16)\n",
        "            self.beta_product *= beta_var\n",
        "            var_hat = self.var/(1-self.beta_product)\n",
        "            under = torch.sqrt(var_hat + 1e-12)\n",
        "\n",
        "\n",
        "            ## L_pg_cmpo first term (eq.10)\n",
        "            importance_weight = torch.clip(first_P.gather(1,action_traj[:,0])\n",
        "                                        /(P_traj.gather(1,action_traj[:,0])),\n",
        "                                        0, 1\n",
        "            )\n",
        "            first_term = -1 * importance_weight * (G_arr_mb[:,0] - to_scalar(t_first_v_logits))/under\n",
        "\n",
        "\n",
        "            ##lookahead inferences (one step look-ahead to some actions to estimate q_prior, from target network)\n",
        "            with torch.no_grad():                                \n",
        "                r1_arr = []\n",
        "                v1_arr = []\n",
        "                a1_arr = []\n",
        "                for _ in range(self.action_space): #sample <= N(action space), now N    \n",
        "                    action1_stack = []\n",
        "                    for p in t_first_P:             \n",
        "                        action1_stack.append(np.random.choice(np.arange(self.action_space), p=p.detach().cpu().numpy()))    \n",
        "                    hs, r1 = target.dynamics_network(t_first_hs, torch.unsqueeze(torch.tensor(action1_stack),1))\n",
        "                    _, v1 = target.prediction_network(hs)\n",
        "\n",
        "                    r1_arr.append(to_scalar(r1))\n",
        "                    v1_arr.append(to_scalar(v1) )\n",
        "                    a1_arr.append(torch.tensor(action1_stack))               \n",
        "\n",
        "            ## z_cmpo_arr (eq.12)\n",
        "            with torch.no_grad():   \n",
        "                exp_clip_adv_arr = [torch.exp(torch.clip((r1_arr[k] + 0.997 * v1_arr[k] - to_scalar(t_first_v_logits))/under, -1, 1))\n",
        "                                    .tolist() for k in range(self.action_space)]\n",
        "                exp_clip_adv_arr = torch.tensor(exp_clip_adv_arr).to(device)\n",
        "                z_cmpo_arr = []\n",
        "                for k in range(self.action_space):\n",
        "                    z_cmpo = (1 + torch.sum(exp_clip_adv_arr[k],dim=0) - exp_clip_adv_arr[k]) / self.action_space \n",
        "                    z_cmpo_arr.append(z_cmpo.tolist())\n",
        "            z_cmpo_arr = torch.tensor(z_cmpo_arr).to(device)\n",
        "\n",
        "\n",
        "            ## L_pg_cmpo second term (eq.11)\n",
        "            second_term = 0\n",
        "            for k in range(self.action_space):\n",
        "                second_term += exp_clip_adv_arr[k]/z_cmpo_arr[k] * torch.log(first_P.gather(1, torch.unsqueeze(a1_arr[k],1).to(device)))\n",
        "            regularizer_multiplier = 5 \n",
        "            second_term *= -1 * regularizer_multiplier / self.action_space\n",
        "\n",
        "\n",
        "            ## L_pg_cmpo               \n",
        "            L_pg_cmpo = first_term + second_term\n",
        "\n",
        "\n",
        "            ## L_m\n",
        "            L_m  = 0\n",
        "            for i in range(5):\n",
        "                stacked_state = torch.cat(( state_traj[:,i+1], state_traj[:,i+2], state_traj[:,i+3], state_traj[:,i+4],\n",
        "                                            state_traj[:,i+5], state_traj[:,i+6], state_traj[:,i+7], state_traj[:,i+8]), dim=1)\n",
        "                with torch.no_grad():\n",
        "                    t_hs = target.representation_network(stacked_state)\n",
        "                    t_P, t_v_logits = target.prediction_network(t_hs) \n",
        "\n",
        "                beta_var = 0.99\n",
        "                self.var_m[i] = beta_var*self.var_m[i] + (1-beta_var)*(torch.sum((G_arr_mb[:,i+1] - to_scalar(t_v_logits))**2)/16)\n",
        "                self.beta_product_m[i]  *= beta_var\n",
        "                var_hat = self.var_m[i] /(1-self.beta_product_m[i])\n",
        "                under = torch.sqrt(var_hat + 1e-12)\n",
        "\n",
        "                with torch.no_grad():                                \n",
        "                    r1_arr = []\n",
        "                    v1_arr = []\n",
        "                    a1_arr = []\n",
        "                    for j in range(self.action_space):  \n",
        "                        action1_stack = []\n",
        "                        for _ in t_P:             \n",
        "                            action1_stack.append(j)    \n",
        "                        hs, r1 = target.dynamics_network(t_hs, torch.unsqueeze(torch.tensor(action1_stack),1))\n",
        "                        _, v1 = target.prediction_network(hs)\n",
        "\n",
        "                        r1_arr.append(to_scalar(r1))\n",
        "                        v1_arr.append(to_scalar(v1) )\n",
        "                        a1_arr.append(torch.tensor(action1_stack))    \n",
        "\n",
        "                with torch.no_grad():   \n",
        "                    exp_clip_adv_arr = [torch.exp(torch.clip((r1_arr[k] + 0.997 * v1_arr[k] - to_scalar(t_v_logits))/under,-1, 1))\n",
        "                                        .tolist() for k in range(self.action_space)]\n",
        "                    exp_clip_adv_arr = torch.tensor(exp_clip_adv_arr).to(device)\n",
        "\n",
        "                ## Paper appendix F.2 : Prior policy\n",
        "                t_P = 0.967*t_P + 0.03*P_traj + 0.003*torch.tensor([[0.25,0.25,0.25,0.25] for _ in range(16)]).to(device) \n",
        "\n",
        "                pi_cmpo_all = [(t_P.gather(1, torch.unsqueeze(a1_arr[k],1).to(device)) \n",
        "                                * exp_clip_adv_arr[k])\n",
        "                                .squeeze(-1).tolist() for k in range(self.action_space)]                \n",
        "        \n",
        "                pi_cmpo_all = torch.tensor(pi_cmpo_all).transpose(0,1).to(device)\n",
        "                pi_cmpo_all = pi_cmpo_all/torch.sum(pi_cmpo_all,dim=1).unsqueeze(-1)\n",
        "                kl_loss = torch.nn.KLDivLoss(reduction=\"batchmean\")\n",
        "                L_m += kl_loss(torch.log(inferenced_P_arr[i+1]), pi_cmpo_all) \n",
        "            \n",
        "            L_m/=5\n",
        "\n",
        "\n",
        "            ## L_v\n",
        "            ls = nn.LogSoftmax(dim=-1)\n",
        "\n",
        "            L_v = -1 * (\n",
        "                (to_cr(G_arr_mb[:,0])*ls(first_v_logits)).sum(-1, keepdim=True)\n",
        "                +  (to_cr(G_arr_mb[:,1])*ls(second_v_logits)).sum(-1, keepdim=True)\n",
        "                +  (to_cr(G_arr_mb[:,2])*ls(third_v_logits)).sum(-1, keepdim=True)\n",
        "                +  (to_cr(G_arr_mb[:,3])*ls(fourth_v_logits)).sum(-1, keepdim=True)\n",
        "                +  (to_cr(G_arr_mb[:,4])*ls(fifth_v_logits)).sum(-1, keepdim=True)\n",
        "                +  (to_cr(G_arr_mb[:,5])*ls(sixth_v_logits)).sum(-1, keepdim=True)\n",
        "            )\n",
        "\n",
        "            ## L_r     \n",
        "            L_r = -1 * (\n",
        "                (to_cr(r_traj[:,0])*ls(r_logits)).sum(-1, keepdim=True)\n",
        "                + (to_cr(r_traj[:,1])*ls(r2_logits)).sum(-1, keepdim=True)\n",
        "                + (to_cr(r_traj[:,2])*ls(r3_logits)).sum(-1, keepdim=True)\n",
        "                + (to_cr(r_traj[:,3])*ls(r4_logits)).sum(-1, keepdim=True)\n",
        "                + (to_cr(r_traj[:,4])*ls(r5_logits)).sum(-1, keepdim=True)\n",
        "            )\n",
        "\n",
        "\n",
        "            ## start of dynamics network gradient *0.5\n",
        "            first_hs.register_hook(lambda grad: grad * 0.5)\n",
        "            second_hs.register_hook(lambda grad: grad * 0.5) \n",
        "            third_hs.register_hook(lambda grad: grad * 0.5)    \n",
        "            fourth_hs.register_hook(lambda grad: grad * 0.5)   \n",
        "            fifth_hs.register_hook(lambda grad: grad * 0.5)  \n",
        "            sixth_hs.register_hook(lambda grad: grad * 0.5)  \n",
        "\n",
        "\n",
        "            ## total loss\n",
        "            L_total = L_pg_cmpo + L_v/6/4 + L_r/5/1 + L_m   \n",
        "          \n",
        "            \n",
        "            ## optimize\n",
        "            self.optimizer.zero_grad()\n",
        "            L_total.mean().backward()\n",
        "            nn.utils.clip_grad_value_(self.parameters(), clip_value=1.0)\n",
        "            self.optimizer.step()\n",
        "            \n",
        "\n",
        "            ## target network(prior parameters) moving average update\n",
        "            alpha_target = 0.01 \n",
        "            params1 = self.named_parameters()\n",
        "            params2 = target.named_parameters()\n",
        "            dict_params2 = dict(params2)\n",
        "            for name1, param1 in params1:\n",
        "                if name1 in dict_params2:\n",
        "                    dict_params2[name1].data.copy_(alpha_target*param1.data + (1-alpha_target)*dict_params2[name1].data)\n",
        "            target.load_state_dict(dict_params2)          \n",
        "\n",
        "\n",
        "        self.scheduler.step()\n",
        "\n",
        "        self.state_traj.clear()\n",
        "        self.action_traj.clear()\n",
        "        self.P_traj.clear()\n",
        "        self.r_traj.clear()\n",
        "\n",
        "        writer.add_scalars('Loss',{'L_total': L_total.mean(),\n",
        "                                  'L_pg_cmpo': L_pg_cmpo.mean(),\n",
        "                                  'L_v': (L_v/6/4).mean(),\n",
        "                                  'L_r': (L_r/5/1).mean(),\n",
        "                                  'L_m': (L_m).mean()\n",
        "                                  },global_i)\n",
        "        \n",
        "        writer.add_scalars('vars',{'self.var':self.var,\n",
        "                                   'self.var_m':self.var_m[0]\n",
        "                                  },global_i)\n",
        "        \n",
        "        return\n",
        "\n",
        "\n",
        "\n",
        "%rm -rf scalar/\n",
        "%load_ext tensorboard\n",
        "%tensorboard --logdir scalar --port=6008\n",
        "\n",
        "device = torch.device('cuda:0') if torch.cuda.is_available() else torch.device('cpu')\n",
        "score_arr = []\n",
        "game_name = 'LunarLander-v2' \n",
        "env = gym.make(game_name) \n",
        "target = Target(env.observation_space.shape[0], env.action_space.n, 128)\n",
        "agent = Agent(env.observation_space.shape[0], env.action_space.n, 128)  \n",
        "print(agent)\n",
        "env.close()\n",
        "\n",
        "## initialization\n",
        "target.load_state_dict(agent.state_dict())\n",
        "\n",
        "## Self play & Weight update loop\n",
        "episode_nums = 4000\n",
        "for i in range(episode_nums):\n",
        "    writer = SummaryWriter(logdir='scalar/')\n",
        "    global_i = i    \n",
        "    game_score , last_r, frame= agent.self_play_mu()       \n",
        "    writer.add_scalar('score', game_score, global_i)\n",
        "    \n",
        "    score_arr.append(game_score)  \n",
        "    if i%10==0:\n",
        "        mean_score = np.mean(np.array(score_arr[i-30:i]))\n",
        "        #print('episode, avg, score, last_r, len\\n', i, mean_score, int(game_score), last_r, frame)\n",
        "\n",
        "    if i%100==0:\n",
        "        torch.save(agent.state_dict(), 'weights.pt') \n",
        "\n",
        "    if mean_score > 250 and np.mean(np.array(score_arr[-5:])) > 250:\n",
        "        torch.save(agent.state_dict(), 'weights.pt')\n",
        "        print('Done')\n",
        "        break\n",
        "\n",
        "    agent.update_weights_mu(target) \n",
        "    writer.close()\n",
        "\n",
        "torch.save(agent.state_dict(), 'weights.pt') \n",
        "agent.env.close()\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "F2MbdWmPFoEK",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 287
        },
        "outputId": "70c9997f-dead-4573-9cc8-f2cfde642a38"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.legend.Legend at 0x7fcddf6a7cd0>"
            ]
          },
          "metadata": {},
          "execution_count": 4
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAD8CAYAAACfF6SlAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOydd5hURdaH37rdPZGBgSEnh4xkFJAgioBiRl0jugrr6iq6fu6uAXNWXHXNuibAHNaIgCIoqKDknONIhiFNYGZ6OtT3x+3u6XA7d880M/U+Dw/ddevWrTvd/atzT506JaSUKBQKhaJuodV0BxQKhUJR/SjxVygUijqIEn+FQqGogyjxVygUijqIEn+FQqGogyjxVygUijpI3OIvhMgQQiwSQqwUQqwVQjziKm8nhFgohNgihPhUCJHmKk93vd/iOp4fbx8UCoVCER2JsPytwHApZW+gD3C2EGIg8DTwvJSyI3AEuN5V/3rgiKv8eVc9hUKhUFQjcYu/1Cl1vbW4/klgOPC5q/xd4CLX69Gu97iOjxBCiHj7oVAoFIrIMSeiESGECVgKdAReBbYCR6WUdleVXUAr1+tWwE4AKaVdCFEE5AEH/dq8EbgRIDs7++SuXbsmoqsKhaKOsHp3EQA9WzUwLG9eP4N9xRU0qZdO8wYZOJwSp4RdR8ootdrJz8smJ8NXIg+WWtlbVAFA+8bZZKf7HndKydo9xWSlmejQpB4AlXYndqeTrLTwcuvuW8em9dhXVEF+XjbxmMZLly49KKVsYnQsIeIvpXQAfYQQucBXQNxKLaV8E3gToF+/fnLJkiXxNqlQKOoQ+ROmA7Bk4nmG5XeO6sIzMzfyt9Pbc885J9L1ge+osDk5t1Njft18kDfG9WdYl6Y+574zbzuPTVsHwKQbBjKoQx5SSqat2suo7s0ptzno/cgP5KSbWfLIKJ/rrfPrR6g+t2tVn2O7i5l86xB6tc6N+W8ghPgj2LGERvtIKY8Cc4BBQK4Qwj24tAZ2u17vBtq4OmYGGgCHEtkPhUKhCIfT6ZvXrMLm9Hnv7Y2ev+UgP28qxCgX2k8bDvD3j5fz4o+b4rLSfa5N8j3hiYj2aeKy+BFCZAJnAuvRB4FLXdWuA75xvZ7qeo/r+E9SZZdTKBTVjM1pLDtuNfKW36vfXsh1kxb51kOv+NJPWwB4dc5W3vx5m+uYu634pG3K/AKW7zgSVxvBSITbpwXwrsvvrwGfSSmnCSHWAZ8IIR4HlgPvuOq/A7wvhNgCHAauTEAfFApFLUG6/Oaz1u3nzG7N6OHns4+lPaOYEodTt/QXbDvsU36w1AoQsRW/cudRz+tX5ugDQanVzs+bCiksscbSZc+1v1y+my+X76YgApdRtMQt/lLKVUBfg/JtwACD8grgsniva7PZ2LVrFxUVFfE2dVySkZFB69atsVgsNd0VhSKhfLNiD7d/ugLQxXTrk+cGrVthczB28iIeubAHXZrnGNax2p1kWEwB5a/O2Qr4ijfAhn0lQASulzBG/XWTFvH34R1DVwpCdYQ/JmTCtybYtWsXOTk55OfnG47qtRkpJYcOHWLXrl20a9euprujUCSUTftLPK8dQVwzbhYXHGbBtsM8Nm0dH/z1FMM6lY4q8Z+z8UBcfYvWixNN/XD3mmiO2/QOFRUV5OXl1TnhB30iKi8vr84+9ShqN6/N3ZrQ9qxeE7njJi82rFNUbou63URLdb/HZyW4xdAct+IP1Enhd1OX712RHCZ8sYor3/y9prsRFZFY1v2fmM2qXUdD1un9yA8BZUY/MRml5LvnAPypsDm49aNl7D5a7ik7UuY1AFXD7/u4Fn+FQpE4Plm8M2DyMxX4ZsVuhkz8KSA0Mxre+nV71OcYya/3YBOtC2j+Fn0dq9MpufWj5UxbtZcnpq8zrOuejE4mSvwVCkVSOHyskoenrqXSrgvZ3I0HsDuiF7W7Pl/F7qPlVBqcG8xAXrjNd+nQr5sLPQuoIsVI26XP6+jU/+q3F7JuTzFfr9jN7PX7AZixeh/5E6YzaZ7v4OT+myUTJf4pgt1uD19JoTiOeHz6Oqb8VsB3a/aycNshxk5ezH9mbUroNbyt7x2HysifMJ0F2w7x7A8bfeodLYvepz/XYHJ44ncbPK/X7C4GoEer+hG3WVZpNwz/fPOXbT7vlfinOMeOHeO8886jd+/e9OjRg08//ZTFixczePBgevfuzYABAygpKaGiooJx48bRs2dP+vbty5w5cwCYMmUKF154IcOHD2fEiBEcO3aMv/zlLwwYMIC+ffvyzTffhOmBQpG6lFToBo2UcKxSf71ub3HU7URqX9/4vp4C5so3F7C4IP6FUeFcRU9/rw8E6ebAMNJgZFhM2A3cV1a7w+d9qTX5xuBxG+rpzSPfrmXdnui/VKHo1rI+D13QPWSd77//npYtWzJ9uv44WVRURN++ffn000/p378/xcXFZGZm8uKLLyKEYPXq1WzYsIGzzjqLTZt0C2jZsmWsWrWKRo0ace+99zJ8+HAmTZrE0aNHGTBgACNHjiQ7Ozuh96ZQVAez1umuDYkk25XU7FgSRc0dn1+dTF25J6pVvJlpJsOQTqufpX+wtDLuvoVDWf5x0LNnT2bNmsXdd9/Nr7/+yo4dO2jRogX9+/cHoH79+pjNZubNm8c111wDQNeuXTnhhBM84n/mmWfSqFEjAH744QcmTpxInz59GDZsGBUVFezYsaNmbk6hSCDu7Jfup4FEk8zgmFDpFW77eHlUbWlCGM57+It/dVArLP9wFnqy6Ny5M8uWLWPGjBncf//9DB8+POo2vK16KSVffPEFXbp0SWQ3FYoaxy3OyRK5XzcfDF8pBka/Oj9gBXA8SCkN3T7VvcALlOUfF3v27CErK4trrrmGO++8k4ULF7J3714WL9YXkpSUlGC32xk6dCgffvghAJs2bWLHjh2GAj9q1Chefvllz2Pk8uXRWRUKhSJ+rjHNoiBjDI+YJ0ck/NGsuXnuh00JX8QWK7XC8q8pVq9ezZ133ommaVgsFl5//XWklPz973+nvLyczMxMZs+ezfjx47n55pvp2bMnZrOZKVOmkJ6eHtDeAw88wO23306vXr1wOp20a9eOadOm1cCdKRSJQ8roY+J9G0hYVyK62N3mTwC4zjyLDxxnslm2Dn1GBDfXU2xjq2zJ9NV7o+pNN1FAK3EQSMHEbnWZUaNGMWrUqIDyBQsWBJRNnjw5oGzs2LGMHTvW8z4zM5M33ngjoX1UKEKxaX8JbRtlGSY+SzWqI/F7PcrJEeUscJ7IQG09Q7Q1bHaEFv9lO0I/HfQSW5ma/gDbnM0ZWfkszogcLpJ3LM8ywrScbbIl8FDkNxEhyu2jUNRRSipsnPX8L/zDlUEzmRjlyI8Y10nRLqrypzFFXGGaE7LOaNNvAMx36POIOZTFdU2Avpqe4qG9to/OYldE5+SLfYww6W7f/zR+NO4+GKEsf4WijuLeuWrR9upL6RBKvssq7cYx8wmy+F9Je4mB2nrO0RaxXzZkgv2vSD/7d4Cmx+7Pc/ZkvJxKPVFu1FRU1KOqjebiEBtkW8N6rcUBbjBN5zpzVYK3sZV3UWYJ/eQRK8ryVygUSScSq73bgzO56/NVwduIYRD40CvNcyv0iKBhppVcYZ7L6drKgPoHZQNKZCbLZSeOUo+/mafzpPmt6C8M5FJCP7HBM4A4peAUbYNh3ZcsLzMv/XYf4X/PfiZznX3QkqTSyvJXKBRJxXvCN5zb54tlkblFIqV9k+ALJNuLfcx1vc7AihkHeaKIQ1JP1/C07UpeSHuNMeY53Gu/Iarr1qeUX9Jvp74oZ7ajL4dlPcrIoIkoMqw/SNMTvM129OVl+8VUkOaZaE7Wfr5K/BWKOkpNZAWPx4Pzn1mbeOD8blG1Z9ICb9IqzZhx0FBUrQiekvZvBmrrAVjq7ATA185TOdWxhpHa0oj72Ihifk7/Bzle7qKRpuVsdLbGjokGlBqcJcmhjDfs5/GU/eqAo44kzXQrt49CoUg6iZCvd+aFzrVjhMlrhBNC70UJWRwhh4ZUib9b+AFKZBb/PLMzAIdlDmlEvip5QfotHuF3TxoDPGW/iqOyHrkiUPzrU0aGsHFA5hq2Ge8m8MFQ4q9Q1HGqI4w+nICFOu49X3Dbx8vZV1S1g124hxdvyz8DPV/OmMr7OCJzaChKaN9Ydwvtkw1Z7cznY/sZbO3/ECNPbAaAFQvphM+zU48yHjK/S5rQE7RdYH2cq233cWLFJEZbH2XURddylGxyORZwbhOhh4oWyoY+5Zed3JouzXKStvpXib9CUUepLq9PJNIVqXE7deUenphRZaWHO829+jadShqLYl60X0LD/N7kiSLOMy1iaMc8BE7yKGZb/VPofMMkrr9wBBaTfp5VWjAJiQlHqMswUlvGOPNMAGY6+rFatgegnAxE6340yLRQFMTyb+oWfxr4lGenm2nWIINkZX5Q4h8HBQUFdO3albFjx9K5c2euvvpqZs+ezZAhQ+jUqROLFi0Kmqa5oKCAoUOHctJJJ3HSSSfx2296fPHcuXMZNmwYl156KV27duXqq69O2mOfQlFdhPsGJ+0bLiEvO42ztUUArHS2RwiQrqEvx1nM3eZPsAgHo0/ty8kn6EkW3U8MViwApBN6P4BeWlU+/j0yz78LCKCIbJfPXzJ2cL7neBN08fd3+5g0wZMX9+Dlq/pGdcuRUjsmfL+bAPtWJ7bN5j3hnIlhq23ZsoX//e9/TJo0if79+/PRRx8xb948pk6dypNPPkm3bt0M0zQ3bdqUWbNmkZGRwebNm7nqqqtYskTPR758+XLWrl1Ly5YtGTJkCPPnz+fUU09N7P0pFC6SbVwIwlv2yepDmlnjl7vOYMpj72GTJuY4+zBICJ6yj+FZyxvcsercKhVsVuWjt5h0u7hK/CspI8PwGvliL38xf+95P9lxdkAdIeCorEeacJCFlVuHd2TKbwVAleV/wM/tY9IErRtmxXTfkaAs/zhp164dPXv2RNM0unfvzogRIxBC0LNnTwoKCoKmabbZbNxwww307NmTyy67jHXrqvbyHDBgAK1bt0bTNPr06UNBQUHN3aBCEQPeYl5Tz6310s1kppnITjfTR2ylgjQkGkLgCed082/bFZA/1PPe7Hb7kAYEt/y7tajPUE03PJc4O5Nf8RE7ZDPfSq6/xVHqAZBLKRkHVjDe9DVp2GglDlIhLZSQCcDZ3ZsDxpFKiaR2WP4RWOjJwjtBm6ZpnveapmG32zGZTIZpmh9++GGaNWvGypUrcTqdZGRkGLZpMpnUFo+K445AQz7MhG8S+tC8ges3dewgQ0xrsUvd1hUIVjg7eOr1qHibHu1b+8S+uqOErNJl+QubYSf75Tck54CeAmJM5X0heiM8A85vGbfBe3CXBYabVtBCHOIwObhnYR6/uAeFpVauG5Qfw11HjrL8k0ywNM1FRUW0aNECTdN4//33cThCTygpFInGPRmaDOH1bzO82ycJnXBTtBOAF+2XAFDpcHKE+pxqfYGX+s/i+7vPY/LYAb7nuPcfCOPzr2+ycqflM71dV11/JPq4ssB5YsCxftomWolD7Mzo6ilrXC+dL24eXDV4JQkl/knmgQcewGaz0atXL7p3784DDzwAwPjx43n33Xfp3bs3GzZsUFs1Kmo14Sd8Q4R6xjgweOz4L/8GQEH9fkDVhjK7ZFNslga0bphFZprJ79zIJnw7lkU+11hKFqdaX2BNx5vgzq2MsD7jOda+S8+I20kUcbt9hBBtgPeAZuif8ZtSyheFEI2AT4F8oAC4XEp5ROjmxovAuUAZMFZKuSzeftQE+fn5rFmzxvN+ypQphseM0jR36tSJVauq8pg8/fTTAAwbNoxhw4Z5yl955ZUE91qhSD4+Pv8I1Dupln/xHgD6DxrGtzO2+RwKtxGL94SvEVkmfVD4pcU4CLIGTcqqgWiXbMqaziPokd2YrbIVl1gf5vO0R7C3OIk/DzyBs7o3M24kCSTC8rcD/5JSdgMGArcIIboBE4AfpZSdgB9d7wHOATq5/t0IvJ6APigUiihJ5nRitG6fpKKZoN/1OE26G8X7voP9DTzbTnr7/A3oWzAJgJWNzw841qOV7uP3f6rxHm+Wyc50sr6HvcuFPHZRD4Z2ahLmZhJH3OIvpdzrttyllCXAeqAVMBp411XtXeAi1+vRwHtSZwGQK4RoEW8/FApF6uAv9okM5fQW7LDtOmxQcRTqNY1qu0U3xeju2HGm7zFyXjUp0aP0Kkz1A449eXGVK8f72m6X0tvX6m4oB6Ya2UwnodE+Qoh8oC+wEGgmpXTvWbYP3S0E+sCw0+u0Xa6y6PY3Q//gY/lAawNq4ZciUaTCVylpfSg/ov+flYcwiKkIJh/u4k2yNfMc3RlpWs7J9k0slVVRe63FAQCKZRZWcz2gEIDv/m8omhBUem1Wb3SZkd2aUS/dTKnVTpq5+qdfE3ZFIUQ94AvgdillsfcxqStVVB+vEOJGIcQSIcSSwsLCgOMZGRkcOnSoToqglJJDhw75hIcqFKmEv6sjrgnfEOeFNf7KDun/Z+V56kZiL1a1K7jDdhMAnTXfdNMPmt8HYKFfFE/TnHS6NM8J0XjVyw5N9dj/9BoQ/4RY/kIIC7rwfyil/NJVvF8I0UJKudfl1jngKt8NtPE6vbWrzAcp5ZvAmwD9+vUL+Pxbt27Nrl27MBoY6gIZGRm0bp2cHX4Uinjxtskk4S37WG24sMbfMX0DF7LycKfVifZaR9CFvDG+ufjdK3OftI9hhFe5/+IsKYMPOJPH9mf93uLj0+3jit55B1gvpfyP16GpwHXARNf/33iV3yqE+AQ4BSjycg9FjMVioV27dnH1XaGoyxyvz8yR9lsIqiz/7MZR7V/gXdW9yvda8w+87LjEU67h5I+8U9m+u4VP25oW+ITh89qr7UbZaQzp2DjyjiWQRFj+Q4A/A6uFEO6doO9FF/3PhBDXA38Al7uOzUAP89yCHuo5LgF9UCgUMZIM12nAhG8NrPAFoMzL8neFa0bm9vF9f1jWo1j6rsXJpgJThv5U0DI301OuiUDLPxWJW/yllPMIHjE1wr/A5f+/Jd7rKhSK44gaSOwmJXDM2+evOxh8Qz0jexz41dmLXmKrT1mWsNK8aWPeHHwyI09sxiPf6pE/JoPRxfs6qRKkolb4KhR1lGQGS/hY+rIGUzofK4SMXDBZolrX4D8oHJPptNP2Y/Ha1SuLCkRaNmd1b+5x9UDgU4PUG0w5lPgrFIqEE+24koxxSPf5H4TsxlXvjepEgHuR1x3mT10lkiyskBaYlsXk5/P3H2RTZRxQ4q9Q1FGS6YpO5ArfuMSzcBNkN3Gd5xblSJz+vm8/sQ8HoL3LdZSGHYtwGIu/O6TU29Xj3XSKqL8Sf4WijlMte/iGdfrH2m5wTNIOB9ZCI1f65miiffzqLpZdWeTswpmmZdSjjEys+gFLoPi7z21WX0/NfkHvlpFfuBpR4q9QKBJOtPMJYQeHGMiWro3em/cAqrQ/VsN7s1NfV/NR2hNk42rbwPJ3P1nk1Utn3aOjGD+sg296B2X5KxSKmiSZIYjS57VM2iKvUDqayTH9RZq+ijaaKBujmg/axwLQTfzBDebprrZDp2LPSjMjRKQxRdWLEn+Foq6ThEEgMM4/OV0IdV6W1HfYIl2PxTcS4GhE2YGJXx09MAsn48wz9ULXwBINqTIUKPFXKOooyXC1xEooN1GsvezgcCXYb5gPgBaF2gV7Suio7fEtSItsg/VUcfV4o8RfoVAkniSmdI6Upk7X6t6m3YDoLO5gNf9eeatvgSkt6n6lykCgxF+hqOMkZw9f75284nP7xKqV9WWx7pYx6wLtFt14xHeNrMontsLZAZoG7st7vJDQfP4KheI4IpkTvkHaDvYEEOrBINZu1pclkNUoZJ2g+fyDlFeQzueO09jqbMnrjgspSA+RujnFUeKvUCiST9gw/8SPRPVlsY/4RxftE7yuO79/NHgPbiq3j0KhqFGqdYWvqySo8CW8M5IB9qVVO3nhHeefGuJb0yjxVygUCcffvVPd872NKQ4oiyqff4LHB+/bT5WhR4m/QlHHSUo65STXD0dj4dp168xHPWXK4vdFib9CUUdJ6grfgFDP2PsSSz/z3OJfr6mnzGtbXq+y6h8QUsTlr8RfoVAkF0l8G7jHgme/XVdGT4gut08yBTpVnkCU+CsUdZxkx/n7lMcQ6hkLjYXL5+8t/qmhuSmDEn+Foo6S1PQO1bXCN0izjUURlZgho4FXaWJCPeMlVQYhJf4KhSLhBIZ66gTzsUczNEQinnkUUyRyfSq7d1qsiYxGNZHeIhxK/BWKOkp16ZH3dYK7fRKb0zlPFFOk1fetKiJX/+T6/FMDJf4KRR0nGYNAIqN9Qp9oXNxYFHFE5PqURbeBe+1Hib9CoUg4gfMJYVb4Jpg8UcxRf/H3GP4yoKw6UT5/hUJRoyQ1vUOUln/M1zG8C0ljiijSGviURrfCN0UUOoko8VcoFNVGokM9jc5rRAkZwhZo+Scgn3+s+HYzNQYWJf4KRR0lmREoAXv4hq0feV/CdfsUbT0A200n+B5wu31qOPAmVR4qlPgrFHWcZMT7B0vsFjTUM4GWvzuvzzZTe59y4z18jfuTKgKdTJT4KxSKhBP1Zi6xXsegrBElABQL341WatSP753Pv+Z64UNCxF8IMUkIcUAIscarrJEQYpYQYrPr/4auciGEeEkIsUUIsUoIcVIi+qBQKKKjOt0f4Z4uonFBhdLwdCoZa57JMZmOQ/juVeU+LZIrqQnfyJkCnO1XNgH4UUrZCfjR9R7gHKCT69+NwOsJ6oNCoYiBZA8CUoZ3+8Tetm/nz9cW0FCUki2sAXVTRc9TZWBJiPhLKX8BDvsVjwbedb1+F7jIq/w9qbMAyBVCtEhEPxQKReQUlduS1nZNuX2aCX3nrjXO/IC6bv9+KqZaqAmS6fNvJqXc63q9D2jmet0K2OlVb5erzAchxI1CiCVCiCWFhYVJ7KZCUXdwOiVv/7qN8koH5788L2nX8XfzhI32iUCPu4kCThR/0LpiY9A6jUURVmnhwsrHA44Z5fapkUVe1X9JQ6plA3cppRRCRDXcSinfBN4E6NevnxqqFYoEMG31Xh6fvp69RRWesqSkdI46q2fo4xdq83kp7VX9TQFgGw2WzIDrdBE7OSBzcRrZtTU631uzq4qNSKblv9/tznH9f8BVvhto41WvtatMoVAkmKIyG1e9uYDdR8sBKK+0A1CcRJdPLIQbGy4wLfAtOLjZfaZ3KwwxraWNZuwpqHL7eJfVXZIp/lOB61yvrwO+8Sq/1hX1MxAo8nIPKRSKBNL70R/4fdshnp2pu0rcAjh3k69Azl63P6HXlUFe/7f8Dni4AZT5TxGGpqEo8S0oPxJQpz7HAFji7GzYRqpY3KnSj0SFen4M/A50EULsEkJcD0wEzhRCbAZGut4DzAC2AVuAt4DxieiDQnE8snDbIR6fti6iug6nZJ+XuyYeCkt8o2H++t6ShLTrxmiR12naSro6t+gF30/gQEkFy3boIh7OKdSUI2xwejkMjhR42nXj3r3rfftIIDCqpiY1NxXnmBMV7XOVlLKFlNIipWwtpXxHSnlISjlCStlJSjlSSnnYVVdKKW+RUnaQUvaUUib2W6dQHEdc8eYC3p63PaK6z/2wkYFP/egZAIZM/ImbP1ga3QXjVMBjVjv3fbWa4ooqt9GBkgr6PT6bDfuKPWXeWvfA12uYunIP/zB/QSlZoJmh/AgXvDyPS177DadTcrAkMDRzuLaMgowxvGZ5gVbiIL86e/KVY4h+cM/ygOs0Qr/+YepjhHswqGkdVnv4KhSKqJi7UXfVHCzVhXL30XK+W7PPp46UEqczSnmLovoni3fy4cIdvDpni6fs542FHCy18uYv24Ke9+uGPfTVtvCZOBvanQ6bf2B/sT6IPTptHWPeXuhTX+BkUtqzAJxrWoRJSPbJRvzDdgt70/Kh7JDeda++t9P0v8UhGUz8I7/PRJMqrh5vlPgrFClAJLHnkWj0+A+X0f7eGYbHHE6Jw2BgqHQ4g7ZXWGLl2kmLOFCiC3WaSVex0gq7p05Wmh40WF7p0PspJSOe+9mnnYaulAtbK3M5uncrAGdoKwCY8ltBwHVzKAPge0d/T9n/HKcBYBcWWD814JyLtPkA7JcNDe/FMLdPNYmyz8ebIgOBEn+FIgUwEmV/3ANEWaWD/AnTDev4Pwl4c+6Lv3LPl6vDXueF2ZtYt0d3obw+dyu/bCrkq2V6QF6mS+g/XLjD05+NLndPmUv8jW4lz+WPPyTr83jlGAByKQ3ah0auCd6Zjn6esmLqAWDVsgCY/Mmn3P/RXM9xgcQhBYfwzePvOe7J71Czjp8U0X4l/gpFdXK0rBKr3RFQ7ohCkC5/4/ewdVbsPOozOfzLpkI27i8JcUYVL8zezJ9e/w0pJX8c0iNoWuRmApCVZvLU237wGIUlVl76SXcBlbnCSJ0G99LIJf6HZQ4L7B0BGGJaG6QHknM13Q20n4ZMcwxkubOj5+i7rR7EKUyM23AjP6TfRTqVAGSKSuY7ewS9L82l/tH8rWszSvwVigSxcudRDh+rDFmnz6OzGDtpcUC5M7jnJSYuenU+f3r9N8/7Q2H65U+5zUG7e2bw4wZ9eU6GWUNKyfOzNnnqzFi9l/5PzPa8tzkkNoczICLpfO13Pkx7Su8H9elyQhsWObtwjraQbMoDrn2h9ht3WT4DYLWzPbfabuPiykc9x99fY2V+h38C0EQU0038QUHGGPpqWzhMTkB7btLMutzZHdUv/j5rC1JkAkCJv6JOUVZp55+frQgr0rEw+tX5XPLa/KDH3W6S37cdCjgWiTUarcHqXtiVCG58fym3f7qCzQeqXDXzthz0qWNzOOl033cM/fccT1kaNp6wvON5v1M2pV6mhadsY8gWVtZmXM9J2b7t9NAKPK9LyDTsz+rWVzKm8l4A7jJ/6ik/IqvE319i013iX2l3etVJDSGuCZT4K+oUnyzayZfLdvPSj5vDV/bD4ZTc8+UqthUG91UXHCoLeuytX4NHw7h97G7sDqePSEFyNohGIBUAACAASURBVF2Jhm9W7PF57/+0stbvHizYedvyLA1EGX+rvJ0OFe9TiQWbw8ly2ZHJ9lEAPG6e5HNec+G9AMxYnP/9/UYKnM0BGGSqWifhHenj/9dKM+kuq1AT3NVBqgw3SvwVdQq3P1qL4dF73Z5iPl60k9s+WR62boXNQYXN17c/dWWVeO4rqmDuxgOe95e/8Ts/bdBX2S7bcYSO931H5/u/i6hf+ROme8I/42G0No+bTYFRNMEI97QyQFvPaabVLHR2ZaazPw508Z2xeh8geMR+HSuc7elmXcHp2krPeS2E/mR0RNYL2f4eGjPP0d2nbLNsHbS+2+1jq2HxTxWqJbGbQpEquPVKi1D7j5ZVkpuVZthGKHo9/AMNsy0svHekp+xwaZWraeBTPwacc9fnq+jdOtfjZwcY/txcpv99KJleE61GbNxXwlJrYMqDaHgx7TUA0kUlr9lHU4klZP1wEUodhD7Y/aNyPEEtePuVfJT2JD3FNraJ5vQVW2gr9Pu/pvKesH3OEPpis2KZydP2q/jR2ddzzP+KFleYqo/bpyayeqaI6a/EX1GncFurWgTqP23VHm79aDlf3zKEPm1yPT/aSMS/0uFkf7GVzxbv5PL+eloCWxixPFha6SP8ANsKj7FubzEnn9Aw5HX/t2QnC7ZFly8nGLebv+R285d87RjMdmcLWopDPGO/goN+IZRWe2gL+lGLvp1HqEnY35w9KJUZNBIlzMy4lyypz1NMcwxkrWwXtq+VUpew22238JPTd1PAkd2a+bxPM/D5VxepGF+k3D6KWk1xhc0n+sTt9onE+pq3WZ+I9PfHGxFskdZdX6zyqhP+mtG07c3XK/b4pFyIlnYiMLfiRabf+IflC64wz2W8+RufYyYcntDOcDx+2YCQx4/IHNqJvR7hL5GZTHUMiqjtUteEsNPgyeL2EZ183rvFv0Fm6CeaZJMqk8zK8lfUKo5Z7fy+9ZDH6jvzPz+zv9hKwcTzgCoBNkXx7O1f1UiKQxn1xRU21u0pjtkvv/SPI1TanWGtR/ciq0jRcPKuZSInaZs92x7ebxuHA42nXBE6B2QuTcVRTtdW0pgiKjHxcdoTdBB7uOzwQ0B7w7Yt6APDG/bz6BBGbA+TQydNX0R2W+WtTHUOjvgeHrSN5bDM4TeD+H7/p7t0s4l//6kXgzrk+UQkVTfK7aNQJIF7vlzN1JV7mP3P06mfaWZ/sa/guv3U5TYHb/y8lRuGtvcRicvf+J3++Q25c1TXAEu9yu3je2DN7iLqpQf/Kf3jkxUB7pxoeOq7DTGfG4qbTFMZalrjU/aBQ5+j+KtpBh20vZxmfZ7bzV9wk3kaSzJu9qn7bfr9fO44jc8dpzFMW8EB2ZBJjnPoKbYxxqTPaeySTTjRHNrB0FXsIF3og8XqCFw93uwjjwn2GyOu73bBVTepuHWkEn/FccfOw2U8+M0aXr36JE9emTkbD9AuL5s/DuuhliP/87PhuW63z+T5BQC0bZTFOT2rtpBetP0wi7Yf1sXfZWuHM9TCbYe4/eCxcLdUTUhOERsoJovv0gMnU0dYn8F9tyMqn/OU2/xk4t+2ywG4y/IZl5p+4VLTL55jg7S1nGla5nl/ROZ43C0AedlpAQvOHrFfxwPm93nAPo7t8vjZzvvKGAeSFDH8lfgrjj8mfr+BORsLmb3+ABf2bgnAuMn6qtmT2uaGPNc/42W5zdhVsrjgsMfynzR/O386OXgIYTjSLaEjdZKJwMl401RGm+bTWTPeMO8C6+OslsbuG4CVzg4AXGR9lBVST7Og4WSkaRkZ2OgidmASEqcUHuHf6mzBCtmROc4+3OT1VGQ2BUrfR44RfOQYEfM91gSagIl/6hVx/VRZ1euNEn/FcYdbwI389mYttIvB3zfv/d770fyy//5OvxP07JCb9pcyef52Tu3YJKb+7j4SfOFXIhhn+o4eWgHP2y+ltSjkdvMXvG8/k5O0zdSjnCvMcwPOedl+Ec/ZL3O9Cy1Ms50n06ViClaqQl6daFziSrlQjzJKyaQe5fzT/DlfO4awSnbw1O3YtCpe32IK/vmMPLEps9fH7h5LZXzcPikyDijxT0HKKu2kmTTMIX4odZmqhVr6+zW7izzHwmh/QNIx94+y1GoPSM3gnQKipMIe8wrb4orIomJiYbC2hocs7wO6y6Wla3XswLT1PvXut41jqmMQGdgYoq3hK+ep+KtQ//yGLC4wXivgLfz+lJLl+f9R+7UBxzO8nnxCLa57/ZqT6XRfZAvbjjfa5mXVdBcCUOqSgnR7cCa3fLQsfMU6RqXdyTMzN1Bq1cXU/Sjt7XOP1vJ3jwX/+mwFm/b7pm2we1WWsqqu+/+nZqwPmlq5OhihLeWjtCcBeMT2Z4/w+3OX7QY+cJxJMfU4QEO+cg7FyPxsmpORzO6G5KoBbUM+FSSL6nLHdG1elXZChXoqQjJzbWI31D5e+WDBHzSul8bZPVrwv6U7eXXOVs8xkyYCfPimIIu3Ku1O0sxagOXvlJKicpvh39t/Bav7vUQy/Nm5bAsykXvPl6sMy2OlndjLBdrvTHGcRTH1qE8paTh40Kxb/A/aruM9x1lYSaOj2M1Pzr7Uo5yZzn7kUE4x2RFdZ3DHPKavDoz3TyRG6Z4Bxg/rYFj+1fjBXPyanp107OB8Fmw7xIZ9kaWmTiaxDBpmTWB3ShXqqVCEQ0rJ/V/roYgFE88LEGOTBqV+C43MQcR/yNM/sfi+kWzyy2nvkJLej/xgeI5/Dhi3cPk/Ifjz8aKdIY9HwiBtLa9bXmCZsxPDTfqOV4NNa5nmGMjjlsmeevfYrudj12Sp0aRppMIPumvLn6w0U9TrB0JRYdP/pie1zWXZjqOe8mCC2LdtQ4Z1acLcjYU0yUmnb9uGKSH+sWA2CZ+nyZpGuX0UKYv3D2XOxgNYbb5iLITgiWm+vu1gaRsKS6xMX7XXsw+um1BL/b2v/8qcLSEXcsXCw+Yp/Jr2f4wx/Ugzqlw2Gk7esPyHXHHMI/wAA7X1PsIP8KVjaML6Y5Sr56lLegaUPXFx8A1TgnFKu0YAno1szuvV0ud4qLmALs319BBCpM4Cqd6tjXcLC4XbrZUit6DEX5G6eIvRuMmLeWKGr9CbhGDlrqMBZcH4ZPGOkNfwx39gCOayiIbGFNGEI7xueZ6x5h9ooxXypOUdFmbcSkHGGEZoS9mWcQ31RTkzHAMok+kA/K3ydmxSnzi90PoYV1gfYJj1uZATsdFitMlJhya+mTXrZ5i5pG/wsNcXruhjWP7BX09h7SOjPAP4mAFtueWMKldPKPF3Dxwnt22YsMVSd5zVGYAzukQfwfXV+MFMHhc6ZYURaSkWwKHcPoqUJVzqXU2IgElCS4jVpIUlgekVQom/f0pm//mFWJiffhvpIngOnnfS9MVVJTKT8bbbSaeS07WV/ODsRyfr+3FfPxoW3DOC5g18J4G/vmVI0Ayj8+4+g9YNs7j90xUBxywmDYtJ4+MbB/Llsl1kWDQuOam1Zw4nlEU/vGszVj18FvUzLHyxbFfsN+TFrcM7cevwTuErGtC3rfEG8eHwWP4p8viSWkORQuFi474SHpu2LmSdzQdKWO0V5gmQaYnuKx3KB+t/LN69X5tx2Ef4Nzpbk1/xIe0qPqBDRZWwr3e2pa/1DUAPsfzB2Z9EOAuCTaq6uX6ob2oFf+Hf8NjZtPd7EhjUPg/QffitG4YPZzz5hIY8cXFPhBA+TxXh9LB+hp4fyN8euGpAW5rmpHve/37PcP5vRKew9xorb13bL3ylIFjMqSH6bpTlr0hJHvl2Lb9t9d3u0B0t4Wb2+sAIncwoV9OGy0nvzcNTg204HhkXmPSN17c5m/OlYyhTHKMAgUTgAPIrPsKEAwcayfAM3zSsA6/N3Rr0eKj8ROAbr++maX1deFtFIPyhiHRzHX+3z1OX9OSeL+HjRTs87fzjzM5x9SUUgzvkxXxuleWfqN7Eh7L8axErdh5lwherUjKJVDRsKyz12SvWjX9qACMfdUaYTU/8iSb6IlyUTyjM2Bln/p6Fzq4Mr/wPrzgu9iyO8kbf7So2dVh6/0huDmHxZiUhzYR78Ix0c5xgRCr+3vMuw7s2DTiebGGNp/00NeGrSBbXvL2QTxbvpMSavBWl1cHw53429M9b/BZwGQl3qEk1owlbh/9GtAnEhIMcykinksmWf9NKHGKy/ey42+3UNHB7w/9c3pu8euncfXbX4P3xUuiXr+obtF40uEM3M8xVA0vDrOjz5bu7dnm/0DmU3OP9i1f2YdLY/gbtpIq0BmKU16gmqTHxF0KcLYTYKITYIoSYUFP9qI1Ux1fM7nDy7MyNFJXHvoFItARY/gbi//JPW4Keb2S5JyLuWsPJALEekAzRVvOm5Tm6ih1szfgzqzP+ysaMsQw1rcEmTcxynhzXtW4b0cnQ+rzkpPCJ57wnGkec2JQxp7QNqHP/eSdG1R+32yfDa65l7h1nRNWGd98Gd2gcsl7VZjxV9+L990i2+MezOtcdiZYq41ONiL8QwgS8CpwDdAOuEkJ0q4m+KGLjuzX7eGXOFp7yCr/MnzCdV+cEF994MflZ/omw2g+VVoavFITuYjuDtLV8k3Y/n6U/xqPmKXyY9hRnmZbyfXqgPXOidbJnE/NI+PTGgbTwm3T9q9+kbKyYNc1Qxv46tD1L7h/JigfP9JS1aZQZtJ02Ll+/93xAgzgs/4v6tuKr8cE3czm9kx6a2aWZ8daQ8bqfkkmqRPm4qakJ3wHAFinlNgAhxCfAaCB0eEct4MixSqb8VsD/jegU0T6yqYo7DNN/H9dnZm7kljM6JuWa/n8uI59/MM7QlrPU2TlgxevnS41DB7OooJ3Yx0bZGomgvdjLZqlb18O0FfzL/Bk9tQKfc641z/J5XybTucP2NwplLmWks2Xi6IhzAZ3fqwWntM+jTcMs9nptQ5llMSUkN4zFJIJaoI3rpfu8n37bUIqDPOHlZOgS4h8Z9O2tp7LzSBklEW4t6S2MjbKDr124rF9rzurejNws4zrJzpsTj367v7+pMiVXU+LfCvBeA78LOMW7ghDiRuBGgLZtAx9Pj1ce+GYN01btpU/bXM7oEjhhVZd485etzN1YyEc3DIyovv9vJtw6ADd9xBYmpz3DT44+3GK7jTct/yFPlHBJ5cNUkG5whmRS2jMM1HwXlb1gv4RFzq5MSfu3T/kGZxu+dgzhn+b/8Yj9On509OXFtFd5wf4nfnd2j6iP3kz7+6n0aKWvIPXPVWQ2aeR6WdavjOlLVphJ7vN7tWDaKt+cPdFYofUzLJ5QSzczbz8NkwbtGtcj3awFuJ16tm5AzyhWwXrfZigBF0IEFX5IzEK8ZOH+LFMlw0PKhnpKKd8E3gTo169fzH+uKfO3075JPU7rHFsu9kTjXjhkC5FWINFcP2UxzRtk8MTFgUv1E0UsEUZPzohue0L/SeCtheF3yGpEMf9n/gKA4aYVfCMe8Gxqcpa21HC/2AFiQ4DwA9xu/tLnfZHMorf1bc/7/zou9Ly+ovLBsH2LBKNEdePP6MjC7YsAON8vTYIRL17Zl2cv6x2yzulR/j7cKRcALusX/9aIifLVh1rkV9O4B9xUGaBqSvx3A97fmNausoTz8Le6J8m9gXfNo38BqvPjd+8fm1zxT1rTRldjkLaOJc4uAVsM+vNx2uN00XZ5NiL33s2qgQicADbh4BmLvsCqf8VrZIty7jD/j7nO3lxv+o4m4ijfOgYxx9mHNc7E+N/98RYHI9dgtEJt0gQmLfjTwaOju3PtoPyo2kw0idD+609tF3atQk3i/ijruvgvBjoJIdqhi/6VwJga6ku1IqrB71cTX63th6pnn9omHOVJy9ucaVrGs7bLeMVxccj6XTTdp/+AbRwW7LyS9jJXVd7Hx2lPMEDbwPuOs3zqn6Ut4QRNHywLyaVQ5nKr7TYAPnecnoQ7CsT7u+GdpXTdo6OiaueNP58cMGHsjdu9kgpapAWJ3omGdo0jz2BaE7jvMRX+3lBD0T5SSjtwKzATWA98JqWMb/nkcULVbznx3wC366U6vlz+m5KP/yD05jPFFTb+MmUxB4orgtZZXHCYs5433nhdR7I4Y7xnn9g7LP/jctMccimhi/BN2paBlRtM0wCYaLuSmc7+THMOIr/iI48f/gLTgoAr2F3ROF8kIFtmX6/9hFuGEGHQ0x648bYMW+ZWnRftZiejujenV+vgexqnUvBJKsfnexPfhK9+cjSrypNJjTnIpJQzpJSdpZQdpJRP1FQ/qptkWVtnv/ALx9x515P83aq0Oz3x9J4BJ8xFP1+yi582HAiZXuDRb9eFXEV7gfa75/W/bVfo/1veYkXG35iZPgFB1TzKlaY53Gf5CIC9slFAW9uczdnhDHSfZKCHfr5mvzDgWLQs98pX/8hoPQ3yuT2bB9QbeWIzxg3J97z3/kved25VBHSojKWR8uD53Rh5YjOfslRYEX4cB75FjKalls8/dWdHaiket0+Q47H+EL03uIh1r9lIMYqyScT3OZQAmLHzctorAIywPsNrjgtZ5OziU6cxxZ7XnUVVCOcuGSjyK2UHpEFUSa5rHqBMGlvqvdsEt6RDUWrVQx7/dppv+oWRJzbjqUt6+mw/6f0dyEwzceNp7YHgexX4YzTAuPnLqe14+zo9OVkq6W2qxcAng1QL9VTiX42s2V3Ed2v2AcFH/0R8MWriyxXOmgnXpcFP/cjKXUVBj5+jLfK83ipbAYLLKx/iqKzy836c9ji5lDBYW8MY809YpZm37OeyXAam7i2TGZygHSATXzfUTeZvATiGsfg3yzEKDa3itwnDDcs7NtGjYxr7nf/A+SfSJCfdJ6rH/09577knRhWw8NKVfVnzSOTzAzWpRUM6xp4orSaIZx2BlmLRPkr8q5EJXnu7RvL5x5o/PllfrYOlVqQ0fq6I55oHSirYUxR8LiCXEh6yvAfAaOujvsdE1dxDR20Pl5jmMdH8FgDFZPGE/RqcBl/zItdir/vMH2LBTl+xGTN2yl2bp/gvBlt03wiu6NeGhy8MjNs/q1uVG8Xo79ClWY4n5r1VbiY//et0WuXqq2bdH7H3xG68LmGzSYso6iUVrO23ru3H3DuG1XQ3qgX3R1znff7HK4sLDpM/YTp/xBndEtTt4/U61vzx8fpwN+8vIX/CdH73Sqm8fm8x/R6fHXx/2ggvaaQ3V7wROPHqzT3mj2ksihltfZSV0nf18D8rb+JLx6ncZbsBgP8zf0FbrRC71LjN9vegbU6ynwPANeYf2ZB+HV+lP8SWjGvpqO3B0ekcBrTznSfISjPz9KW9aJkbmOrAO+eQ0YDtXgXrpn2TeqS54tHdVqC35X9S29hcS8cjWWlm8lM0SmfiJT19JuIhMRO+KaL9Svyj5fMlui/ZP9d8tAQTaO/ynzcW8tHCwK0Hw7UV73fr9236vc1YXbUqdGuh7guft6XQsO/xXNM/csibPIq4wjwXgFWyfcDxL52n8U/beD5znMG79jNpIMoAGGp9MeTq2oM04H7bOABMwm9j+Eb52P3mNUL95kutVTt+ef9prj9VXwdgJBjuDUjcFr/7//75DTFX83Z/KeKF8BCrwCb6Nq4c0Jb++YHBArFSFeadGn9wJf4xUh0PzH99bwn3frU6bD1/SyIZ3y3vKCVDt084n3+MnXon7VkAbq8cjwzzdf3QMRKA3Q1OZi/hfckfOEayydmKQlmfkyr+W3Xg5OsCHs1DpePt6LUjlXcStFBx569dfRITL+nJCXl6nVRb+q9IPMryr8N4TxYF08JYvhf+QpWoaB/vdrwjFYz6HukVo50w66PpoaFfO4eErbtJtmFwxUvM6POap+zec4PntwfB+ZVPMsL6LIepz4O263jMdjU0PRGbX9I472gc7z0Dpt92Kv88q2rnKCGER/SNUjO4yauXzpUDqnJWuQeXmvAHp4gWHRfEY/SpUM8aIlGPWqGENZp5gGDtxNLNgC9TMix/r0gFQ7dPhNeMZmCqh+7CedZ2Gd4/u3l3B88Xv4fGYNITfw1qn8eNp3UIGSlTiYVidMv9Pcco3nHode1+6aK9dTzdK39M95YNPO/bNtLTG7sFPJT4+5PtmqAtrcaNeFLNDeEmFSaik4GK9qkh2t0zIyG55t2fm//3c/6Wg5z+zFy+Wm6cIjhYO4kg0PJPDN5Wuvf6BKO+J+MLPUpbAsAame9THm6jcHdfE6kh3oLkv0esxaTx8lV9+exvgwAv8Y+iA3nZ+hzA4WOx7y8QLclOfxwvqTgGxDMwpVpunzoj/lC1yXMi8P/huBdZrdwZPFbd+3sT7Ok+FpeNf1SQlDBt1R7+/M7CqNsKhrvrUsb35Y1GcMaYf2SDsw2/OENnpAx6rTjEIycj+IYkfzLYNeuC3i09Oe17t9HDOrPT9VQRkdyzO4d9ojZrUaQeHsu/+hL6hiR1U+Algey0+G83nOxFKjjBo32i6w+Aw+Fv+Utu/Wh59A3598XH5++e8A0S559gYyaPIvqKLcxpPg7nH9HZKG7rLJjodmxajy0GG8R7M7RTY5b+cSTIBUJf/7nL+nDT6SWUVTpCV/TCpIkUyjxbs6SgwZ8Qxg/rwPIdRzjDYOP5mqBOWf6xIKXkvz9vZW9Rueu964AIrBdVuwnomxsjy99zLIYJRP8f39fLd3O4THdHOKU0tPwT7Tc+w7QCTUhGjL4u6nNDicfnNw0y3Lzcf+Pw24Z34rv/M07uFm6Az0wz+SZUq61qlmRS4c/m/1nH06dOzXKYe+cZIXcqq07qlPjH4lLZfvAYE7/bwE3vL/UpD/YliNitIRMnmP4Li7zfRbrblREfLNjB6l1F3P7pCu76fFVV23FE+0yav50N+4rD1jtPW6AnZGth7PJx57sJhZFI98tvZLj69ek/9fJ5r2mCE1vUN2w30gyUyXbthsrhEwnVkV5ckbrULfGP4Utudwmr+xE+3AASqdvnri9W0e6eGZ6dveLpY6DlLz39KKmIPnrEu7X7v/ZdZxAszt+I9xf8wTVvB847uDOCGtFTbKMgYwxnmFYyx9E76B/0kpNaBW2jq2uXqUtPDvTNByOaibxorb9kWLAFE8/jtatPjquNVLCsjYh2riaZ9+HfdipOQsdKnRJ/I16fu5X8CdMDVnS6cbs4Ah7//ApitZ6OlMUf3WHk2rG44tLf+70gvsb97jO426fqtc3hZMr87Tzw9RrmbTkYcBz0tNBGfJt+v+f1145TAZg0th8vXtnHp14o67t5gwwKJp7H6D7GA0S8P+DjJfd8pCQ7C2ysRDog/314J07r3ITRfcJvaamook5N+Bp9xV/+aTMAVrvTcFn92S/8CkT+gw9Vy+iY/6RgND/EI8cqsdqdAdEDUoLFJKh06PcVD2YtcJDzFvKvV+zhlPZ5PgPCu78V8Pj0wD1w3UxftZfpfhuKA2RT7nl9XeXdLJInAjC8azOsdt+/U6gQ+mjFOVgmzmBEPKmfoqLqpra4fZo3yOC9vwyolmvVpjUIdUr8jXB/lIUlVs9Cm5Ak+IdSZo3d7TPgydnYHJKf/nV6wDGLWYNKB3ZH6AZ/33qIwlIrh0utDO7YmM7NcnwGKf9Y9aPllQGW/z1frvbkqgE4Zo08ysWbs1xx/X+yPsRS6Zur36L5D8zBf4RGv8/cLAtHy2yu474VjJK1hSJqt0SK6kWqClmqrz+oLdR58XfvfnX9u4v58V/DgtbzhDq63ru/nhe8PI9LT24ds5UXz4pOdwoC/ysfPlbpSUfgCBNUfNVbVRk1zZpgy5Pn+hz3X6W6ZndxkEVeVa9D5cEJxbmmheyWeSwzyL/vv5FJKN0ysvwX3DOiaoFeTL3zunYUk/qK2FFDQHKp8+Lv5kCxNeTxQJ+//v/q3UWs3l3EPed0NawXDv9onFj0wl+MR78637NxdzRpoe1OydbCUh74pmo7ZaMUBUZNeqdDiMWgzKKC07RVvO84M2wCN4heGDIspug7Bfx+z/CAJ5m6sOVgKpCiDya1hjo14Rt1LL5Xfe9FToZ1Xf9H+yj9wuxNTJ6/PaK6FTYHj01bx7GAp4XAPsX6u/lymW96CveEre/VAq/ndqmA7961kTJKW0y6sPNzjKt5vQn3EUTzEbVokEnHpvV8ylLVXRIr6gGlblKnxD9avDM7+u+9G+z3H1IWDE5atuMoj3y7zvM+1AD13u8FvDNvO//9Ofgm6P4kIzIl3LqxWev2B5SFcoud1byUJy3vUCjrs9TZOWg9b0IJcLh7jtenHH2oZ2oOFt4pO1KJVBpbU6kviaZOiX+o77jRMW9XRoXNwXwvKziSH7TTKSksCe1OcnPKk7OZt/mgYT/ci7jcaw4q/V1FBie5xdFbCN3zCyUVNsoqjecaIhECo6eBeLjn2NNkikquqryfsiB750ZDsn+wQsBfT23Hl+MHJ/dCySbFhS1VB83aQp0S/2jxtvw37S/l6rcXUhBk1ykj0Xxt7hb6PzGb3UfLWbO7iL1HywMrudhfbOXJGcbhkZ61Bq4fwxs/b+OHtfuqrh32TnR6PDQTgJ4P/0C3B2eSP2F64H1E0E5xuS18pQjpLbbQzqY/yTx/y5UJaTOcaMQ7OAghuP/8bpzUtmHIeilmUAclZUNSlfYnFTXh68LI3WK08CvsilmvL+ycjYUA7D1azqX//T1sHxxOaTiIHC6rZMATP3KK176yN3qlmwhlrSfDCk6kK2mg5hrwzn/es8l5JMTTg+rWlFR1HaSqZZ2avap91CnLf1vhMb5bHbi4CIytNP/dnLzx/0F7W0/llQ52Hy33DCiR5lbbuL+E37cGulTW7tFz4SzcftjwvFCrhJPh8492i9michtPzthgeKyftonD6W2g31+iajNVBdWbVPOlK6InVQfIRFDnLP+bP1wWcepco6Ro7t/zY9PW0blZjqf808U7Af3L0v2h730EP5ooo/EfLgso/MfseQAAIABJREFUC/f1u37K4qDHov3qRtLVaAeU3o/8YFjeiGJO0dZRkDOcaLfJzowxdBOodtMy1QeqVB2kUvzPdtxTpyz/UBjlXjdagOUW8oOllfz5nUWe8j8O6VsOChFo6ZfbIl/xavSU8O1K46cVN8dC5I1PhvD4p3yIlStNc6gvylne/LKI6r8ypi//u0nfLatp/QzeurZfTNetLmsuZX3pLlJ2UErVftUy4hJ/IcRlQoi1QginEKKf37F7hBBbhBAbhRCjvMrPdpVtEUJMiOf6ySYRSdcAxk4ObplHwhfLItsa0ohoY9IjEaxESdogbS3rnW04kHNiRPXP79WS/vlVzwhDOuYlqCfJpTa7DhTHL/Fa/muAS4BfvAuFEN2AK4HuwNnAa0IIkxDCBLwKnAN0A65y1a1Wlu8IskOTH94LlzxlXpEuRgbwZy73T03j1vyvl+/2KV+3J3Qu/UhcAD9tOBD02BnactqIwDh/f9KppL+2kd+d3WMWx5jPC5kaIqYmja/j6l+qWtgp2i0Pqfp3qy3EJf5SyvVSyo0Gh0YDn0gprVLK7cAWYIDr3xYp5TYpZSXwiatutWKUY94II/+794BwwCCG/1A1bsAdCf59vO2T0Ns7RrLz16+bjeP8G1HM5LRneN3yYtg2hmqryRA25ju7+/zI7xzVJfhJfsQqDsFO++TGgfxy1xmxNWrAoA55jB2czzOXxr9qOZkkehe2eEmlJ6VovmP9TmhI64bRJQmsSZLl828FeJvAu1xlwcoDEELcKIRYIoRYUlhYmNDOhfKRA+w+Wk6fR40nKY93wu1dG8u2j246Cf0po4dWAEC+2MsrlpdII/AJapzpe3bJxvzs7O1ZvAYwdnB+zNePl4Ht82jdMCth7Zk0wcMXdvds7J5qpHpK51QaBCLh85sHM+/u6NKD1yRho32EELMBo/3i7pNSfpP4LulIKd8E3gTo169fwr+ef/94OQeKKwyPfbtyj6HLpy6wr8j4bxIJZ5mWeF6bcPC05S1O0TbwgWMkC5y+3r2O2m7mOPpgx4zNa88Bo0RywQhmlQVkfw447/gSlWRxvImrIrGEFX8p5cgY2t0NtPF639pVRojyauXblXsCyvInTGfxfbHcbuqx60jw1cSh+N5r5XC0DNHWeF6fpS0hE93lZJeBapxDOSXoVrZ3SK23LnsvajPC32Id1D6Pc3u1oGlOaEtbSd7xQSqN0a1yM8lOjyO8OAVJlttnKnClECJdCNEO6AQsAhYDnYQQ7YQQaeiTwlOT1IeY2FpYmrKPwTWNhpNrTLPIpcTweFNxhFkOfV/Z5y2v0UvTs5XWE74DkRk7WcJKiXSJv5fbx3sNwUc3DAzZH/80zc0bZPDngSeEvY9UEhVFIKn0+bi7cnm/Nvzwj8BNk45n4g31vFgIsQsYBEwXQswEkFKuBT4D1gHfA7dIKR1SSjtwKzATWA985qqrOA74l/kzHrdM5jpT4HyIBTuNRCmrne0AyBBVbrP/Wl4gnUpA8mfTD2zJuBaAQ9QH8HH7eIt/NC4giG3i8rnLUnsytjpIVVsnhcaAWklcK3yllF8BXwU59gTwhEH5DGBGPNdNJlKm/uKcmiADK7eY9Ye0Siye8gu038gQlcxz9ASgkAZcXXkPH6Y9VXWusHGDaTo7ZDMes0zxlH/l2qC9S/OqldLxhFpG+ql5+7qHdmoc+wWPc1J9wleRXOpcegd/GmRaKPKK3ZdIFmwzzqFzvDBUW8VFpvlkYqWRKOHGyn9QTL3wJ/pRn2O0EgdZL0/gZG2Tp3yUaRGvOy4kX+zl5bRXAHiOSwE4IHOZ7+zJRdZHucP8Kf+wjWdxxi1cbJpHM6Gvr9h72bcMel93HX1766l0b1nf03Y8k7ERi5gI8rqOkaq3nqr9qm3U+fQOo/u0DCj7ZVNiQ0urm7ctz/In06+ca1rEQG09b6X9J6Z2HrK8x3fp99CUI5ylLaFS6j72Pto2NJzMTf+Xp+5o028AFMpcAFbIjlxju49CGjLf0Z0O2l7qiQquq7ybyuYne87r2bpBwP68saIM2NhI1SfdVIrKStW/UTzUefEPsBZrwWe8XvpOep6ibSCPorDnXWOaxY2mbxE4GW/6mj+ZfgWgq7aD4doKfnb2Zp6jO04p6CYKACiXaVRICy3FIUC3/P35zFE1UbbC2SHW20oYKaQpNUq6a8LcEm2a1rpELf6y1PlPvbaN6KO0xfTR9M1RfnT0ZYezCQBLM24Oek4WFQzVVvG4ZTL3Wj7mQu037rJ85jn+XtrTtNEK+c4xgLnOPmhCcpFpPgAXVz7KQRqQJfSwzkME5uT/znmK53UR9WjeIIPG9dJ4888nB9SNh0gnfH29PrX3xx2O609tx83DOnD9qe1quis+pJLFX5up8z5/f714fvYm44rHCW+kPQ/ANMdAbrXdxpPmtxmj/QToC68c+IZHZlLBugzfXPovpr1m2Pa3zsH0EVsA+JPpV4plJltlSw7LHFqLgxySOdgMvlKVWDjP+gTNhT6Xkm42seT+M+O7URdz7hjGf+du5dMlO2vZMJ58Miwm7j67a013IyhqCEguyvL3e7+4ILKkb6lIhkVjv8zFKQWP2PRwyvqiatvJS01V+fcysPKe5SnO0qpW5e6TDSmXaZ737So+YIT1GUplBtdV3o0Ns2dhVkNRygpnR2yYOSr1yeRDsmri1p+1sh0/OhNr6QO0a5zNqe6InQjVX1mWimipjRFRtdryj8QNUJs+1FM7NiZrm5XJjrMpRPe9P2G7hnTsnGlaytOWt+gptnG//Xq6iwJOM63mNNNqz/kTbH9lsbMrV5jmMst5EhKNrbIVPayTPHVKqUpctUvqLqU1sh2nsZrOWo0s1q4KWYxQ/X3cPmocSDnUR1I91Grxj4yaV/9P0x7Fgp1LKh8NWU9322gE+3nY7A5yRLmPQO8lj7/bbmWDaRwA15h/ZLLjbAZogclY1ztP4BiZTHKcE7QPR2RVyOgeqefTf8Z+OTZMrHXmh+x/NHRokh1xXbffvjYN5ArUKJBkarX4RyIGyRKMDk2y2Vp4LGy9n9Nu5wRNz4/fXRTQQhxitoF7ROBkdfpf+dxxGg/ax6HhxOnntTPZ9OuVSN+0shWkc7ftBp62vAXAj+l3BrT/mO0a9kewmeIxMlnvbMuJ2g6K0AVaovG8PbLduCKxtOfeMYxG9dLCV/RrM9LPUln7CkUt9/lHogVGgnFa5yZxXztSv7Jb+AGmp9/L22nPGdZrTBFZwsq15lm8bnmemWl3B9Q5v/gTAI7InIBjXzqG8qL9EsO2B1e8xDuOc8P29abT9TDNR+z6fMIyZ6ew5/gTyV8lv3E29TMs4StG0aZvfeH1WpFqpNLgnEJdSTi1W/wjMAWdBnUSuZsT6KuIjcggcDMY0BOf+ZPjlRztHNNiOmm7ferlUMYlZXp45hYZuHDNhpnn7Zdyv22cp+w86xPcXPl/7CGyFAfjhuQDsMDZjfYVH7BWRhYi2KlplavoxBbBJ4XjpbaF7dZ1Ukl4a+M3q1aLfyRUx4cazJJpI4xXEjflaEBZPQJTNDf0yq45UFsHwP76vVgpOwbty1THYADKZDprZTufGPxweN+Gv8spUt6/PvLrRYpy+9Qu3J+jispKLrVa/GN1+ySCSNLH9G/qNCx3x8N7458WGSBPVIl/f20jTi2NWae8E7JfxWTxX/sFXCdCTy4bEuNvMTNNX1swtFNjGmVH7suPHNeEbyxnKoFR1FHUhG812P5aEIHJdupbKjqk4HrbHaRj542053Xx9+uW0VNCI1HsqddDbOdYwxNxiHDiKphov4ockxkM3Euhz4xcKK8Z2JaN+0pYXHCEvOw03r9+AL3bBKZ+SAQqO6UiWdRm26BWW/4RYSAYiRAR7y9NsC9QttSjc4ZV/oe5zr4sdOqrLZuLwIVmbcQB7FLjoNdCqjyKAX2OoIe2ndJGPeLvuIsnL+4ZUBbND+GUdnmMOaWt6zzB0E5NoprEjYaqbkUY5+/92SS6M4q4cX+KtVl4U4FaLf6RWPVGNRJvQBp/i3Odum/fHZ1zlHrYpUaeCEzClkMZxWRxpvXfXGp9EIA8oYv/CWI/9UU5xY37Rv6DCVPPLdyx4pTS86SQ7N+w23UTsc9fSf5xQUp9SrXwsbJ2i38En5dRtE8ieLr0Af5l/owsKnja8QydxC6f4xpOBlX+xkFZn1JXygQQlJJJNoGbqNcXZZTILI5Qn6WyMwAPW94jDRvdxB8A2NNzI/+OxnDb0fwYvfuRbAsu6lDPlFIVhaJmqNU+/0hwGrp9jJXRjJ0srBQTfvVpX8dK+ppX8mfTLHLlMTBbud5WtbjqfG0BXewbmWi/0ue8UjJ9wjpBHyh6iW0ccW3IIr3G7K/THqSbpou/w5xFMgk3Odq3bS7Ld+hPMxLpmeuoLqMptgnfhHdDESexbMepiJ5abflHQjSW/9uW51iVcUPYeoKqKJ5cV2I1t4sGoK/YzEtpr1BBOm86zvc5t0RmBoR1bkn/Mx20vUx1DPGU7XalVnALP4DU4vepp5s1Ft07wvDY/7d35tFSVHce//y638r2Housj11cwEg0HIWMZiAgizqSjGQGxoxblOM2OknUgGiSwTgxwZmJiYnocZzJgkvUODBGj6KjZ8Y/XNARXBB9IAoMBFABAXm81/2bP+p2v+ru6n2rV30/5/R5Vffeqvr1fVXfuv279/5usk4umXdCQjjgx6/qti/qPZCpLHR3+OYf28fiX/wwEivILsJAi39u4R1SC6U7bnp4PeCEQc7EIA6kpLVwML4dE+w7+t6QMl5+mw6Ou4hCRHm84fuExDFok7bFy32t49aMNhTCgi+1sf4Hsxncr8kzP/lZnHXiEG45d6JnWfdLtexun3hgt1zLB/eBtpSHIP4WCbb45/Av82qhZjvOazSOm7safp6S1k8OE7uFWs2L4PX61Bg+H+ng+Fq3x8l2Tgm1x/O2RLtn7saidgIs6Pg+yzov5bNBp2S0KxtN9SGa6sMp6SNam/njtWektIIyaaiSf0dsoRQT2C3ILbueShCF1o8EWvwz0UAn9XQRcSlGCwdp4aCniNS7xsS3ulrxbq4JP86Gxss4PfRuQvrHtDJQPuPFxuto5CiDZD/7tRedHmPyD9FMb+lAiDLINepn6pFfsJOBCWWXd/4tFx+9gXV6Aqsis9AidSydeA5vbWLS8NQVutLNXwAY3tJcOVnNt+VfNkMspcT+n8pLoMU/U0twXeOVvN90YdztM4ADrG9azMr6n3ke53b1tIh3tM7r6x8xLXxY3dDty98SctbUbZO9nCgfcUnd054rXgEcVMfl0osOehvf/z1d57ArSfgB7o/M44Voca19N8lfOxbILd46TnoaMz2c8QVWsKN9LPnhp/7eIN8rwRb/NOlCNC7SQzsc//t36h4BYFr4HVRTfUF9XOI/WTbzZuO3+Eb4hXjaCBJn4O4Ij6BLnep9JTQ5nh5zCQ2SA57upUMmFv9E+TA+6ue3kdyWPCzWhZH80B0bC8gW0/6k02dq+bvLV2y0T2F+H4tPCbLw+oFgi38aMWigi83RYQCM6dgIKN+sey6e3yua6tZxx9b5dv1j9JXP+WHdr+Npf133fEL5/aEWVkb+AoD/DnUHM2uTvQA80PVVT9tisfgfaVzOONkJwMGk+PzpyCdUhXfJxFRJ8zeen+XhrJQ/Pd8OXNvh629sdNbKEGjxT0cHDZx1dAUAl318B0vqHkzIb4mkduhODm1OSest3SGZF4WTxF9auKPrrzj+yL+zPTSc6R3/RMTllE8XW/8dHR3fvqpuDXu1H/vo41m23Igk/y1MNCvl9vGTu8BSCuxLupwEWvwzaUGUEEfVGdlyRd0TCXmtHuI/UnanpIEzHBNgtyYGLdsdGgwIHTidult1GCd33BfP/5wGT7HarCMS9t+KjqUcD0EuZ4yLfoFhGirVwM53DV+vYy0+wocv8SA2LIoSfxFZISLvisgGEXlcRFpdeUtFpF1ENonIHFf6XJPWLiJLirl+NrL9w55MimV/1dFrAegXSQ2pPJRP2andyxw+E3GGad5Ut4oxspP96sz6/VrHcmZ2rGBXeFjKOQ651taNvRS8+JfO8+Pb66LHZf4SBeJVNaMHJs5cLtZtUyldtcM1g4kfXsw+MKFsFNvyXwucpKonA+8BSwFEZCKwEJgEzAV+JSJhEQkDvwTmAROBRaZsVfjHzgsS9reYFbBao6kt/6HyCbuM+B/QZp6ITAXgsrqneKHxu/SWIzwfmcwbeiybdUTWG7eD9LNx74ycz4Qjv+F7nZdzd+S8fL5SUVPjLz9zXMJ+qtsHz/xqU0zHsk++gsVFP7PyXWy0maU8FBXbR1Wfce2+BCww2/OBh1S1A/hARNqB00xeu6puARCRh0zZd4qxI72BmbN30z9hf4sOo0PraI2mtvyHyKd8oMP4y44fcoSGhDH4AJNDW+IvBIC3/697lq+3SEpGseqkjocjMzJ/gSLwMimcZv3KZPdPzteolNvH/A3iT/NapKk+zNbbz6m2GYGnlIHdLgUeNtsjcF4GMbabNIBtSemlX9fPkI8PeGbHCo5Szx5aaY1+yk11q6gjwvKuCxGiDJDPeC06gddNRM1R/CnlHJ/lMCpnZscKTpBtWcsVSq6dsrnUTGwoZ9znn7eYV7ZdbUeJWMpFEO+trOIvIs8CQz2ylqnqalNmGc6yUKtKZZiILAYWA4waVVxs+Uz8qus8RsnueEfrXm1hSNcuZtS9bfLns67pSgD20Td+3Ec6hB91XsDN9d1f+RDeMXHcbNYRKZ26pUK1tBER/eLWycYpo/ozua2Fm8/J34Noh31aMhHk2yOr+KvqrEz5InIxcC4wU7uVZwcw0lWszaSRIT35uvcC9wJMmTKlIEXLpIOLThvFg698xE+TQirv0VbOiHR7oU4LbYxvf6J9E8reFzmHd3UUv2v4MQBH0/jx091Afm9LJLf4U33+uU3yKjfNDWFWX3NGZS5msQSEYkf7zAVuBM5T1cOurDXAQhFpFJGxwATgFeBVYIKIjBWRBpxO4TXF2JCJTOKaxr3NHm2hWbsndM0Mvx7f/pzGlPIvRr/AjzsXAdBMR0p+sVw/+zheuH56TmVLLbZBbvVYLLVOsT7/u4BGYK1pBb6kqleo6tsi8nucjtwu4GpVjQCIyDXA00AYuF9V3y7ShrRkcoGky9lDYgCz88MvxreT4+zHiE3Cet8VcrlUtPRqYMyg7IvHpKO1Vz37DncWdGzKjN58O3wLumpl6Qk2WizloNjRPsdmyLsNuM0j/UngyWKuWwqiXkt4AQc0dTWs+7vmcozs4+HIdM9jHon8OZ9pM09FT/PMT0dO/vk8fPieAenqw+yjQPFPmtlrfwlYapUgjiQL9Azfvk3px9J3pRH/w65O2z9GHDHfEB3H33Vem9Dh6yZKiCejUxOWV+wJ/Oc1Z/DYldPS5if79JO1vyHcs76vxZIvQR4QEOg1fBvq0otTJI34H9Ju8b+r6+ts08F5t+iTqWarIdOt+4W21Bj9bsLJ4p+0f0zf1D6QTOX9SA8w0WIpC4EW/0yka/m7h2tu1NFs7BrtWa6aTB7Zyvpt+1LS8xGyXFxOoVBiWffpmz1W/CrkGhaLpTrU7O/2SJoVxl+NHk+EEDd3XlKya5VaA08a3q8i14lN8vI6r20xW2qJIDZjalb8uyJpOnzpw9ktq/ldjguo5EJHV8QzvVCxzraISoyrZ4xP63rJxSWTfJ18Bb9HuH3seB9LjVKz4p/O5w/pxbpQjnR6/8rIBS8rkzX1TNeSiTEumjaaG+ackP68Obx5kkMlu8U8F8m0bh+Lxb/UrPin8/kDbP34cNq8Qvi80/tl8ou/Kd36uzGGtzrxhZLDMxdCplZxT2jV50JAvobFkjc12+GbqeVfqWsdN8R76Gg2MunV7ElDeeDy05k6NnXB94Rz5KF6nj7/HI4LygvCYgkiNdzyL9wVU22yieqXxw8iFMo8MSun0T4FxMkf0dpM74ZwztewWHoCQbyVbcvfkp5MyyOmean8z40zAjkywmIJGlb8eyCVjhKaj9sn5IqYZ90+Fot/qVm3T08R/+rODjbj/L3ychB26/ax9HSC3H6pWfHPNNrH76QbheOVWszNG+QbP0YtfEdL8QRxJa/aFf80k7x6AhUXLDvD11KjBHkSYM2K/3WzJlTbhIJJdzuW+nUWXxi96DMH9wGyBJsgtvhj1Kz4Txndv9omFEw+re5iWi6SIbZPriEmHPz7AAW5ZWexZKJmxb9Wnvk5k4Z4pucix93hHTzyCrbIYuk5BLlxULPi35P/qfkMoVwy70RGDUhdnSyn6xR0VHnPZLFUBf/+eC2Y2hX/HqxH+ZgeDgkD+zQUdQ6vIZv51Z9/n5yefB9YLMVQu+JfbQOKIcn4M451onq29W/O+RS5uX0y1VKPrkGLpeapXfFPErYHL5+adVnCauDZ6k4S3sVfGcdLS2cy/pg+Jb32F0e2cuG00dy5sNjoo/59UfjXMosfCPIvw9oV/6T9aeMHsvjMcRmP8ePLoS4kiAhDW5rSlvGc/JXDucMhYfn8kxjp0WcQFLePxZILQbyDa1f8C3ij+6UR4LY90yL1mbAj9y2W2qZmxd+LnvITz21mY4HiX7QNedWVfyvWBp+z5EIQ75KaFX+voZ7tuw9mPsYnd4Dbjsa6cHVsyG+8UNnssFgqQRDv4JoVfy/t+vTw0SyH+EP93XY01mf/F14949hymmOxBBZ/PPHloWbFvxyt+JPbWkp+zmwtjpOGZ7/mzBOHsPX2cwiV8DsHxu1TbQMslioRePFfseBkz3TvETCZpWDS8H4Z888/tS1Xs4oiJrzDW5pY8Q3v7+cvgvij2WLp2RQl/iJyq4hsEJE3ROQZERlu0kVEfi4i7Sb/VNcxF4nI++ZzUbFfIBtnTfSObVNIR9+5k4cVa05GVn7zSzmVi1k+dfxAejVUZzE222K21BJBXJioWOVYoaq3AIjItcD3gSuAecAE8zkduBs4XUQGAD8ApuA0B18TkTWq+mmRduRNIeKVXyTL/Bk7qHduBTNFXCuABy4/nZbm+ryOye/l6d9XhV868S3+JMj3R1Etf1U94NrtTbcczQd+ow4vAa0iMgyYA6xV1U+M4K8F5hZjQ6F4/VNDWWojm+AV2zrwOn2m9XPzvdqZE47xPO+Xxw9iUg59B4UTvFaTpbYIYMMfKV6w5DbgQmA/MENV94jIE8DtqvqiKfMc8D1gOtCkqj8y6bcAn6vqHR7nXQwsNrvHA5uKMHMQsLeI4yuFtbO0WDtLi7WztFTCztGqeoxXRla3j4g8Cwz1yFqmqqtVdRmwTESWAtfguHWKRlXvBe4txblEZJ2qTinFucqJtbO0WDtLi7WztFTbzqzir6qzcjzXKuBJHPHfAYx05bWZtB04rX93+gs5nt9isVgsJaLY0T7uhXDnA++a7TXAhWbUz1Rgv6ruBJ4GZotIfxHpD8w2aRaLxWKpIMWO9rldRI4HosCHOCN9wPkFcDbQDhwGLgFQ1U9E5FbgVVNuuap+UqQNuVAS91EFsHaWFmtnabF2lpaq2ll0h6/FYrFYeh6Bn+FrsVgsllSs+FssFksNEmjxF5G5IrLJhJlYUmVbRorI8yLyjoi8LSLXmfQBIrLWhLtYazrCM4bIqJC9YRH5XzNnAxEZKyIvG3seFpEGk95o9ttN/pgK2tgqIo+KyLsislFEpvmxPkXk2+Z//paIPCgiTX6oTxG5X0R2i8hbrrS866/cIVvS2LnC/N83iMjjItLqyltq7NwkInNc6WXVAy87XXnfFREVkUFmv2r1GUdVA/kBwsBmYBzQAKwHJlbRnmHAqWa7L/AeMBH4KbDEpC8BfmK2zwaewpnQOxV4ucL2fgd4AHjC7P8eWGi2VwJXmu2rgJVmeyHwcAVt/DVwmdluAFr9Vp/ACOADoNlVjxf7oT6BrwCnAm+50vKqP2AAsMX87W+2+1fAztlAndn+icvOieZZbwTGGg0IV0IPvOw06SNxRjV+CAyqdn3G7arEA1CNDzANeNq1vxRYWm27XPasBs7Cmbk8zKQNAzaZ7XuARa7y8XIVsK0NeA74KvCEuUH3uh62eN2am3qa2a4z5aQCNrYYUZWkdF/VJ474bzMPc52pzzl+qU9gTJKo5lV/wCLgHld6Qrly2ZmU93VgldlOeM5j9VkpPfCyE3gUmAxspVv8q1qfqhpot0/soYux3aRVHfNT/hTgZWCIOnMgAHYBsTCk1bT/Z8CNOEN4AQYC+1S1y8OWuJ0mf78pX27GAnuAfzPuqftEpDc+q09V3QHcAXwE7MSpn9fwX33GyLf+/PCcXYrTiiaDPVWxU0TmAztUdX1SVtXtDLL4+xIR6QM8Bvy9JgbGQ51XfVXH3orIucBuVX2tmnbkQB3OT+y7VfUU4BCOmyKOT+qzP84EyLHAcJwAiFUJZpgvfqi/bIjIMqALJ8KArxCRXsBNONGOfUeQxT9diImqISL1OMK/SlX/YJL/JE7EU8zf3Sa9Wvb/GXCeiGwFHsJx/dyJE5k1NinQbUvcTpPfAnxcATu3A9tV9WWz/yjOy8Bv9TkL+EBV96hqJ/AHnDr2W33GyLf+qvacicjFwLnABeZFRQZ7qmHneJyX/nrzPLUBr4vIUD/YGWTxfxWYYEZVNOB0nq2pljEiIsC/AhtV9Z9dWWuAWI/+RTh9AbF0rxAZZUVVl6pqm6qOwamz/1LVC4DngQVp7IzZv8CUL3trUVV3AdvEmWEOMBN4B5/VJ467Z6qI9DL3QMxOX9Wni3zrryohW0RkLo5r8jxVPZxk/0Izamoszpoir1AFPVDVN1V1sKqOMc/TdpxBH7vwQ32WoyPBLx+cHvX3cHr5l1XZljNwfkJvAN4wn7Nx/LnPAe8DzwIDTHkBfmlsfxOYUgWbp9Mb54UEAAAArElEQVQ92mcczkPUDjwCNJr0JrPfbvLHVdC+LwLrTJ3+B87oCN/VJ/APOHGv3gJ+izMSper1CTyI0w/RiSNM3yqk/nB87u3mc0mF7GzH8Y3HnqWVrvLLjJ2bgHmu9LLqgZedSflb6e7wrVp9xj42vIPFYrHUIEF2+1gsFoslDVb8LRaLpQax4m+xWCw1iBV/i8ViqUGs+FssFksNYsXfYrFYahAr/haLxVKD/D9HI+7KuzyzoQAAAABJRU5ErkJggg==\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "## Earned score per episode  \n",
        "window = 30\n",
        "mean_arr = []\n",
        "for i in range(len(score_arr) - window + 1):\n",
        "    mean_arr.append(np.mean(np.array(score_arr[i:i+window])))\n",
        "for i in range(window - 1):\n",
        "    mean_arr.insert(0, np.nan)\n",
        "\n",
        "plt.plot(score_arr, label ='score')\n",
        "plt.plot(mean_arr, label ='mean')\n",
        "plt.ylim([-300,300])\n",
        "plt.legend(loc='upper left')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "KTGyfRFEzUHG"
      },
      "outputs": [],
      "source": [
        "## game play video\n",
        "agent.load_state_dict(torch.load(\"weights.pt\"))\n",
        "env = gnwrapper.LoopAnimation(gym.make(game_name)) \n",
        "state = env.reset()\n",
        "state_dim = len(state)\n",
        "game_score = 0\n",
        "score_arr2 = []\n",
        "state_arr = []\n",
        "for i in range(1000):\n",
        "    if i == 0:\n",
        "        stacked_state = np.concatenate((state, state, state, state, state, state, state, state), axis=0)\n",
        "    else:\n",
        "        stacked_state = np.roll(stacked_state,-state_dim,axis=0)        \n",
        "        stacked_state[-state_dim:]=state\n",
        "    with torch.no_grad():\n",
        "        hs = agent.representation_network(torch.from_numpy(stacked_state).float().to(device))\n",
        "        P, v = agent.prediction_network(hs)\n",
        "        action = np.random.choice(np.arange(agent.action_space), p=P.detach().numpy())   \n",
        "    if i %5==0:\n",
        "        env.render()\n",
        "    state, r, done, info = env.step(action.item())\n",
        "    print(r, done, info)\n",
        "    state_arr.append(state[0])\n",
        "    game_score += r \n",
        "    score_arr2.append(game_score)\n",
        "    if i %10==0: \n",
        "        print(game_score)\n",
        "    if done:\n",
        "        print('last frame number : ',i)\n",
        "        print('score :', game_score)\n",
        "        print(state_arr)\n",
        "       \n",
        "        break\n",
        "env.reset()\n",
        "env.display()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "er3f1i6AI8nD",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 283
        },
        "outputId": "cdd57036-34b4-4add-dfd6-ea65ec6042fc"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.legend.Legend at 0x7fcdd598e730>"
            ]
          },
          "metadata": {},
          "execution_count": 6
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAD4CAYAAADvsV2wAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deVzU1f748ddhl00FURFF0HADBBRw19Tc0lzKrU3btFXtW7dssZtdf31vt+Xeyts3r6mp93rLPc2szNLU3EUTcUMEFUQBRQRkGWbO7w9GQgUEBAaY9/Px4MHMmfP5zPvMB95z5nzOfI7SWiOEEKL+s7F0AEIIIWqGJHwhhLASkvCFEMJKSMIXQggrIQlfCCGshJ2lAwBo0qSJ9vPzs3QYQghRpxw4cCBNa+1V3vq1IuH7+fmxf/9+S4chhBB1ilLqTEXqy5COEEJYCUn4QghhJSThCyGElagVY/glMRgMJCYmkpuba+lQhBClcHJyomXLltjb21s6FFEOtTbhJyYm4ubmhp+fH0opS4cjhLiJ1ppLly6RmJiIv7+/pcMR5VBrh3Ryc3Px9PSUZC9ELaWUwtPTUz6F1yG1NuEDkuyFqOXkf7RuqdUJXwgh6rOPN59k28nUGns+Sfh11OzZs/nwww/LrPPNN99w9OjRCu/b1dW1smHVmHvvvZcrV65YOoxaaevWrYwYMeKO93Pp0iX69++Pq6srL7zwQhVEJm722ZZT7Dp9qcaeTxJ+PVbZhF8TCgoK7mj7jRs30qhRoyqKRpTEycmJOXPm3LZjISrPaNLY2dTcsJgk/DKMHj2arl27EhgYyPz584vKf/jhB7p06UJISAgDBw4EICsri8cff5zg4GA6d+7M6tWrgRt7y6tWreKxxx4D4LHHHuPZZ5+le/futGnThq1bt/LEE0/QsWPHojplbV/cF198QUREBCEhITzwwANcu3aNnTt3sn79el555RVCQ0OJi4sjLi6OoUOH0rVrV/r06cPx48cBiI+Pp0ePHgQHBzNr1qwSX4vs7GyGDx9OSEgIQUFBLF++HIB9+/bRs2dPQkJCiIyMJDMzk9zc3KLXIiwsjC1btgCwePFiRo4cyYABAxg4cCDZ2dk88cQTREZGEhYWxrp16wCIiYkhMjKS0NBQOnfuTGxs7C3x+Pn5kZaWRkJCAh07dmTKlCkEBgYyePBgcnJybqmfkJDAgAED6Ny5MwMHDuTs2bNFx2H69On07NmTNm3asGrVqqJtPvjgAyIiIujcuTNvv/12ia/LwoULadeuHZGRkUyZMqWoJ/ztt9/SrVs3wsLCuOeee7h48SJQ+Mls8uTJ9OnTh9atW7NmzRpeffVVgoODGTp0KAaDAYADBw7Qr18/unbtypAhQ0hOTr7luR977DGeeeYZwsPDadeuHRs2bLilzt69e+nRowdhYWH07NmTEydOANC3b18OHTpUVK937978/vvvN2zr4uJC7969cXJyKrHt4s6YTBqTBtsaTPi1dlpmce98G8PR81erdJ+dWrjz9n2BZdZZtGgRHh4e5OTkEBERwQMPPIDJZGLKlCls27YNf39/Ll++DMCcOXNo2LAh0dHRAKSnp982hvT0dHbt2sX69esZOXIkv/32GwsWLCAiIoJDhw4RGhparrbcf//9TJkyBYBZs2axcOFCpk2bxsiRIxkxYgRjx44FYODAgcybN4+AgAD27NnDc889xy+//MKMGTN49tlnmTRpEp999lmJz/HDDz/QokULvvvuOwAyMjLIz89nwoQJLF++nIiICK5evUqDBg345JNPUEoRHR3N8ePHGTx4MCdPngQgKiqKw4cP4+HhwRtvvMGAAQNYtGgRV65cITIyknvuuYd58+YxY8YMHn74YfLz8zEajWW2PzY2lq+++oovvviC8ePHs3r1ah555JEb6kybNo3JkyczefJkFi1axPTp0/nmm28ASE5OZseOHRw/fpyRI0cyduxYNm3aRGxsLHv37kVrzciRI9m2bRt9+/Yt2uf58+eZM2cOUVFRuLm5MWDAAEJCQoDCBLp7926UUixYsID333+fjz76CIC4uDi2bNnC0aNH6dGjB6tXr+b9999nzJgxfPfddwwfPpxp06axbt06vLy8WL58OW+++SaLFi26pe0JCQns3buXuLg4+vfvz6lTp254vEOHDmzfvh07Ozs2b97MG2+8werVq3nyySdZvHgxH3/8MSdPniQ3N7codlEzjOblZWuyh18nEr6lfPrpp6xduxaAc+fOERsbS2pqKn379i2ad+zh4QHA5s2b+frrr4u2bdy48W33f99996GUIjg4mGbNmhEcHAxAYGAgCQkJ5U74R44cYdasWVy5coWsrCyGDBlyS52srCx27tzJuHHjisry8vIA+O2334o+kTz66KPMnDnzlu2Dg4N5+eWXmTlzJiNGjKBPnz5ER0fj7e1NREQEAO7u7gDs2LGDadOmAYUJp3Xr1kUJf9CgQUWv2aZNm1i/fn3RkEFubi5nz56lR48evPvuuyQmJnL//fcTEBBQZvv9/f2LXquuXbuSkJBwS51du3axZs2aoja++uqrRY+NHj0aGxsbOnXqVNQT37RpE5s2bSIsLKzo9YuNjb0h4e/du5d+/foVtWfcuHFF7UxMTGTChAkkJyeTn59/wzz1YcOGYW9vT3BwMEajkaFDhxa9xgkJCZw4cYIjR44waNAgAIxGI97e3iW2ffz48djY2BAQEECbNm2KPrVdl5GRweTJk4mNjUUpVfQJYty4ccyZM4cPPviARYsWlfjJUVQvo6kw4dva1NxAS51I+LfriVeHrVu3snnzZnbt2oWzszN33313peYbF5+2dvP2jo6OANjY2BTdvn7/+hh3Wdtf99hjj/HNN98QEhLC4sWL2bp16y11TCYTjRo1uuFjfGlxlqRdu3ZERUWxceNGZs2axcCBAxkzZkyZ25TExcWl6LbWmtWrV9O+ffsb6nTs2JFu3brx3Xffce+99/Kvf/2LAQMGlLrP4q+dra1tiUM6ZSm+vTb3urTWvP766zz99NMV2td106ZN46WXXmLkyJFs3bqV2bNn3/J8NjY22NvbF73214+71prAwEB27dp12+e5+bjdfP+tt96if//+rF27loSEBO6++24AnJ2dGTRoEOvWrWPFihUcOHCgUu0UlVdgqvkevozhlyIjI4PGjRvj7OzM8ePH2b17NwDdu3dn27ZtxMfHAxQN6QwaNOiG4ZDrQzrNmjXj2LFjmEymok8LFVGe7TMzM/H29sZgMLBs2bKicjc3NzIzM4HC3re/vz8rV64EChPa9THbXr16FX06Kb59cefPn8fZ2ZlHHnmEV155haioKNq3b09ycjL79u0riqOgoIA+ffoU7efkyZOcPXv2lqQOMGTIEObOnVuUZA8ePAjA6dOnadOmDdOnT2fUqFEcPny4Yi9aCXr27HlDG/v06VNm/SFDhrBo0SKysrIASEpKIiUl5YY6ERER/Prrr6Snp1NQUFD0KQkK/358fHwAWLJkSYVibd++PampqUUJ32AwEBMTU2LdlStXYjKZiIuL4/Tp07e8zsXjWLx48Q2PPfXUU0yfPp2IiIhyfSIVVctovN7Dl4RvcUOHDqWgoICOHTvy2muv0b17dwC8vLyYP38+999/PyEhIUyYMAEoHDtPT08nKCiIkJCQohOV7733HiNGjKBnz56lfiwvS3m2nzNnDt26daNXr1506NChqHzixIl88MEHhIWFERcXx7Jly1i4cCEhISEEBgYWnST95JNP+OyzzwgODiYpKanE54iOji46kfrOO+8wa9YsHBwcWL58OdOmTSMkJIRBgwaRm5vLc889h8lkIjg4mAkTJrB48eIbetHXvfXWWxgMBjp37kxgYCBvvfUWACtWrCAoKIjQ0FCOHDnCpEmTKvy63Wzu3Ll8+eWXdO7cmX//+9988sknZdYfPHgwDz30UNHJ7LFjxxa9eV7n4+PDG2+8QWRkJL169cLPz4+GDRsChSdnx40bR9euXWnSpEmFYnVwcGDVqlXMnDmTkJAQQkND2blzZ4l1fX19iYyMZNiwYcybN++WE6yvvvoqr7/+OmFhYbfMjOratSvu7u48/vjjpcbi5+fHSy+9xOLFi2nZsmWtnfVVFxWYTEDNJny01hb/6dq1q77Z0aNHbykTorbJzMzUWmttMBj0iBEj9Jo1a2rsuSdPnqxXrlxZ6e2TkpJ0QECANhqNdxSH/K9WzsWMHN165gb9710Jld4HsF9XINdKD1+IOzB79mxCQ0MJCgrC39+f0aNHWzqkclm6dCndunXj3XffxaYGTxqKP1hiDL9OnLQVoray5JeSbh6Tr4hJkyZVyVCZqLw/ZunIGD7wx4wJIUTtJP+jlXc94dvZSsLHycmJS5cuyR+UELWUNl8PX76JWzkFMg//Dy1btiQxMZHU1Jq7kpwQomKur3glKs4oY/h/sLe3l1V0hBD1liWmZdbaIR0hhKjPik7a1uAiMpLwhRDCAorG8OWkrRBC1G+WGMOXhC+EEBYg8/CFEMJK/NHDr7k0LAlfCCEsoKA29vCVUq2UUluUUkeVUjFKqRnm8tlKqSSl1CHzz73FtnldKXVKKXVCKXXrahxCCGHljOZpmbVtHn4B8LLWOkop5QYcUEr9ZH7sH1rrGy4mopTqBEwEAoEWwGalVDutddnr1AkhhBUpqI3Xw9daJ2uto8y3M4FjgE8Zm4wCvtZa52mt44FTQGRVBCuEEPVFrT9pq5TyA8KAPeaiF5RSh5VSi5RS15fM8QHOFdsskRLeIJRSU5VS+5VS++XyCUIIa1OrlzhUSrkCq4EXtdZXgc+BtkAokAx8VJEn1lrP11qHa63Dvby8KrKpEELUebW2h6+Usqcw2S/TWq8B0Fpf1FobtdYm4Av+GLZJAloV27yluUwIIYRZQW2clqmUUsBC4JjW+u/FyosvsDoGOGK+vR6YqJRyVEr5AwHA3qoLWQgh6j6TBS6tUJ5ZOr2AR4FopdQhc9kbwINKqVBAAwnA0wBa6xil1ArgKIUzfJ6XGTpCCHGjWrnEodZ6B1BSRBvL2OZd4N07iEsIIeo1o1weWQghrEOtnqUjhBCi6lyfpWMjCV8IIeo36eELIYSVqLXz8IUQQlSt69fSqVXz8IUQQlS967N0arCDLwlfCCEswag1djYKJYuYCyFE/VZg0jU6fg+S8IUQwiKMRl2jM3RAEr4QQlhEgUnX6Bx8kIQvhBAWYTRJD18IIaxC4Rh+zaZgSfhCCGEBRpNJevhCCGENZJaOEEJYCaNJY1eDi5+AJHwhhLAIo/TwhRDCOsgsHSGEsBIFJo1NDV5WASThCyGERcgYvhBCWAmZhy+EEFZC5uELIYSVKDDKLB0hhLAKMktHCCGshHzTVgghrIRJSw9fCCGsQq0cw1dKtVJKbVFKHVVKxSilZpjLPZRSPymlYs2/G5vLlVLqU6XUKaXUYaVUl+puhBBC1DW19dIKBcDLWutOQHfgeaVUJ+A14GetdQDws/k+wDAgwPwzFfi8yqMWQog6rsBkwq62zcPXWidrraPMtzOBY4APMApYYq62BBhtvj0KWKoL7QYaKaW8qzxyIYSow2prD7+IUsoPCAP2AM201snmhy4Azcy3fYBzxTZLNJfdvK+pSqn9Sqn9qampFQxbCCHqtoLaPC1TKeUKrAZe1FpfLf6Y1loDuiJPrLWer7UO11qHe3l5VWRTIYSo82ptD18pZU9hsl+mtV5jLr54fajG/DvFXJ4EtCq2eUtzmRBCCLOC2njxNKWUAhYCx7TWfy/20Hpgsvn2ZGBdsfJJ5tk63YGMYkM/QgghsEwP364cdXoBjwLRSqlD5rI3gPeAFUqpJ4EzwHjzYxuBe4FTwDXg8SqNWAgh6gGjSWNbw9fDv23C11rvAEqLamAJ9TXw/B3GJYQQ9ZpRLo8shBDWocBkqn1j+EIIIaperZ2lI4QQompZYh5+eU7aCiGE1TGZNEqBUgqtNXkFJkz6xq8b6RK+fVTSF5L0TRVNunDb2jhLRwghrILBaOKX4yms3J/I1hMpGLXG3saGfKOpWp7P0c62WvZbGkn4QgirpbXGaCrsvf/jp5N8cyiJtKx8vNwcebRHa9yc7DEYTdjb2uBoZ1PiEExJMytVCRMbb65noxQjQ1tUVVPKRRK+EMKqXM01sPFwMhrYcPg8yRm5PNClJQt2xDM0sDnjI1rSN8ALO9v6d4pTEr4QwmqcvJjJ41/uI+lKzg3lB89eobm7E/Me7WqhyGqGJHwhhFU4fuEqD87fjZ2tDcundgdg7cEkvt53jl1xaQT6NLRwhNVPEr4QwirMXHUYe1sbVj7Tg9aeLgA0dnHg633nyM430trD2cIRVr/6N0glhBA30VoTm5LFiM4tipI9gG+xJN/aUxK+EELUeenXDFzLN9KycYMbyp3sbWnR0AkA32JvBPWVJHwhRL2XmH4N4JaED+DXpDDRy5COEELUA4nphbNyWja+NalfT/h+VtDDl5O2Qoh6L8mc8H1K6OHfH+ZDA3tbGjrb13RYNU4SvhCi3ktMv4abkx0NG9ya1MP9PAj387BAVDVPhnSEEPVeYnpOicM51kYSvhCi3itM+LcO51gbSfhCiHpNa01i+jVJ+EjCF0LUcxk5BrLzjTKkgyR8IUQ9d31Kpk8j6eFLwhdC1GtlfenK2kjCF0LUKQajiQsZueWuf72H30qGdCThCyHqlj+vO0L3v/7MC/+NIq/AeMNjJpNm28lUcvL/KE9Mz8HN0Q73BvK1I3kFhBAlMpo0Zy9fw9PVAXenO/sWqtaaH2Musvv0JR7u5ktAM7cK7yPlai4bDifz9b5zhLZqxIbDybg52fO/Y4L45lASf/v+BL6ezuyNv0z3Nh58+VgkDRxsSUzPwadxA1RJaxFamdsmfKXUImAEkKK1DjKXzQamAKnmam9orTeaH3sdeBIwAtO11j9WQ9xCiGp04kImLy4/xLHkqzjY2fD5w10Y2LFZhfahtWZn3CVizmewK+4SW06kohT8e/cZHu7my5vDO5Z7Ee9cg5GJX+zmdGo2TVwdWfJEJJ9vjWPer3GcSslkX0I6LRs3YF/CZYYGNmfT0QuMnbeT+ZPCZUpmMeXp4S8G/gksvan8H1rrD4sXKKU6AROBQKAFsFkp1U5rbUQIUSekZ+fzxOJ95BWYeGdkIKujEnn2P1GsfKYHIa0alWsfp1KyeGXV7xw8ewUAFwdbZg3vyMiQFsz95RRLd53BRilmjwy87b5MJs3b62I4nZrNvEe60KNtExo2sOfVIe3JyjOwbM9Znujlz8xh7TEYNa6Odmw5nsL0rw4y9ONtZOUVcHf7pnf0mtQXt034WuttSim/cu5vFPC11joPiFdKnQIigV2VjlAIUaPe+/44qZl5rHimB6GtGjEypAUj5u5g2lcH2TC9922HdwxGE1OX7if9Wj7v3R/M0KDmONnb4mRf2JufMzoIO1vFl78l0M3fg2HB3mXu76/fH2P5/nM8378tQ4P+qGtjo5gzKog/DW5PI2cHABzNGa1/h6Z880IvXln5O8E+DXlhwF138IrUH3dy0vYFpdRhpdQipVRjc5kPcK5YnURzmRCimiVdyWHJzgROXMjkuWUHSM3Mq/A+zl/JYXVUIg9GtiLU3Jtv7OLAJxNDOX8lh+lfHcRo0gCkZeXdctIUYNnuM5xOy+aj8SFMjPSlkbNDUbK/7vVhHQlp1YhXVx3mzKXsUuM5ev4qC3fE82CkL38a3P6Wx5VSRcn+Zm29XFnzXC/eGRWEq6OcroTKJ/zPgbZAKJAMfFTRHSilpiql9iul9qempt5+AyFEmV5bfZi318fw6MI9bIy+wPs/HCfXYOT/tp4i5ertpzEaTZo5G44CMKVvmxseC/fz4C+jgth6IpX3vj/G8QtX6fneL4S8s4mXVhzi3OVraK0pMJqY9+tpuvl70L+MYRQHOxv++WAYSsHz/40i13DrG4fWmr9siKFhA3teG9pBTrpWgUq97WmtL16/rZT6AthgvpsEtCpWtaW5rKR9zAfmA4SHh+vKxCGEKLQzLo3tsWk42tmQkplHy8YNWHkgEQ2sOpDI9pNpLHuqGzY2JSdNk0nz2urDfH/kAq8P61DiZQge6ubLiQtX+WJ7PKujknBztGNIUHPWRiWxIzYNJ3tbGtjbcuFqLu+MCrxtgm7l4cxH40OZsnQ/r6+J5u/jQ27YZtPRi+w+fZk5owKt4lr1NaFSPXylVPFBtzHAEfPt9cBEpZSjUsofCAD23lmIQojb+XhzLM3cHVn8eCQDOjRl9bM9aebuyKoDiTRytmfX6Uss2HG61O3//tNJVh5IZPqAu3i6X9tS6701ohMzBgbgaGfDu2OC+N8xwXzzfC80oNGcuJiJd0MnBnYo30nSQZ2a8fKgdqw9mMTDC/aQkFY4vGM0af72/XECmrryYKRvhV4LUbryTMv8CrgbaKKUSgTeBu5WSoUCGkgAngbQWscopVYAR4EC4HmZoSNE9dqXcJm98Zf584hO9GjrSY+2ngC8ObwT0786yP+OCWb9ofN88OMJuvl73jLT5kJGLvO3n2Z0aAv+Z1C7Mp/LztaG/xnU7oZ67Zu7sWNmf+xtbNh09AKNnB2wsy1/X/KFAXfh5mTHPzbHMnbeLr6e2o2TF7M4nZbN/z3cpUL7EmVTWlt+NCU8PFzv37/f0mEIUSdNXrSXI0kZ7Jg5gAYON54cPZ2ahX8TF65cMzBi7g6y8gpY9lQ3gnwaFtWZvT6G/+w+w5Y/3U0rCy7kHXsxk4cW7MHB1gY7W4WNUmx+qR+2pQxDCVBKHdBah5e3vrx1ClHLZeQYuLljlp6dT2pmHtGJGfx6MpUnevvfkuwB2ni5opSisYsDX0/tjqujHY8s3EPM+Qyg8AtNq6MSGdHZ26LJHiCgmRuLJkdwOTufPIOJd8cESbKvYjJXSYhaLDPXwN0fbCHS34PPH+6KjY1iZ1wa0/57EAc7G3w9nHFzsuPRHq1vu69WHs58NaU7E+fv4uEFe/hqSnfiUrPIzC1gXHir225fE4JbNuTXV+/G3cn+lqmc4s5JwheiFttwOJn0awZ+jLnIPzafJC41i43RF/Bp1IDzGTkkZ+Ty/0YHlftaN76eznw9tQfj/7WLp5bsx9PVAe+GTnRv41nNLSm/pm5Olg6h3pKEL0QNu5SVR2J6TrkuU7B83znaNXOlc8tGzP3lFAAzBgYwtW8bFu9MIC4li4cqOIvF19OZeY92Zdy8naRm5vHBuM4ydGIlJOELUYNy8o08vGAPsSlZrHi6B11bNy617smLmRw6d4VZwzvySPfWXLlmoKO3W9EMmef7V/5yAaGtGrHi6R64OdlzV1PXSu9H1C2S8IWoIZm5BmZ8fYgTFzPxdHHgxeUH2Ti9D26lDMcs33cOe1vFmDAfnOxtWTC53JMxyiXMt/Q3G1E/ySwdIWrAyYuZ3P9/O/n1ZCp/GRXE5490JSk9h7fXxxCXmsUba6NvuPxBfoGJtQeTuKdjMzxdHS0YuahPpIcvRDV7fc1hvtp7jkbO9ix9IpJedzUB4IUBAXz6cyxbjqeQfs3Ab6fS+M+T3Wjl4czmYxe5nJ3P+IjaMXtG1A/SwxeiGp1KyeSrved4oEtLNv1P36JkD/DiwACGd/Ym/ZqBV4a058o1A2Pn7ST2YibL953Du6ETfQO8LBi9qG+khy9ENVqwPR5HOxveHN4RD5cbL+NrY6P4dGIYM4d0wNfTmXs6NuPRhXsY9dlv5BiMvND/Lpk9I6qU9PCFKKeMnMJhl/K6ll/AukPnGRPmc0uyv87WRuHrWfgN1/bN3Vj1TE/u6diMZ/q15ZkyLmImRGVID1+IcigwmpiyZD97Ey7z4bgQ2jVz5dVVh3llSHvC/Tx47/tjjAr1ueELTD8dvUiOwcjosPKvAeTr6cynD4ZVRxOEkIQvRHl8tiWOvQmXae3pzBtro2nr5crxC5k8tXQ/7Zu5cfxC4bj72/cFMrmnH98dTubzrXE0d3ci0s/D0uELAUjCF+K24tOy+WzLKUZ09uadkYHcN3cHx5KvMmNgAPFp2az//TxP92vD6dRs3l4fw+HEDFZHJeLqaMeL9wSUuuiIEDVNEr4Qt/Hud8dwsLPhzyM64enqyBeTw/nP7rNM6dsGFwdbnu7Xho7N3TFpzYvLD7E6KhHvhk5sfqkfLrKWqqhF5K9RiDIcOJPO5mMX+dPgdjR1L7yoV2CLhvz1/uCiOoEtCq8tb4Piw3EheLg4MCzIW5K9qHXkL1KIUmitef+H4zRxdeTxXv7l2sbJ3pa/jAqq5siEqByZlilEKbbFprEn/jLTB94lvXVRL8hfsRDFXF9ZKtdg4p1vY2jl0YCJEbKItqgfJOGLanfyYiabYi5g0jAsqDkBzdxqPIaMHAPfRyczJLA5G48kc19Ii1sWDdkZl8Yba6Jp5eGMi4Mdp1OzWfZUNxzs5IOwqB9kEXNRrY6ev8qoz3ZgMBb+nTna2fDOyEC2n0rj4Jl0xnTx4ZUhHao1hvwCE5MW7WH36cs4O9hyLd9In4AmLHosAnvbwmSemplH3/e30MTNgYsZeZi05k9D2su3XUWtVtFFzKWHL6qN0aSZufowDRvYs2FaH2xsYMZXh3htTTQA4a0b89mWOEJbNWZQp2YV2neB0cRTS/fTvY1niUlZa41ShfPf3/v+OLtPX2ZiRCs2H7vIyJAWfL3vHG+vj+Hd0UEopfjXr3HkFRhZ8ngkJq2xUYo2XrIwiKhfJOGLarP6QCLRSRl8+mAYzRsWTmn895ORfLE9nmbujozo3IKR/9zBm2ujifT3oGGD8q3LCrDyQCJbT6SyPTaNu9t70aG5O6dSMrmWb2TbyVQ2HE7mqyndOZZ8lUW/xTO5R2veGRVU9Ebg6erAZ1viyDUYebR7a5buPsOYsJaS5EW9JkM6olpk5hoY9PdtNG/oxNrnehb1tm8WnZjB6P/7jQ7N3ejaujFv3xd42ytEbj2RwoyvD9Ha05nE9By8XB2ZNaIjz/0nCqPWaA05BiM923qSmVvA5ex8fn65H072tkX7MJk0H28+yWdb4zCaNO5Odmx+uZ8soC3qlOs2Aw8AABIQSURBVIoO6cjZKFHlTCbNyyt+JzUrj7dGdCo12QMEt2zIS4PacTk7n6W7zvDyikN8vPkkmbkG8gqM/HT0IvkFpqL68WnZPLVkP94Nnfh0YhhzHwwjLjWLRxfuLZo6mWMw8nz/tuw6fYnopAxmDAy4IdlD4aWJXxrcnnXP96L3XU346/2dJdmLek96+KLKzf05lo9+OslbIzrxZO/yfWFJa82Mrw+x/vfzAPh5OtPM3Yk98Zfp4tuILyaF8110Mj8cucChc1f49ZX+eLkVLv23PTaV5IxcBnZoStTZKySkZTOlbxt+jLnA1hOpzBkViJ2t9G1E/VPRHv5tE75SahEwAkjRWgeZyzyA5YAfkACM11qnq8Ku3CfAvcA14DGtddTtgpCEX3/sjb/MhPm7GB3qw9/Hh5TZu79ZfoGJs5ezScvK58/rjnDyYhYPd/Nl5f5E3BvYk5aVB8D0gQG8NKhddTVBiDqjOmbpLAb+CSwtVvYa8LPW+j2l1Gvm+zOBYUCA+acb8Ln5t7AC2XkFvL0+hhYNG/DumKAKJXsABzsb7mrqxl1N4ccX+5KWlY+XmyMdmrvx1roYHoxsxdS+bfH1cK6mFghRv9024Wuttyml/G4qHgXcbb69BNhKYcIfBSzVhR8bdiulGimlvLXWyVUVsKiddp++xONf7iPHYOTzh7vg7HBnE8CUUkVDNo90b02Yb2M6NHeToRkh7kBl/yubFUviF4Drk6h9gHPF6iWay25J+EqpqcBUAF9f+ep6XWY0aWavj6GJmwN/Hx9KRBUv+KGUIsinYZXuUwhrdMfdJXNvvsJnfrXW87XW4VrrcC8vrzsNQ1jQpz/HcvxCJq8O6VDlyV4IUXUqm/AvKqW8Acy/U8zlSUCrYvVamstEPfXZllN88nMs93fxYXiwt6XDEUKUobIJfz0w2Xx7MrCuWPkkVag7kCHj9/XXukNJfPDjCcaE+fDh2BBZyk+IWu62Y/hKqa8oPEHbRCmVCLwNvAesUEo9CZwBxpurb6RwSuYpCqdlPl4NMYta4MylbN5ce4QIv8b87YHOkuyFqAPKM0vnwVIeGlhCXQ08f6dBidpLa8222DReWfk7SsE/JoTK5YOFqCPk4mnitrLyCjhx4SqZuQUs3BHP9tg0Wns6s/TJSFo2ljnxQtQVkvBFmfIKjNw3dwfxadkAuDjY8ucRnXi4uy+Odra32VoIUZtIwhdlWrrzDPFp2cwZFUj75u609XLB09XR0mEJISpBEr4o1cboZD748QR3t/fi0R5+lg5HCHGH5GybKNHVXAMvrThEkI87H08ItXQ4QogqIAlflGj9ofPkGkzMHhlII2cHS4cjhKgCkvDFLbTWrNh/jg7N3QiWa9gIUW9Iwhe32Bt/mcOJGTzczbfClzgWQtRekvBFkau5BgxGE3N/OUUTVwfGhbe6/UZCiDpDZukIAGIvZjL80x2gCleemjW84y3rwAoh6jZJ+AKAub+cws5WMSbMh37tvBjUqdntNxJC1CmS8K1crsHI51vj+Pbweab2bcPrwzpaOiQhRDWRhG/Fjl+4ynPLojidms3wYG+e73+XpUMSQlQjSfhWJtdgZE1UEr8cv8jOuEu4Otrxnye70TugiaVDE0JUM0n4ViQuNYspS/ZzOi0bP09nhgQ257VhHWjm7mTp0IQQNUASvhWZ+3MsqZl5LH0ikj4BTWSOvRBWRhK+lcgvMPHzsRSGBjWnbztZNF4IayRfvLISO+PSyMwrYFhwc0uHIoSwEEn4VuBqroEPN53AzcmOnm3l5KwQ1kqGdOqpA2fSMWmNh4sDzy+L4lRKFvMndZVvzwphxSTh1zPXFxf/9vdkcgxG7GwUrk52LHwsgn4ydi+EVZOEX4+cuZTNygOJAHi4OPBQN19yDUZmDAygqUy9FMLqScKvRzZGXwDgo3EhdGrhTkdvdwtHJISoTSTh1xNaa76LPk9Iq0Y80LWlpcMRQtRCMkunnth1+hJHkq7yQBcfS4cihKilJOHXAyaT5uOfYmnq5sh4WbRECFGKO0r4SqkEpVS0UuqQUmq/ucxDKfWTUirW/Ltx1YQqSvPWuiPsTbjMi/e0k2mXQohSVUUPv7/WOlRrHW6+/xrws9Y6APjZfF9UkxMXMlm25yxP9PLnoW6+lg5HCFGLVcdJ21HA3ebbS4CtwMxqeB6rlmswsmRnAicuZGJro3i+f1tLhySEqOXuNOFrYJNSSgP/0lrPB5pprZPNj18ASlwrTyk1FZgK4OsrPdOK0Frz9roYlu8/B8CADk3xdHW0cFRCiNruThN+b611klKqKfCTUup48Qe11tr8ZnAL85vDfIDw8PAS64hbaa35343HWL7/HGO7tuT4has81dvf0mEJIeqAO0r4Wusk8+8UpdRaIBK4qJTy1lonK6W8gZQqiFOYbTiczBfb45nUozWz7wvExkauaS+EKJ9KJ3yllAtgo7XONN8eDPwFWA9MBt4z/15XFYFau73xl3l11e8UmDRtvVwk2QshKuxOevjNgLXmVZPsgP9qrX9QSu0DViilngTOAOPvPEyxYPtpEi5dA+DDcSGS7IUQFVbphK+1Pg2ElFB+CRh4J0GJG6Vl5fHL8RQe6uZL77uaMCRQFjERQlScXEunFtoUc4F9CZd5dWgHjiRl8MJ/D2LUmsd7+hHQzM3S4Qkh6ihJ+LVE1Nl07GwUnVs24pOfY4k5f5XzV3JJvJKD0aRZ8nikJHshxB2RhF8LpGTmMmnhXgpMJj6eEErM+asEtnDnu+jCrzP89f5gWXhcCHHHJOHXAh/8cIK8AiPN3J145j9RhWVjQ9gZl8a22DQe6CKXOxZC3DlJ+BZiMJp4eMEe2jRxYXVUIk/29ueR7q154POdONrZ0tHbjU4t3HmqTxtLhyqEqCck4VvIxuhk9sZfZm/8ZVwcbHn27rvwcHFgw7Q+5BiMmKe7CiFElZGEbwEFRhOfb42jrZcLHbzd6d7GEw8XBwCaN5S1Z4UQ1UMSvgW8/+MJjl/I5J8PhTGicwtLhyOEsBKy4lUNi72Yyfxtp3m4m68keyFEjZKEX8OW7ErAwc6Glwa1s3QoQggrIwm/BmXkGFh9IIlRIS3k+vVCiBonCb8Grdh3jhyDkcd6+Vk6FCGEFZKEX0PyC0ws2ZVApL8HgS0aWjocIYQVkoRfA3INRh5duIfE9BymyhephBAWIgn/DmTnFZCZa7htvY3RyeyJv8zfHgjmnk4lLvErhBDVTubhV1JqZh7j5u0kNTOP8RGtmBjhS/vmJV/N8tvfz+PTqAHjw1vVcJRCCPEH6eFXwOXsfAxGExk5BiYt2svFq3n0DmjCst1nGfv5TtKy8sgrMN6wzZVr+WyPTWN4Z2+5XIIQwqKkh19OWXkFDPxoK/d0bMb5jBxOpWSyYHIE/dp5cSoli6Efb6P/h1txtLPlxxf7FE27/PK3BApMmtGhPhZugRDC2kkPv5xWH0gk/ZqBlQcS+e3UJf4yKoh+5mvU39XUlaf7tcHV0Y70a/l89NNJoHDY54vtpxke7E2nFu6WDF8IIaSHXx5Gk2bxzgQ6ebuTlpVHkE9DJkbcOB7/ypAOvDKkA3/59ihf7oxnZEgLDide4Vq+kZcGy7dqhRCWJwm/HL45mER8WjafPdSFPu2a4OJgV+p4/MuD27HlRAovLT+EewN7gn0a0tbLtYYjFkKIW8mQThl+OHKBAR9t5a11Rwhs4c6woOa4O9lja1P6yVcXRzs+nhBKSmYexy9kMjSoeQ1GLIQQpZOEf5OkKzm8viaaDYfP88x/DuBga0OPNp78ZVQQNmUk+uJCWjXipcHtsLNRDA/2ruaIhRCifJTW2tIxEB4ervfv31/jz7ty/znW/36et+8L5K6mrphMmolf7GZv/GUAmro58usr/WngYFup/V/Ozi9a2EQIIaqaUuqA1jq8vPWtdgw/5nwGb649Qr7RxIi525l9XyAGk2Zv/GX6tfPi15OpTBsYUOlkD0iyF0LUKlaT8FcdSMRGQWtPFy5l5fHeD8dp7GLPkici+X8bjvHammgcbG3o5u/B4scjOH4hkw6lfHNWCCHqompL+EqpocAngC2wQGv9XnU9V0kOJ14h5WoeGw6fx9bGhtVRiTfFB8ue6kaH5u4seSKSd76NYeX+RGaPDEQpRUdvmTcvhKhfqmUMXyllC5wEBgGJwD7gQa310ZLq3+kYvtGkuZydT2augdaeLqw9mMSfVv4OgIuDLdcMRgKaunJ/l5YUGE34N3FFKbj3phOquQYjTvaVH8IRQoiaVFvG8COBU1rr0+agvgZGASUm/MraGJ3MtK8OYjT98abl7GDLtXwjve7yZNqAADo0dyP9moFGDexpfJsxdUn2Qoj6rLoSvg9wrtj9RKBb8QpKqanAVABfX99KPUkbLxee6dcGWxsbPF0ccLK34dC5K3Ro7s648JY4OxQ2r5GznDwVQgiLnbTVWs8H5kPhkE5l9tGhuTsdmt841j4honJvHkIIUd9V1xevkoDiF5tpaS4TQghhIdWV8PcBAUopf6WUAzARWF9NzyWEEKIcqmVIR2tdoJR6AfiRwmmZi7TWMdXxXEIIIcqn2sbwtdYbgY3VtX8hhBAVIxdPE0IIKyEJXwghrIQkfCGEsBKS8IUQwkrUiuvhK6VSgTOV3LwJkFaF4dQG0qbar761B6RNdcHN7WmttfYq78a1IuHfCaXU/opcPKgukDbVfvWtPSBtqgvutD0ypCOEEFZCEr4QQliJ+pDw51s6gGogbar96lt7QNpUF9xRe+r8GL4QQojyqQ89fCGEEOUgCV8IIaxEnU74SqmhSqkTSqlTSqnXLB1PZSmlEpRS0UqpQ0qp/eYyD6XUT0qpWPPvxpaOszRKqUVKqRSl1JFiZSXGrwp9aj5mh5VSXSwXeelKadNspVSS+TgdUkrdW+yx181tOqGUGmKZqEunlGqllNqilDqqlIpRSs0wl9fZ41RGm+rycXJSSu1VSv1ubtM75nJ/pdQec+zLzZedRynlaL5/yvy4X5lPoLWukz8UXnY5DmgDOAC/A50sHVcl25IANLmp7H3gNfPt14C/WTrOMuLvC3QBjtwufuBe4HtAAd2BPZaOvwJtmg38qYS6ncx/f46Av/nv0tbSbbgpRm+gi/m2G3DSHHedPU5ltKkuHycFuJpv2wN7zK//CmCiuXwe8Kz59nPAPPPticDysvZfl3v4RQula63zgesLpdcXo4Al5ttLgNEWjKVMWuttwOWbikuLfxSwVBfaDTRSSnnXTKTlV0qbSjMK+Fprnae1jgdOUfj3WWtorZO11lHm25nAMQrXnq6zx6mMNpWmLhwnrbXOMt+1N/9oYACwylx+83G6fvxWAQOVUqq0/dflhF/SQullHezaTAOblFIHzIu7AzTTWiebb18AmlkmtEorLf66ftxeMA9xLCo2zFan2mT+2B9GYe+xXhynm9oEdfg4KaVslVKHgBTgJwo/iVzRWheYqxSPu6hN5sczAM/S9l2XE3590ltr3QUYBjyvlOpb/EFd+Hmtzs6frevxF/M50BYIBZKBjywbTsUppVyB1cCLWuurxR+rq8ephDbV6eOktTZqrUMpXAs8EuhQVfuuywm/3iyUrrVOMv9OAdZSeJAvXv8Ibf6dYrkIK6W0+OvscdNaXzT/M5qAL/hjOKBOtEkpZU9hYlymtV5jLq7Tx6mkNtX143Sd1voKsAXoQeGQ2vUVCovHXdQm8+MNgUul7bMuJ/x6sVC6UspFKeV2/TYwGDhCYVsmm6tNBtZZJsJKKy3+9cAk8yyQ7kBGsSGFWu2mMewxFB4nKGzTRPOMCX8gANhb0/GVxTyuuxA4prX+e7GH6uxxKq1Ndfw4eSmlGplvNwAGUXhuYgsw1lzt5uN0/fiNBX4xf1IrmaXPSt/hGe17KTwzHwe8ael4KtmGNhTOHPgdiLneDgrH4X4GYoHNgIelYy2jDV9R+NHZQOH44pOlxU/hLITPzMcsGgi3dPwVaNO/zTEfNv+jeRer/6a5TSeAYZaOv4T29KZwuOYwcMj8c29dPk5ltKkuH6fOwEFz7EeAP5vL21D45nQKWAk4msudzPdPmR9vU9b+5dIKQghhJerykI4QQogKkIQvhBBWQhK+EEJYCUn4QghhJSThCyGElZCEL4QQVkISvhBCWIn/D88sgbILvtXEAAAAAElFTkSuQmCC\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "plt.plot(score_arr2, label ='accumulated scores in one game play 1')\n",
        "plt.legend(loc='upper left')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "hpeJyIbOeY2q"
      },
      "outputs": [],
      "source": [
        "## game play video 2\n",
        "agent.load_state_dict(torch.load(\"weights.pt\"))\n",
        "env = gnwrapper.LoopAnimation(gym.make(game_name)) \n",
        "state = env.reset()\n",
        "state_dim = len(state)\n",
        "game_score = 0\n",
        "score_arr2 = []\n",
        "state_arr = []\n",
        "for i in range(1000):\n",
        "    if i == 0:\n",
        "        stacked_state = np.concatenate((state, state, state, state, state, state, state, state), axis=0)\n",
        "    else:\n",
        "        stacked_state = np.roll(stacked_state,-state_dim,axis=0)        \n",
        "        stacked_state[-state_dim:]=state\n",
        "    with torch.no_grad():\n",
        "        hs = agent.representation_network(torch.from_numpy(stacked_state).float().to(device))\n",
        "        P, v = agent.prediction_network(hs)\n",
        "        action = np.random.choice(np.arange(agent.action_space), p=P.detach().numpy())   \n",
        "    if i %5==0:\n",
        "        env.render()\n",
        "    state, r, done, info = env.step(action.item())\n",
        "    print(r, done, info)\n",
        "    state_arr.append(state[0])\n",
        "    game_score += r \n",
        "    score_arr2.append(game_score)\n",
        "    if i %10==0: \n",
        "        print(game_score)\n",
        "    if done:\n",
        "        print('last frame number : ',i)\n",
        "        print('score :', game_score)\n",
        "        print(state_arr)\n",
        "       \n",
        "        break\n",
        "env.reset()\n",
        "env.display()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ArdvgJqDediI",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 286
        },
        "outputId": "10deff31-9ab4-4f27-ed12-5fd939b054cd"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.legend.Legend at 0x7fcdd53e9b20>"
            ]
          },
          "metadata": {},
          "execution_count": 8
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD7CAYAAACRxdTpAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deVxU9f748ddnWEVRRHHBDVDcEZDFfd/N1EzbbqVtdlu0fnXL6trNtm/dbO9WZlczy1uampq2uGvlvosbm6ggioossgjMfH5/zDCBgIA4Dozv5+PBgzOf8zln3nMc35zzOZ/z+SitNUIIIRyLwd4BCCGEuP4kuQshhAOS5C6EEA5IkrsQQjggSe5CCOGAJLkLIYQDKje5K6VaKKU2KKUOK6UOKaWespTPUEolKaX2WX5GFtnmRaVUrFLqmFJqmC0/gBBCiJJUef3clVJNgaZa6z1KKU9gNzAWuAO4pLV+94r6HYHvgEjAF1gLtNVaG20QvxBCiFI4l1dBa50MJFuWM5VSR4BmV9lkDPC91voycFwpFYs50W8ta4OGDRtqPz+/ysQthBA3vd27d5/XWvuUtq7c5F6UUsoPCAW2A72AJ5VS9wO7gGe11hcxJ/5tRTZL5Op/DPDz82PXrl2VCUUIIW56SqkTZa2r8A1VpVQdYAnwtNY6A/gcaA2EYD6zf6+SQU1WSu1SSu06d+5cZTYVQghRjgold6WUC+bEvkBrvRRAa31Wa23UWpuALzE3vQAkAS2KbN7cUlaM1nq21jpcax3u41PqVYUQQohrVJHeMgqYAxzRWr9fpLxpkWq3AVGW5RXAXUopN6WUPxAI7Lh+IQshhChPRdrcewH3AQeVUvssZS8BdyulQgANJACPAmitDymlFgGHgQLgiWvpKZOfn09iYiK5ubmV3VQIcYO4u7vTvHlzXFxc7B2KuEK5XSFvhPDwcH3lDdXjx4/j6elJgwYNMF88CCGqE601Fy5cIDMzE39/f3uHc1NSSu3WWoeXtq7aPqGam5sriV2IakwpRYMGDeTqupqqtskdkMQuRDUn/0err2qd3IUQwpF9uDaazdG26Qouyb2GmjFjBu++++5V6yxbtozDhw9Xet916tS51rBumJEjR5KWlmbvMKqljRs3MmrUqCrvZ82aNYSFhREUFERYWBjr16+/DtGJoj7dEMvW+As22XelnlAVNcuyZcsYNWoUHTt2tHcoJRQUFODsfO1fv59//vk6RiNK07BhQ3766Sd8fX2Jiopi2LBhJCWVeGRFVIFJg5ONmrbkzP0qxo4dS1hYGJ06dWL27NnW8l9//ZWuXbsSHBzMoEGDALh06RIPPPAAQUFBdOnShSVLlgDFz4IXL17MpEmTAJg0aRKPPfYY3bt3JyAggI0bN/Lggw/SoUMHa52rbV/Ul19+SUREBMHBwdx+++1kZ2ezZcsWVqxYwXPPPUdISAhxcXHExcUxfPhwwsLC6NOnD0ePHgXMPZN69OhBUFAQ06dPL/VYZGVlccsttxAcHEznzp1ZuHAhADt37qRnz54EBwcTGRlJZmYmubm51mMRGhrKhg0bAJg3bx6jR49m4MCBDBo0iKysLB588EEiIyMJDQ1l+fLlABw6dIjIyEhCQkLo0qULMTExJeLx8/Pj/PnzJCQk0KFDBx555BE6derE0KFDycnJKVE/ISGBgQMH0qVLFwYNGsTJkyet/w5Tp06lZ8+eBAQEsHjxYus2M2fOJCIigi5duvDKK6+UelzmzJlD27ZtiYyM5JFHHuHJJ58E4KeffqJbt26EhoYyePBgzp49C5ivuCZOnEifPn1o1aoVS5cu5fnnnycoKIjhw4eTn58PwO7du+nXrx9hYWEMGzaM5OTkEu89adIk/v73vxMeHk7btm1ZuXJliTo7duygR48ehIaG0rNnT44dOwZA37592bdvn7Ve79692b9/f7FtQ0ND8fX1BaBTp07k5ORw+fLlUo+DuDZGk8Zgo9sWNeLM/dWfDnH4dMZ13WdH37q8cmunq9aZO3cu3t7e5OTkEBERwe23347JZOKRRx5h8+bN+Pv7k5qaCsDrr79OvXr1OHjwIAAXL14sN4aLFy+ydetWVqxYwejRo/nzzz/573//S0REBPv27SMkJKRCn2XcuHE88sgjAEyfPp05c+YwZcoURo8ezahRoxg/fjwAgwYNYtasWQQGBrJ9+3Yef/xx1q9fz1NPPcVjjz3G/fffz6efflrqe/z666/4+vqyatUqANLT08nLy+POO+9k4cKFREREkJGRQa1atfjoo49QSnHw4EGOHj3K0KFDiY6OBmDPnj0cOHAAb29vXnrpJQYOHMjcuXNJS0sjMjKSwYMHM2vWLJ566in+9re/kZeXh9F49cckYmJi+O677/jyyy+54447WLJkCffee2+xOlOmTGHixIlMnDiRuXPnMnXqVJYtWwZAcnIyf/zxB0ePHmX06NGMHz+e1atXExMTw44dO9BaM3r0aDZv3kzfvn2t+zx9+jSvv/46e/bswdPTk4EDBxIcHAyYk+W2bdtQSvHf//6Xd955h/feM4/QERcXx4YNGzh8+DA9evRgyZIlvPPOO9x2222sWrWKW265hSlTprB8+XJ8fHxYuHAh//znP5k7d26Jz56QkMCOHTuIi4tjwIABxMbGFlvfvn17fv/9d5ydnVm7di0vvfQSS5Ys4aGHHmLevHl8+OGHREdHk5uba429NEuWLKFr1664ubld9d9CVJzJZO6GbrBRdq8Ryd1ePv74Y3788UcATp06RUxMDOfOnaNv377Wfr3e3t4ArF27lu+//966bf369cvd/6233opSiqCgIBo3bkxQUBBgPktKSEiocHKPiopi+vTppKWlcenSJYYNKzmE/qVLl9iyZQsTJkywlhWehf3555/WK4377ruPadOmldg+KCiIZ599lmnTpjFq1Cj69OnDwYMHadq0KREREQDUrVsXgD/++IMpU6YA5uTSqlUra3IfMmSI9ZitXr2aFStWWO8d5ObmcvLkSXr06MGbb75JYmIi48aNIzAw8Kqf39/f33qswsLCSEhIKFFn69atLF261PoZn3/+eeu6sWPHYjAY6Nixo/UMe/Xq1axevZrQ0FDr8YuJiSmW3Hfs2EG/fv2sn2fChAnWz5mYmMidd95JcnIyeXl5xfqBjxgxAhcXF4KCgjAajQwfPtx6jBMSEjh27BhRUVEMGTIEAKPRSNOmRR8I/8sdd9yBwWAgMDCQgIAA69VYofT0dCZOnEhMTAxKKeuVwYQJE3j99deZOXMmc+fOLfWKsNChQ4eYNm0aq1evLrOOqDyj5RkjWzXL1IjkXt4Zti1s3LiRtWvXsnXrVjw8POjfv/819ect2lXsyu0Lz4IMBkOxMyKDwUBBQUG52xeaNGkSy5YtIzg4mHnz5rFx48YSdUwmE15eXsUuxcuKszRt27Zlz549/Pzzz0yfPp1BgwZx2223XXWb0tSuXdu6rLVmyZIltGvXrlidDh060K1bN1atWsXIkSP54osvGDhwYJn7LHrsnJycSm2WuZqi2xc+1Ke15sUXX+TRRx+t1L4KTZkyhWeeeYbRo0ezceNGZsyYUeL9DAYDLi4u1mNf+O+utaZTp05s3VrmKNlWV/67Xfn65ZdfZsCAAfz4448kJCTQv39/ADw8PBgyZAjLly9n0aJF7N69u9T9JyYmcttttzF//nxat25d0Y8vKsBo4zN3aXMvQ3p6OvXr18fDw4OjR4+ybZt5FOPu3buzefNmjh8/DmBtlhkyZEixJo3CZpnGjRtz5MgRTCaT9SqgMiqyfWZmJk2bNiU/P58FCxZYyz09PcnMzATMZ9X+/v788MMPgDl5Fbax9urVy3rVUXT7ok6fPo2Hhwf33nsvzz33HHv27KFdu3YkJyezc+dOaxwFBQX06dPHup/o6GhOnjxZIoEDDBs2jE8++cSaUPfu3QtAfHw8AQEBTJ06lTFjxnDgwIHKHbRS9OzZs9hn7NOnz1XrDxs2jLlz53Lp0iUAkpKSSElJKVYnIiKCTZs2cfHiRQoKCqxXP2D+/jRrZh7p+uuvv65UrO3atePcuXPW5J6fn8+hQ4dKrfvDDz9gMpmIi4sjPj6+xHEuGse8efOKrXv44YeZOnUqERERpV5ppqWlccstt/D222/Tq1evSn0GUb7CwQGcJLnfWMOHD6egoIAOHTrwwgsv0L17dwB8fHyYPXs248aNIzg4mDvvvBMwt3VfvHiRzp07ExwcbL2J+PbbbzNq1Ch69uxZ5qX11VRk+9dff51u3brRq1cv2rdvby2/6667mDlzJqGhocTFxbFgwQLmzJlDcHAwnTp1st7A/Oijj/j0008JCgoqszfEwYMHrTc5X331VaZPn46rqysLFy5kypQpBAcHM2TIEHJzc3n88ccxmUwEBQVx5513Mm/evFLbal9++WXy8/Pp0qULnTp14uWXXwZg0aJFdO7cmZCQEKKiorj//vsrfdyu9Mknn/DVV1/RpUsXvvnmGz766KOr1h86dCj33HOP9Ubz+PHjrX8oCzVr1oyXXnqJyMhIevXqhZ+fH/Xq1QPMN04nTJhAWFgYDRs2rFSsrq6uLF68mGnTphEcHExISAhbtmwptW7Lli2JjIxkxIgRzJo1C3d392Lrn3/+eV588UVCQ0OtV4OFwsLCqFu3Lg888ECp+/7Pf/5DbGwsr732GiEhIYSEhJT4AyeuXWGzjK1uqKK1tvtPWFiYvtLhw4dLlAlR3WRmZmqttc7Pz9ejRo3SS5cuvWHvPXHiRP3DDz9c8/ZJSUk6MDBQG43GKsUh/1evTVp2nm41baX+cnPcNe8D2KXLyKty5i5EFcyYMYOQkBA6d+6Mv78/Y8eOtXdIFTJ//ny6devGm2++icEgacAeCnvL2KpZpkbcUBWiuirvKWFburINvTLuv//+69LcJa6dSds2uVfrP9m6GgxHLIQom/wfvXaFbe62Gnyt2iZ3d3d3Lly4IF8eIaopbRnP/cqbuKJiTCbz75uun3vz5s1JTExEJs8WovoqnIlJVJ71ISYbnWJX2+Tu4uIis7sIIRyWdfiBm61ZRgghHFnhDVVJ7kII4UCMNu4KKcldCCHswHrmLsldCCEch9HGvWUkuQshhB2YbNxbRpK7EELYQWGb+033EJMQQjgy65m7JHchhHAc0ltGCCEckCW3S28ZIYRwJH89xGSb/UtyF0IIO7A2y0ibuxBCOA6TTJAthBCOx3gzT9YhhBCOynpD1V7NMkqpFkqpDUqpw0qpQ0qppyzl3kqpNUqpGMvv+pZypZT6WCkVq5Q6oJTqapPIhRCiBvtryF/b7L8iZ+4FwLNa645Ad+AJpVRH4AVgndY6EFhneQ0wAgi0/EwGPr/uUQshRA1n937uWutkrfUey3ImcARoBowBvrZU+xoonPZ9DDBfm20DvJRSTa975EIIUYMZq9N47kopPyAU2A401lonW1adARpblpsBp4pslmgpu3Jfk5VSu5RSu2QqPSHEzUZXlxuqSqk6wBLgaa11RtF12hxlpWay1lrP1lqHa63DfXx8KrOpEELUeIVD/tr1zF0p5YI5sS/QWi+1FJ8tbG6x/E6xlCcBLYps3txSJoQQwsLWE2RXpLeMAuYAR7TW7xdZtQKYaFmeCCwvUn6/pddMdyC9SPONEEIIbD9BtnMF6vQC7gMOKqX2WcpeAt4GFimlHgJOAHdY1v0MjARigWzggesasRBCOABb95YpN7lrrf8Aynr3QaXU18ATVYxLCCEcmqk69ZYRQghxfcgE2UII4YBkgmwhhHBA1oeYZIJsIYRwHFrmUBVCCMdjtHFXSEnuQghhB0aZrEMIIRyPqbqMLSOEEOL6kd4yQgjhgArP3G2U2yW5CyGEPZjsPVmHEEKI688oXSGFEMLxmKS3jBBCOB6Ttl2TDEhyF0IIuzBqjQ1zuyR3IYSwB5NJ2+zpVJDkLoQQdmE0aWmWEUIIR2PStuspA5LchRDCLkxa2+wBJpDkLoQQdiHNMkII4YCMWpK7EEI4HOktI4QQDsikJbkLIYTDMZrkCVUhhHA4Jq1tNjk2SHIXQgi7MJq09HMXQghHYz5zl+QuhBAORW6oCiGEA5JmGSGEcEBGk+0m6gBJ7kIIYRcmrXGS3jJCCOFY7N7mrpSaq5RKUUpFFSmboZRKUkrts/yMLLLuRaVUrFLqmFJqmK0CF0KImsxYDYYfmAcML6X8A611iOXnZwClVEfgLqCTZZvPlFJO1ytYIYRwFCZ7Dxymtd4MpFZwf2OA77XWl7XWx4FYILIK8QkhhEOqzr1lnlRKHbA029S3lDUDThWpk2gpK0EpNVkptUsptevcuXNVCEMIIWoek6ZaDj/wOdAaCAGSgfcquwOt9WytdbjWOtzHx+cawxBCiJqpWg75q7U+q7U2aq1NwJf81fSSBLQoUrW5pUwIIUQR1XKyDqVU0yIvbwMKe9KsAO5SSrkppfyBQGBH1UIUQgjHY+szd+fyKiilvgP6Aw2VUonAK0B/pVQIoIEE4FEArfUhpdQi4DBQADyhtTbaJnQhhKi5TNq247mXm9y11neXUjznKvXfBN6sSlBCCOHozP3cbbd/eUJVCCHswO5PqAohhLj+jCbb3lAtt1lGCCFuZiaTZu2Rs/wadQaj1hhNmpw8I5cLTNY6Gv3X8l+LxZeL1AE4dTGbtk08bRa3JHchhCjDwcR0pi05wOHkDLxru1LHzRlng8LdxQk3F0OxZpWi5+BXtraoomsti12aezGyc1NsRZK7EEKUIiM3n0e/2YVRa96dEMzYEF+cbTlG73UmyV0IIa6QlJbDI1/v4kxGLkse60loy/rlb1TNSHIXQgjg2JlM4s9dQgNv/XKEtOx85kyMqJGJHSS5CyEEvxxMZur3e8k3mm96+tZz59uHuhHcwsvOkV07Se5CiJva8fNZPLNoP0HN6jFjdCdy8010aV4Pd5eaPRWFJHchxE0r32ji6YX7cHU28OnfutK0Xi17h3TdSHIXQtyUjCbN9B+j2H8qjU/vcazEDvKEqhDiJrXxWAoLd53iiQGtuaWL7fqb24skdyHETenEhWwAHuodYOdIbEOSuxDippSSeRkXJ0V9Dxd7h2ITktyFEDellMxcGnm6o2w4MqM9SXIXQtyUUjIu4+PpZu8wbEaSuxDippSSmUvjupLchRDCoaRkXqaRp7u9w7AZSe5CiJtObr6RtOx8GkmzjBBCOI5zmZcBaFxXztyFEMJhpFiSu4+0uQshhONIycgFkGYZIYS43rIuF3DsTCY5ecYb/t6FZ+6OfENVBg4TQlTJt9tOkJyew3PD2ld4G601D8zbyY7jqdSr5UL7Jp68MKJ9hSbGyCswYdK60kPybo+/gJNBEe7nTUpmLk4GRYParpXaR00iZ+5CiGtmMmk+XhfDpxviiDmbWeHtVh8+y47jqYwJ8aVfWx+2H0/l0w2xFdr279/uZtxnWzCZdLl1C4wmYlMu8dHaGO6cvY3xs7YSm3LJ/ABTHTcMBsd8OhXkzF0IcRUFRhMnUrNp7VOn1PV7Tl60NnF8sTmedycEV2i/n22Mw79hbd6bEIyzk4Hm9Wsxa1McZ9JzaVKv7KaSPScvsv5oCgCrDibTo3UD/ow9T4SfN75exYfsvZiVx71ztnPodAYA4a3qcyQ5g38tj8LZyUAjB76ZCnLmLoS4io/XxTD0g82cSs0use7S5QJm/nYMVycDY0J8Wbw7kX8tjyL+3KWr7nPPyYvsP5XGg738cHYyp6A7I1pg0vDDrlPWeodPZ3DiQhbb4y+Qk2ckNSuPZxftp0FtV9o0qsMbqw7T750NPPX9Pvq/u5GopHQAlu9L4veYc3y4NpqjZzIZ2rEx9T1c+OSeUP7fkLZsibvAnhMXHbq9HeTMXQhRhszcfL7akoDRpFl5IJnH+rcutn7en8fZfjyV9+8IJrCRJ8v3nWb+1hPEnbvE7PvCqe1WenpZvDsRD1cnxnVtbi1r1aA2vdo04Pudp3hiQBsMBsXIj3+3rr87siWZufkkpeXwv4e7YTAoxn++heb1Pfjv7V14fMFu3v7lKO/fGcxzPxzAw82JS7kF3BXRgjdvC6LAaMLZycDdkS35z4ZY8wNMcuYuhLgZfbvtJJm5BTSp686K/adLrN8cfZ7OzeoyrmtzgprXY3LfAPwb1ubP2At0euU37pi1ley8AsD8ROgzC/exLf4Cv0WdYWD7RiWS/92RLUlKy+HL3+P56s/j1nKl4LsdJ1l5IJkn+rch3M+bri3rs+jRHix+rAc9WjfgiQFt+CP2PE9/v488o4m07Hzq13bluWHtAKxXCLXdnJnU0w9w7G6QIGfuQgiLbfEXOH/pMqO6+JKbb2TOH8fpE9iQwR0a88qKQ+w/lUbbxp5k5ubj4ebMnpMXebjPXxNdvDSyA4/1a82LSw/SpJ4787YkMPeP4zw5MJCFO0+xdG8SS/cmAXBLUMmZj4Z1akJtVyfe+uWotezZIW3p09aH91Yfo6NvXf7e/6/3C/fzti7fEdGCN1YdYUvcBcaFNiO4hRehLb3w8ijZG2ZSTz9+O3SW8FbeJdY5EknuQggA3vr5CIdOZ9C+SV22WhL9Y/1D6NysHv/+9Shfb0lg+/FUktJy6ObvTYFJ0yewYbF91K/tyqz7wgA4nZbDrE3xRPh588n6GJrXr8XptBxGdG7KoA6NS7y/i5OBaSPa88bKI+QZTQA81r81zk4Gvnmo21Vjr+vuwgO9/Fi8K5F/3tKBBnXKPiv38nDll6f6VPbw1DhK66t3J1JKzQVGASla686WMm9gIeAHJAB3aK0vKvOo9x8BI4FsYJLWek95QYSHh+tdu3ZV4WMIIaoiIzefkFdXY9LQJ7AhCReyaFDbjR8f74lSipd+PMj/tp8stk3bxnX45am+OJXRnTA25RKD398EgI+nGwse7kbjuu7UdXe+6gQZuflGFu48RXpOPlMHBVb4MxhNmrwCE7VcK9f/vSZTSu3WWoeXtq4ibe7zgOFXlL0ArNNaBwLrLK8BRgCBlp/JwOfXErAQ4sbaEZ+KScPgDo35PeY8p1JzeHZoW2sSvr9HKwA8XJ1Y8WQvmtevxcujOpaZ2AHaNKrDU4MCGdyhEcue6EXbxp7Uq+VS7sxH7i5OTOzpV6nEDuBkUDdVYi9Puc0yWuvNSim/K4rHAP0ty18DG4FplvL52nw5sE0p5aWUaqq1Tr5eAQshqq7wir0w0f4Rex5XZwMf3BnMY9/uoUfrBvQJ9LHWb9+kLiODmtC0Xi26NPfi9+cHVGh6uv83pK1tPoAo17W2uTcukrDPAIUNaM2AU0XqJVrKJLkLUU1orek7cwOD2jdmxuhOaK1Zc/gsfQMb4unuwrcPl96+/dnfwqzLjjrvqCOp8g1VrbVWSpX/HPAVlFKTMTfd0LJly6qGIYSooCPJmZxKzWHelgQuZOUxqktTktJyeGpw5ZpBRPV2rf3czyqlmgJYfqdYypOAFkXqNbeUlaC1nq21Dtdah/v4+JRWRQhRCenZ+aw6kEx5nSQ2RZ+zLv+0/zRTvtuLq7OBIaX0YBE117Um9xXARMvyRGB5kfL7lVl3IF3a24WwLa01uxJSmbU5jif+t4fNMefLrGsyaX6JSqZ9E0/+93A3xoT4kldg4tG+AdR34BESb0blNssopb7DfPO0oVIqEXgFeBtYpJR6CDgB3GGp/jPmbpCxmLtCPmCDmIUQRXywNoaP18VYX7+/Jpq+gQ1LbRefvzWBA4npvHN7F3q2aUinZvUIbu7F3ZHSNOpoKtJb5u4yVg0qpa4GnqhqUEKIiolNyeST9X8l9tY+tdl/Ko2Nx84xoH0ja3lmbj513JyZtyWBSD9vJoSbx3WpV8uFB3v73/C4he3J2DJC1GCzNsXj5mzgP/eE4unmzBf3hdHCuxYfrI22tr3/uDeR0NfW8NnGOBIuZDM2tJn0drkJyPADQtRQuflGftp/mtvDmjOqiy+juvgCMGVAIM8vOcC6IykUmEy8sfIIBSbNzN+OYVAwtJPcOL0ZSHIXoobaezKNywUmBrZrVKz8tq7N+M+GWP61PIrT6bm08K7F9FHBbI4+z7BOTWh4lXFXhOOQ5C5ENXDpcgHuzgbr0LQVsTX+AgYFkQHFRzd0cTIwdVAg//hhP57uzvw8tQ+e7i7cFtq8jD0JRyRt7kLYmdGkGfbBZqYvi6rUdltiz9O5WT3quruUWDc2xJdIf2+mDgzEs5T1wvFJchfCznZYhtFdtOtUuVPUFUrJyGX3yYsMbN+o1PXOTgYWPdqDR/oGlLpeOD5J7kJU0Na4C8z94ziXC4zXdb+/RCXj7mLA1dnAJ+tjK7TNr4fOoHXpk14IAdLmLkSF5BtNPLZgN2nZ+aw/msL8ByMxXGW424oyPzF6hgHtGtHS24Mvf4/niQGtadPIs1i9rMsF/Bp1hpYNPPh+xyliz12iTaM6BDb2LGPP4mYnyV2IClix7zRp2fkMbN+I9UdTWLD9BPf18KvyfnefvMi5zMuMCGpKr9YN+N+Ok8xYcZhvHopEKcXcP46Tk2/k+50nOZWag4uTIt9o7r8+bXj7Kr+/cFzSLCNEOUwmzaxNcbRv4sl/7w+nm783766OZumeRHLzq9ZEs3L/aVydDQxs34gGddx4flg7/og9z0+WAcBeW3mYmb8dI+liDq0aeJBv1NRyccKgYFzXZtfpEwpHJMldiHJsjE4hJuUSf+/XGoNB8eTANqTn5PPMov18sSn+mvd7MSuPH3YnMqJzE+q4mS+i7+nWiqBm9Xh95WEOJKZb6/6/wW1Z9ngvpg5sw6qpvfn6wUga13Wv8mcTjkuSuxBlSM/JR2vND7sSaVjHlVu6mG9e9m7TkH+N6oibs4Fvt58gKsmchLPzCsi6XMCp1GwKLBM8lyXmbCa3f76F7Dwjj/dvYy13MijeGhdEek4+D8zbCcDKKb2ZMiiQ+rVdeWZoOwJ86hSbJUmI0khyFw7lXOblSm+z+0Qq4W+s5UhyhrUsJSOXsNfXcM+X21l3JIVbg31xsTxgpJTiwd7+fPVABBez8rj1P3+w/1QaD87bSadXfqPPOxv4969HS32v7LwCNrqIrfQAABa/SURBVBxL4cvf40lOz+Wju0Jo16T4TdHOzerx5tjOpGbl4epsKLFeiIqQ5C4cxvytCUS8uZZ1R85WeButNW+sOsL5S5f5fGNckX2doMCk2Rp/gTyjiTvCW5TYtmfrhmx6fgB13V1465cjbItPta77essJEi9mF6ufdbmAQe9t4oGvdrJoVyJ92zZkTEjp7eYTwlswdWAb7o5oYf2jIkRlSG8Z4RASzmfxxsojALy3OpqB7RtVaOTDHcdT2XsyDf+GtVl1MBmT1pzLvMyxs5kMat+I/u0b0SOgAW0a1Sl1+2ZetXi4tz/vrYkG4LtHutOqgQf9393IB2tieO+OYGvdVQeTSU7Ptb7u17b0B5AKPTO0XbnxC1EWOSUQNVq+0cQ/fzxI/3c34mRQvDCiPYeTM/jtUMmz96NnMnjtp8Pk5P3Vw2Xx7kTquDkzZ2I4BgUrDySz/Xgqadn5PNqvNfd1b1VmYi80sZcfdd2dcXM2ENrSC1+vWkzs0YqlexM5diaz2HsFNKzNlhcGMiGsubUNXwhbkOQuarT/+/kIC7afxNmgeKx/ax7u7Y9/w9p8uDYak+mvuUQzc/OZPH83c/88bp3c4q1fjvDj3iRGBjUhwKcOY0Oa4epkoIV3LYJbeBHhV79CMdR1d+H1sZ15enBb3F2cAHi8fxvquDqbuzGm5bAl9jw7E1IZHeKLr1ctZk4Ipl4tGfNF2I40y4gaaXv8BX46cJqle5IY17UZb44Nwt3FgFKKqYPa8P8W7ufXQ2c4mJROJ9+6nEnP5WRqNhF+9Zm9OZ4+gT7M3hxP30Af68NAM0Z34tF+rfGu7YpBUakJLa5sO69f25W/92/NzN+OEZWUzpkMc3PMoPYylrq4MSS5ixrHZNJMXxZFTIp5kK3Rwb7UcnWyrh8d3IxP1sfy1i9HOJWaA4BvPXe6tvTii/vCGfTeRiZ/swutYeqgQBpYxjev7eZcbhNMZTzQy495WxKsiR2gk2/d67Z/Ia5GmmVEjZKTZ+QfP+y3JnaAHq0bFKvjZFA83r+NNbEDnE7PZWJPP7xru/LPWzqSmVuAp7szwc3r2SxWD1dnXh3didHBvjw9OJDXxnS6LuPRCFERcuYuapT5WxNYujeJyX0DGNe1GalZebg5O5WoNzrYl3d/O4aXhwtTBwXSsI4bkf7mSS1u79qM1YfO0Liue6Umx7gWI4OaMlJGbhR2IMld1Bi5+UbmbUmgR0ADXhrZ4ap1XZ0NfPOQeeTG1j7Fm1qUUsy+P9yWoQphd5Lchd2dzcjl0uWCEkm4qBkrDrHhWApnMnKZOT64zHpFyXC44mYmbe7ihvt22wmOn88CYFdCKt3+bx0jPvq9zKED0nPymbclgRMXsvnorlB6Bza8keEKUSNJchc3VGxKJtOXRfHGysMAzPnjOAB5BSbeW32s1G12JZgf6/9+cndGB/vemECFqOEkuQubOZuRi9a6WNmqA2cAWHc0hdiUTP6IOc+d4S14uLc/C3edso6wWNS2+Au4OhsIaeF1Q+IWwhFIchc2cSo1m55vr+erPxOsZVtiz7Ng+wnaN/HE1cnA4wv2kHm5gP7tfMxD2nq48trKw8X+IOTkGfn54BnCWta3Pv0phCifJHdhE7/HnMdo0nyyPoaUzFw+3RDLfXN3UNvNmX/f3oVRXZoSffYSnm7O9A5sSL1aLvxjaDt2HE9l1cFk634+3xhLUloOTw0OtOOnEaLmkd4ywia2xJ2njpszF7PzGfrBZtKy8xncoREf3hVKHTdnHh/Qmthzl3jl1o54upvHWLkzogULtp/gtZ8O07etD/tOpjFrUzxjQnzpHtCgnHcUQhSlrmwTtYfw8HC9a9cue4chijiYmI5Ja4KvoZ27wGii2/+to29bH/KNJlYeSOYfQ9vy5MDyz773n0pj7Gd/0syrFokXc/DycOG3p/vKlHJClEIptVtrXepDG3LmLopJzcrj6JkMXlhykNx8I3++MLDSk0Ws2H+aC1l5jOjchNCW9QlvVZ+/dW9VoW2DW3gxsYd5TJZxoc34160d8fJwvZaPIsRNTZK7KOblZVHF2rx/iTpTqe6HJpPmPxtiad/Ek8EdGmMwKCb18q9UDM8Pb0fbxp6MDfXFw1W+okJciyrdUFVKJSilDiql9imldlnKvJVSa5RSMZbfFRsUW9jcqdTsEl0TCyWn5zDlu73FEnszr1p8vSWhUu+xKfoc8eeyeKx/62seJMvD1Zl7urWUxC5EFVyP3jIDtNYhRdp9XgDWaa0DgXWW18KONkWf49eoM/R/dyPztiRw8kJ2iTofronhp/2nARjXtRmf3B3KQ7392X3iIjuOp/LJuhhWHUgu849Dobl/HqeRpxsjOstgWULYky1OjcYA/S3LXwMbgWk2eB9RAWnZeUyev4vLBSYAXv3pMK/+dJglj/Xgx71JpOcU8PKoDizZk4iPpxvnMi/zzJC2NK/vQUZuPu+tPsbDX+8kI7cAgK8mRTCgfelzf8aczeT3mPM8O6Qtrs7Sy1YIe6pqctfAaqWUBr7QWs8GGmutC6/tzwClTj2jlJoMTAZo2bJlFcMQZVm8O9Ga2F2dDeRZlj9cG8Pek2lculxAgdFEgUkz/8FIfL1qWad/q+vuwn09/Ji1KY7wVvVJTs/lk/Ux9G/nU+osRV/+Ho+rs4F7usm/pxD2VtXk3ltrnaSUagSsUUodLbpSa60tib8Eyx+C2WDuClnFOG5KWmuy84zUdiv9n9Fk0izYfpKgZvVIycxlct/WRPjVZ8PRc3ywNhowJ/xfos7QpK477Zt4lkjaj/TxZ0vceZ4b1o6YlEtMXxbF4Pc3cXdkSx7uE2Ctd/RMBot3JzKpp791ZiMhhP1U6dpZa51k+Z0C/AhEAmeVUk0BLL9TqhqkKN2/fz1Gj7fWcSY9t9T1W+IucPx8Fg/29mPLC4N4sJcfXZp7MamXH56WPwgzx3cBoF/b0s/GG9RxY8WTvekW0IC7IlrQwrsWceeyeGPVEeLO/TUb0oJtJ3FzdmLqoDY2+KRCiMq65uSulKqtlPIsXAaGAlHACmCipdpEYHlVgxQlxaZc4svf48nILeCjdTGl1vny93i8a7syonNTnAzKmrzr1XLh2aFtGR/WnDEhzfj47lCmVCApOzsZ+N/D3fnsb12p7erEjBWHrDdY/4w9T/cAb+mTLkQ1UZVmmcbAj5aE4Qz8T2v9q1JqJ7BIKfUQcAK4o+phiiv9dugMRpPm1mBfFu06xcN9/ItNdrE9/gKbos/x4oj2pQ64VbTveWX6sbfw9qCFtwcXLl3m5eWH+HbbCXy9ahF/Pkva2oWoRq45uWut44ESU+JorS8Ag6oSlCjfnhMXCfCpzb9GdWTdkbO8vzqaT//WlVOp2Tz53V6cDQovDxcm9vSzyfvf270Vqw+f5eXlh6xlfdv62OS9hBCVJ0+J1EBaa/aeSmNg+0b4eLrxcJ8APl4Xw6oXVlHX3dnabfHuyBY2GyZXKcXM8cE8+b89DOrQmODm9Wgr09oJUW1Icq+B4s9nkZqVR9eW5od/H+njzzdbE7iYnW9N7AAjg2z7IFGTeu4sfqynTd9DCHFtJLnXQJ9tiMPVyUC/duZmEE93F359ui+pWXlM/mYX/xjajoycfHq1lrlGhbhZSXKvYQ6fzmDJnkQe7RdAM69a1vLGdd1pXNed358faMfohBDVhTwjXsO8vyaauu7OPN5f+pMLIcomyb0aOpKcwfAPN/OLZYTG5fuSeOuXI+w/lcbaI2d5pE+AdYgAIYQojTTL2JnWutiTobn5RibO3UFK5mWeW3yArDwjLyw5QIFJ88WmeOp7uPBA78qNjy6EuPnImbsd7TuVRp93NvB7zDnOX7oMmM/SUzIvM3N8F5SCf/ywnxbeHoS29KKuuzOvjelMnTLGkhFCiEKSJezEaNK8vCyKxIs53DdnB27OBn55qg/zt56gfRNPxoc1p24tF2b+dozP/taVVg080Bqb9VsXQjgWSe52snDnKQ4mpfNwb382Rp8jNuUS4z7fQlp2Pq+O7oRSimGdmjCsUxN7hyqEqIGkWcYOCowm3lt9jEh/b/55SwfWPtOPf98eRFp2PlC5sV6EEKI0cuZuB1GnM7iQlceM7q2sN1PvCG9B/PksXAwG6teWkRWFEFUjyd0OtsZdAKB7QANrmVKKF0d0sFdIQggHI80ydrAl7jyBjerg4ykzFgkhbEPO3G+gA4lp7Dieyta4CzwofdWFEDYkyf0G2ZmQyn1ztpObb56g+p5ImdhCCGE7ktyr6MonTEtT2Kfdx9ONsJb1qe3mjF/D2jcoQiHEzUiSexVk5uZz++dbuCO8BQ/3CSiz3tojZzl6JpOP7w6Vbo5CiBtCbqhWktaa9Bxzf/T/bIgl+uwlPlgTzQXL8AGl2XjsHJ5uzozsLA8kCSFuDEnulZCdV8A9X26n51vr+GhtDF9siqdPYENy8o18vjGuzO3+jD1P99YNcHaSwy2EuDEk21SQ1pppSw6yNf4CWXlGPlgbTb+2PsyZGMG4rs2Zv+0EJy5kWet/vC6G5fuSOJWazcnUbHq1bnCVvQshxPUlyb2CtsZd4Kf9p3l2SFvGhvji5eHCzPFdcHU28PTgQGq5OHHfnB2kZ+dzMDGd99dE88KSgyzYfhKAvm197PwJhBA3E7mhWkHL9iVRx82ZR/oG4GRQZF0uwMvDPExA8/oezJ0UwYRZW/hgbTSn03LwdHPmstHErE1xtPapTYBPHTt/AiHEzUSSewXk5hv5JeoMQzs1tg65W5jYC4W1qs+93Vsxb0sCAE8PDsTFycDM344VG2ZACCFuBEnuFbBi/2kycwsY37X5Veu9MKI9qw+d5UxGLg/09KeOuzPuLk7cGtz0BkUqhBBmktwrYN6fCbRr7EmPcm6Kerg6s/4f/UjLzqeeh3mO04dkmAEhhB3IDdVyxKZkcjg5g3u6tSz3SVQwJ3hfr1o3IDIhhCibJPdy/Bp1BkBmRBJC1CjSLFOKNYfP8sGaaIZ1asLKA6cJaeFFk3ru9g5LCCEqzKHP3PefSuP5xfvJyTNWeJsCo4n/+/kIh5Mz+GBtNDEpl5jct+xxY4QQojpy6DP3l5dHcSAxHSeDgbfGBZVaJ+tyAQVGbb0BuvJAMsfPZzHr3q5si08l32hihIwJI4SoYWyW3JVSw4GPACfgv1rrt231XqU5nZbDwaR0AL7feZLH+7emhbdHsTp5BSZu++xPCkya1U/3RSnFJ+tjaN/Ek6EdmzC8s3RhFELUTDZpllFKOQGfAiOAjsDdSqmOtnivsqw8cBqt4fvJ3VHA/K0JJeq8t+YY0WcvEX8ui5UHkvn5YDJx57KYMjAQg6H8njFCCFFd2erMPRKI1VrHAyilvgfGAIev55sYTRoFxRJxdl4BLy87xJI9iXRuVpfuAQ0Y1cWXL38/jpeHK08MaAPAkt2JfLEpnrsjW7D3ZBofro3G2clAYKM60gwjhKjxbJXcmwGnirxOBLpd7zdZc/gsjy/YTSNPd/q382Fg+0Z8u/0km6PPATC0ozlJvzshGKNJ8/6aaAa2b4R/w9q889tRQlt68cbYIH4+mMyU7/YCMOvernLWLoSo8ex2Q1UpNRmYDNCy5bXNJ+rfsDZPDmjDidRslu5J4vud5r8n/dr60NLbg3u6mffr6mzgjbGd+SP2PB+ujWZoxyaczbjMB3eG4GRQjAxqyprDZ4n095Z2diGEQ7BVck8CWhR53dxSZqW1ng3MBggPD9fX8ibtmnjSrkk7AP4xNJszGbm4OhkIbFwHD9fiH61+bVfu6daSLzbFcTE7n/oeLnT3Nw8n4GRQfHx36LWEIIQQ1ZKt+rnvBAKVUv5KKVfgLmCFjd4LgBbeHkT4eRPcwqtEYi90f49WOBkUO46n0j2ggTS/CCEclk2Su9a6AHgS+A04AizSWh+yxXtVRtN6tRgd3AyA0JZedo5GCCFsx2Zt7lrrn4GfbbX/a/WvWztSt5Yz48NalF9ZCCFqKId+QrU09Wq58MqtnewdhhBC2JRDjy0jhBA3K0nuQgjhgCS5CyGEA5LkLoQQDkiSuxBCOCBJ7kII4YAkuQshhAOS5C6EEA5IaX1NY3Zd3yCUOgecuMbNGwLnr2M4tibx2lZNircmxQoSr61dS7yttNY+pa2oFsm9KpRSu7TW4faOo6IkXtuqSfHWpFhB4rW16x2vNMsIIYQDkuQuhBAOyBGS+2x7B1BJEq9t1aR4a1KsIPHa2nWNt8a3uQshhCjJEc7chRBCXKFGJ3el1HCl1DGlVKxS6gV7x1MapVSCUuqgUmqfUmqXpcxbKbVGKRVj+V3fjvHNVUqlKKWiipSVGp8y+9hyvA8opbpWg1hnKKWSLMd3n1JqZJF1L1piPaaUGnYjY7W8fwul1Aal1GGl1CGl1FOW8mp3fK8Sa7U8vkopd6XUDqXUfku8r1rK/ZVS2y1xLbRM84lSys3yOtay3q+axDtPKXW8yPENsZRX/bugta6RP4ATEAcEAK7AfqCjveMqJc4EoOEVZe8AL1iWXwD+bcf4+gJdgajy4gNGAr8ACugObK8Gsc4A/lFK3Y6W74Qb4G/5rjjd4HibAl0ty55AtCWuand8rxJrtTy+lmNUx7LsAmy3HLNFwF2W8lnAY5blx4FZluW7gIU3+LtQVrzzgPGl1K/yd6Emn7lHArFa63itdR7wPTDGzjFV1Bjga8vy18BYewWitd4MpF5RXFZ8Y4D52mwb4KWUanpjIi0z1rKMAb7XWl/WWh8HYjF/Z24YrXWy1nqPZTkT83zCzaiGx/cqsZbFrsfXcowuWV66WH40MBBYbCm/8tgWHvPFwCCllLpB4V4t3rJU+btQk5N7M+BUkdeJXP3LaC8aWK2U2q2Ummwpa6y1TrYsnwEa2ye0MpUVX3U95k9aLl3nFmniqlaxWpoBQjGfsVXr43tFrFBNj69SykkptQ9IAdZgvnpI01oXlBKTNV7L+nSggT3j1VoXHt83Lcf3A6WU25XxWlT6+Nbk5F5T9NZadwVGAE8opfoWXanN12DVtstSdY8P+BxoDYQAycB79g2nJKVUHWAJ8LTWOqPouup2fEuJtdoeX621UWsdAjTHfNXQ3s4hXdWV8SqlOgMvYo47AvAGpl2v96vJyT0JaFHkdXNLWbWitU6y/E4BfsT8JTxbeIll+Z1ivwhLVVZ81e6Ya63PWv7TmIAv+atpoFrEqpRywZwsF2itl1qKq+XxLS3W6n58AbTWacAGoAfm5gvnUmKyxmtZXw+4cINDBYrFO9zSHKa11peBr7iOx7cmJ/edQKDl7rgr5pskK+wcUzFKqdpKKc/CZWAoEIU5zomWahOB5faJsExlxbcCuN9yJ787kF6kecEurmiHvA3z8QVzrHdZekn4A4HAjhscmwLmAEe01u8XWVXtjm9ZsVbX46uU8lFKeVmWawFDMN8n2ACMt1S78tgWHvPxwHrLVZM94z1a5I+8wnx/oOjxrdp34UbeMb7eP5jvKEdjbmv7p73jKSW+AMw9CvYDhwpjxNzWtw6IAdYC3naM8TvMl9v5mNv1HiorPsx37j+1HO+DQHg1iPUbSywHLP8hmhap/09LrMeAEXY4tr0xN7kcAPZZfkZWx+N7lVir5fEFugB7LXFFAf+ylAdg/iMTC/wAuFnK3S2vYy3rA6pJvOstxzcK+Ja/etRU+bsgT6gKIYQDqsnNMkIIIcogyV0IIRyQJHchhHBAktyFEMIBSXIXQggHJMldCCEckCR3IYRwQJLchRDCAf1/w2XSAWqwH2EAAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "plt.plot(score_arr2, label ='accumulated scores in one game play 2')\n",
        "plt.legend(loc='upper left')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "aFotlh87SfLB"
      },
      "outputs": [],
      "source": [
        "## game play video 3\n",
        "agent.load_state_dict(torch.load(\"weights.pt\"))\n",
        "env = gnwrapper.LoopAnimation(gym.make(game_name)) \n",
        "state = env.reset()\n",
        "state_dim = len(state)\n",
        "game_score = 0\n",
        "score_arr2 = []\n",
        "state_arr = []\n",
        "for i in range(1000):\n",
        "    if i == 0:\n",
        "        stacked_state = np.concatenate((state, state, state, state, state, state, state, state), axis=0)\n",
        "    else:\n",
        "        stacked_state = np.roll(stacked_state,-state_dim,axis=0)        \n",
        "        stacked_state[-state_dim:]=state\n",
        "    with torch.no_grad():\n",
        "        hs = agent.representation_network(torch.from_numpy(stacked_state).float().to(device))\n",
        "        P, v = agent.prediction_network(hs)\n",
        "        action = np.random.choice(np.arange(agent.action_space), p=P.detach().numpy())   \n",
        "    if i %5==0:\n",
        "        env.render()\n",
        "    state, r, done, info = env.step(action.item())\n",
        "    print(r, done, info)\n",
        "    state_arr.append(state[0])\n",
        "    game_score += r \n",
        "    score_arr2.append(game_score)\n",
        "    if i %10==0: \n",
        "        print(game_score)\n",
        "    if done:\n",
        "        print('last frame number : ',i)\n",
        "        print('score :', game_score)\n",
        "        print(state_arr)\n",
        "       \n",
        "        break\n",
        "env.reset()\n",
        "env.display()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WlLErFQhShIo",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 283
        },
        "outputId": "61743c75-a0aa-4dcc-e30f-d983d4506c57"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.legend.Legend at 0x7fcdd537ae20>"
            ]
          },
          "metadata": {},
          "execution_count": 10
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3xUVfr48c9JgQQSSiBACJACoYckEEJHqhRZilJcFbBiBV1dVFxcdfm5xbK76NcGgqDLSkdQUREEkRpC7+mBhABJCDG9zJzfHzOZDaQBSZji83695pU7555755k7yZMz5557rtJaI4QQwrE4WTsAIYQQtU+SuxBCOCBJ7kII4YAkuQshhAOS5C6EEA7IxdoBADRv3lz7+/tbOwwhhLArBw8eTNdae1e0ziaSu7+/P1FRUdYOQwgh7IpSKqmyddItI4QQDkiSuxBCOCBJ7kII4YBsos+9IsXFxSQnJ1NQUGDtUIQQlXBzc6NNmza4urpaOxRxHZtN7snJyXh6euLv749SytrhCCGuo7UmIyOD5ORkAgICrB2OuI7NdssUFBTQrFkzSexC2CilFM2aNZNv1zaq2uSulGqrlNqulDqllDqplHrWXP66UipFKXXE/BhbZpt5SqlYpdRZpdSoWw1OErsQtk3+Rm3XjXTLlAAvaK0PKaU8gYNKqR/N6/6ltX6nbGWlVFfgXqAb0BrYqpTqqLU21GbgQghh7xZujaGnXxMGBVV4HVKNVNty11qnaq0PmZezgdOAbxWbTABWaq0LtdYJQCwQURvBiv95/fXXeeedd6qs89VXX3Hq1Kmb3reHh8ethnXbjB07lqtXr1o7DJu0Y8cOxo0bV+P9REZGEhoaSmhoKCEhIWzYsKEWohNl/d/2GPbEZdTJvm+qz10p5Q+EAfvNRc8opY4ppZYqpZqay3yB82U2S6aCfwZKqVlKqSilVFRaWtpNBy6qd6vJ/XYoKSmp0fabN2+mSZMmtRSNqEj37t2JioriyJEjfP/99zz++OM1/tzEtQxGjXMddW3dcHJXSnkA64DntNa/Ah8B7YFQIBV492ZeWGu9SGsdrrUO9/au/a8ktWHixIn06tWLbt26sWjRIkv5999/T8+ePQkJCWH48OEA5OTk8NBDDxEcHEyPHj1Yt24dcG0reO3atTz44IMAPPjggzz55JP07duXwMBAduzYwcMPP0yXLl0sdaravqzFixfTu3dvQkJCuOeee8jLy2PPnj1s2rSJuXPnEhoaSlxcHHFxcYwePZpevXoxaNAgzpw5A0BCQgL9+vUjODiY+fPnV3gscnNzueuuuwgJCaF79+6sWrUKgAMHDtC/f39CQkKIiIggOzubgoICy7EICwtj+/btACxbtozx48czbNgwhg8fTm5uLg8//DARERGEhYWxceNGAE6ePElERAShoaH06NGDmJiYcvH4+/uTnp5OYmIiXbp04bHHHqNbt27ceeed5Ofnl6ufmJjIsGHD6NGjB8OHD+fcuXOWz2HOnDn079+fwMBA1q5da9nm7bffpnfv3vTo0YPXXnutwuOyZMkSOnbsSEREBI899hjPPPMMAF9//TV9+vQhLCyMESNGcOnSJcD0jWvmzJkMGjQIPz8/1q9fz4svvkhwcDCjR4+muLgYgIMHD3LHHXfQq1cvRo0aRWpqarnXfvDBB3niiScIDw+nY8eOfPPNN+XqREZG0q9fP8LCwujfvz9nz54FYPDgwRw5csRSb+DAgRw9evSabRs0aICLi6nntqCgQPrXa5nWGqMGJ6e6Oa43NBRSKeWKKbGv0FqvNwd2qcz6xUDpb1YK0LbM5m3MZbfsja9PcurCrzXZRTldWzfitd91q7LO0qVL8fLyIj8/n969e3PPPfdgNBp57LHH2LlzJwEBAVy5cgWABQsW0LhxY44fPw5AZmZmtTFkZmayd+9eNm3axPjx49m9ezeffvopvXv35siRI4SGht7Qe7n77rt57LHHAJg/fz5Llixh9uzZjB8/nnHjxjF58mQAhg8fzscff0xQUBD79+/nqaee4qeffuLZZ5/lySefZMaMGXzwwQcVvsb3339P69at+fbbbwHIysqiqKiIadOmsWrVKnr37s2vv/6Ku7s7CxcuRCnF8ePHOXPmDHfeeSfR0dEAHDp0iGPHjuHl5cUrr7zCsGHDWLp0KVevXiUiIoIRI0bw8ccf8+yzz3L//fdTVFSEwVD16ZqYmBi+/PJLFi9ezNSpU1m3bh0PPPDANXVmz57NzJkzmTlzJkuXLmXOnDl89dVXAKSmprJr1y7OnDnD+PHjmTx5Mlu2bCEmJobIyEi01owfP56dO3cyePBgyz4vXLjAggULOHToEJ6engwbNoyQkBDAlCz37duHUopPP/2Ut956i3ffNbV/4uLi2L59O6dOnaJfv36sW7eOt956i0mTJvHtt99y1113MXv2bDZu3Ii3tzerVq3iT3/6E0uXLi333hMTE4mMjCQuLo6hQ4cSGxt7zfrOnTvzyy+/4OLiwtatW3nllVdYt24djzzyCMuWLePf//430dHRFBQUWGIva//+/Tz88MMkJSXxxRdfWJK9qDmj+Q6nddVyr/aTUqZ/10uA01rrf5Yp99FalzYnJgEnzMubgP8qpf6J6YRqEBBZq1HfJu+9956ln/H8+fPExMSQlpbG4MGDLeN6vby8ANi6dSsrV660bNu0adPyO7zO7373O5RSBAcH07JlS4KDgwHo1q0biYmJN5zcT5w4wfz587l69So5OTmMGlV+gFJOTg579uxhypQplrLCwkIAdu/ebfmmMX36dF566aVy2wcHB/PCCy/w0ksvMW7cOAYNGsTx48fx8fGhd+/eADRq1AiAXbt2MXv2bMCUXPz8/CzJfeTIkZZjtmXLFjZt2mQ5d1BQUMC5c+fo168fb775JsnJydx9990EBQVV+f4DAgIsx6pXr14kJiaWq7N3717Wr19veY8vvviiZd3EiRNxcnKia9eulhb2li1b2LJlC2FhYZbjFxMTc01yj4yM5I477rC8nylTpljeZ3JyMtOmTSM1NZWioqJrxoGPGTMGV1dXgoODMRgMjB492nKMExMTOXv2LCdOnGDkyJEAGAwGfHx8KnzvU6dOxcnJiaCgIAIDAy3fxkplZWUxc+ZMYmJiUEpZvhlMmTKFBQsW8Pbbb7N06dIKvxEC9OnTh5MnT3L69GlmzpzJmDFjcHNzq7CuuDklRiMALs7Wa7kPAKYDx5VSpd/jXgF+r5QKBTSQCDwOoLU+qZRaDZzCNNLm6ZqOlKmuhV0XduzYwdatW9m7dy8NGjRgyJAhtzSet+xX2eu3r1+/PgBOTk6W5dLnpX2bVW1f6sEHH+Srr74iJCSEZcuWsWPHjnJ1jEYjTZo0ueareGVxVqRjx44cOnSIzZs3M3/+fIYPH86kSZOq3KYiDRs2tCxrrVm3bh2dOnW6pk6XLl3o06cP3377LWPHjuWTTz5h2LBhle6z7LFzdnausFumKmW3L71hvNaaefPm8fjjj9/UvkrNnj2b559/nvHjx7Njxw5ef/31cq/n5OSEq6ur5diXfu5aa7p168bevXurfZ3rP7frn7/66qsMHTqUDRs2kJiYyJAhQwBTl8vIkSPZuHEjq1ev5uDBg1W+TpcuXfDw8ODEiROEh4dXG5eonjm342StPnet9S6ttdJa99Bah5ofm7XW07XWweby8WVa8Wit39Rat9dad9Jaf1cnkdexrKwsmjZtSoMGDThz5gz79u0DoG/fvuzcuZOEhAQAS7fMyJEjr+nSKO2WadmyJadPn8ZoNN7SaIMb2T47OxsfHx+Ki4tZsWKFpdzT05Ps7GzA1KoOCAhgzZo1gCl5lfaxDhgwwPKto+z2ZV24cIEGDRrwwAMPMHfuXA4dOkSnTp1ITU3lwIEDljhKSkoYNGiQZT/R0dGcO3euXAIHGDVqFO+//74loR4+fBiA+Ph4AgMDmTNnDhMmTODYsWM3d9Aq0L9//2ve46BBg6qsP2rUKJYuXUpOTg4AKSkpXL58+Zo6vXv35ueffyYzM5OSkhLLtx8w/f74+prGESxfvvymYu3UqRNpaWmW5F5cXMzJkycrrLtmzRqMRiNxcXHEx8eXO85l41i2bNk16x599FHmzJlD7969K/ymmZCQYGlkJCUlcebMGeS+C7XHYP69d66jS0lt9gpVaxs9ejQlJSV06dKFl19+mb59+wLg7e3NokWLuPvuuwkJCWHatGmAqa87MzOT7t27ExISYjmJ+Pe//51x48bRv3//Sr9aV+VGtl+wYAF9+vRhwIABdO7c2VJ+77338vbbbxMWFkZcXBwrVqxgyZIlhISE0K1bN8sJzIULF/LBBx8QHBxMSkrFp0eOHz9uOcn5xhtvMH/+fOrVq8eqVauYPXs2ISEhjBw5koKCAp566imMRiPBwcFMmzaNZcuWXdM6LvXqq69SXFxMjx496NatG6+++ioAq1evpnv37oSGhnLixAlmzJhx08fteu+//z6fffYZPXr04IsvvmDhwoVV1r/zzju57777LCeaJ0+ebPlHWcrX15dXXnmFiIgIBgwYgL+/P40bNwZMJ06nTJlCr169aN68+U3FWq9ePdauXctLL71ESEgIoaGh7Nmzp8K67dq1IyIigjFjxvDxxx+X6zJ58cUXmTdvHmFhYeVGuvTq1YtGjRrx0EMPVbjvXbt2WV5/0qRJfPjhhzf9XkTlDOZO97pquaO1tvqjV69e+nqnTp0qVyaErcnOztZaa11cXKzHjRun169ff9tee+bMmXrNmjW3vH1KSooOCgrSBoOhRnHI3+qtuZJTqP1e+kYv3RV/y/sAonQleVVa7kLUwOuvv05oaCjdu3cnICCAiRMnWjukG/L555/Tp08f3nzzTZycJA1Yw/+6Zaw4FFIIUbHqrhKuS9f3od+MGTNm1Ep3l7h1xjrulrHpf9na/J9NCGGb5G/01tV1y91mk7ubmxsZGRnyyyOEjdLm+dxl3PutKT2harWLmKylTZs2JCcnI/POCGG7Su/EJG6eZZz7b63P3dXVVe7uIoRwWJYrVH9r3TJCCOHIjOYu57pquUtyF0IIKzCYu2WsPuWvEEKI2mM5oSrTDwghhOOwdMtIy10IIRzH/1ruktyFEMJhGOSEqhBCOB5jHV/EJMldCCGsQLplhBDCAUlyF0IIB/SbnThMCCEcWV3fiUmSuxBCWIFRWu5CCOF4ZPoBIYRwQJZuGZl+QAghHId0ywghhAOq6zsxSXIXQggrkPnchRDCAZUYpOUuhBAORy5iEkIIB2SU6QeEEMLxSMtdCCEckNHa0w8opdoqpbYrpU4ppU4qpZ41l3sppX5USsWYfzY1lyul1HtKqVil1DGlVM86iVwIIeyYLcwKWQK8oLXuCvQFnlZKdQVeBrZprYOAbebnAGOAIPNjFvBRrUcthBB2zjxYxnqjZbTWqVrrQ+blbOA04AtMAJabqy0HJpqXJwCfa5N9QBOllE+tRy6EEHbMaEvTDyil/IEwYD/QUmudal51EWhpXvYFzpfZLNlcdv2+ZimlopRSUWlpaTcZthBC2DebOaGqlPIA1gHPaa1/LbtOa60BfTMvrLVepLUO11qHe3t738ymQghh92xiPnellCumxL5Ca73eXHyptLvF/POyuTwFaFtm8zbmMiGEEGZWP6GqlFLAEuC01vqfZVZtAmaal2cCG8uUzzCPmukLZJXpvhFCCEHdTxzmcgN1BgDTgeNKqSPmsleAvwOrlVKPAEnAVPO6zcBYIBbIAx6q1YiFEMIBGLVGqbqbOKza5K613gVU9urDK6ivgadrGJcQQjg0g1HXWasd5ApVIYSwCoPWddZqB0nuQghhFUZpuQshhOMxGOtupAxIchdCCKswak0d5nZJ7kIIYQ0Go5aWuxBCOJoSSe5CCOF4jJLchRDC8Ri0jJYRQgiHYzTKOHchhHA4Bi3dMkII4XBk+gEhhHBARpl+QAghHI+03IUQwgEZjHU33S9IchdCCKswGI0412EGluQuhBBWYNB1dxcmkOQuhBBWIVeoCiGEA5KJw4QQwgEZtMZJumWEEMKxSLeMEEI4IJl+QAghHJDRKN0yQgjhcKTlLoQQDshgRFruQgjhaOQKVSGEcEAyzl0IIRyQUYOzU92lYEnuQghhBaYpf+tu/5LchRDCCgzWvoeqUmqpUuqyUupEmbLXlVIpSqkj5sfYMuvmKaVilVJnlVKj6ipwIYSwZ0Zt/Zt1LANGV1D+L611qPmxGUAp1RW4F+hm3uZDpZRzbQUrhBCOoq5PqLpUV0FrvVMp5X+D+5sArNRaFwIJSqlYIALYe8sRCiGEFRSVGFlz8DxfHU4hJTOf+q7OFBQbyC82YDDoa+rq67bVuur1AHlFBlzqsNO92uRehWeUUjOAKOAFrXUm4AvsK1Mn2VwmhBA2q6jESGpWPilX80nOzCe7oISd0Wn8HJ1G51ae9G3fjKISI+6uzri5OuN63QD163tXVFXrzAUKuLtnm9p/M2a3mtw/AhZg+oe0AHgXePhmdqCUmgXMAmjXrt0thiGEELduf3wGi3+JZ2d0OkUGY7n1CyZ044G+fpaEbE9uKblrrS+VLiulFgPfmJ+mAG3LVG1jLqtoH4uARQDh4eEVfWsRQog6cTGrgAXfnuLbY6k096jPA3396OzjSZsm7vg2dcfd1Zm0nEK6tW5s7VBv2S0ld6WUj9Y61fx0ElA6kmYT8F+l1D+B1kAQEFnjKIUQopZ8f+Iic9cepajEyPMjOzJrcCBuruXHfbRo5GaF6GpPtcldKfUlMARorpRKBl4DhiilQjF1yyQCjwNorU8qpVYDp4AS4GmttaFuQhdCiOoVlhio7+JMdkEx7/xwluV7k+jRpjHv/z4Mv2YNrR1enVHXn9W1hvDwcB0VFWXtMIQQDiS/yMBzqw7zw8lLeDWsR25hCUUGIw/29+flMZ2p72L/o7SVUge11uEVravJaBkhhLBJWmueW3WYLacuMbOfH0UGjburMxPDWtOjTRNrh3dbSHIXQjicj36O44eTl5h/VxceHRRo7XCsQuaWEULYvWKDkUU740jPKWTF/iTe+eEs43r48MjAAGuHZjXSchdC2L1DSZn8dfMZ/rr5DAADOjTjH/f0sMvx6bVFkrsQwu5dyMq3LL8xvhvT+/rV6YyL9kCSuxDC7l24WgDAqb+MokE9SWsgfe5CCAdw4Wo+TRu4SmIvQ5K7EMLuXbiaj09jd2uHYVMkuQsh7F5qVgGtm0hyL0uSuxDC7qVczce3iX3PBVPbJLkLIWxGbmEJn+9NZMPh5HI3vKhMdkEx2QUl0nK/jpx9EELYjP/37Wm+jDwHQKtG7vRr36zabS79ahop06qxtNzLkpa7EMLqLv1awL2L9rLywDnu69OOxu6ufLIzjuhL2dVueyW3GIBmDevXdZh2RZK7EMLq3tsWw8GkTB4ZEMArY7swNbwNO86mMe79XVzJLapy29L1TRu63o5Q7YYkdyGEVa2OOs/qqPNM692W+eO64lHfhedGdGTuqE4UlRj55tiFKrcvTe5eDevdjnDthiR3IUSNaa05npyF0Xhz94fYcfYyL649Rli7pvxhREdLecP6Ljw9tAOdW3my4XCFd+q0yMwzt9wbSHIvS5K7EKLGVh44z+/+bxcfbI+94W0MRs1fvjlFQPOGfPFIBM08yveZTwrz5fC5qySk51a6nyu5RTSs51zhrfJ+yyS5CyFqJLugmHe3ROPipPj3thjOXPz1hrY7mnyV+LRc5gzvUOldkSaE+qIUfFVF6z0zt4im0iVTjiR3IUSN/P27M1zJLWTJg71p4OrMu1uib2i7HWcu46RgaKcWldZp1diN/u2b8dWRlErHvWfkFtFMkns5ktyFEOUUlhg4fyWv2nqxl7NZsf8cDw8I4I6O3swaHMiPpy6xJy7dUue746m8ty0GMPXNlybpHdFp9GzXlCbV9JVPDPUlKSOPQ+euVrg+M09a7hWR5C7Eb0hSRm61QwsB/rklmuHv/kxSRvm+7tjL2ZZ9rI5KxsVJ8cSQ9gA8OigQv2YNmLf+OPlFBgCeXHGIf/4YzYr9SYz8105eWHOUEylZHEvOYkTXltXGMrp7K9xcndhwOLnC9Vdyi/CSk6nlSHIX4jcgp7CE81fyGPf+LiZ/tIfcwpJK6+YXGfgy8hxFBiPvXNfFUlRi5J6P9jLny8PkFxlYfyiZ4V1a0Nx8MtS9njN/uzuYpIw8Zizdz4ylkZZt/7ThBInpuaw/lMIf1xylkZsL9/VpV23snm6ujOrWio1HLpBXVD7uK7lFMgyyApLchXBQGw4n8/R/D5GRU8jMpZEMems7uYUlJGbkMnft0UqHLa49lMyvBSUMCmrO10cvsDvW1MWy7mAyc9ceJSu/mF2x6Tz2eRTpOUU8PODa+5T2b9+c+/q040BiJjuj0wB4dVxX/nFPMDtfHIq3Z31iLucwd1QnGrnd2IVHD/T1I7ughK8OXzvmvaDYQF6RQbplKiBzywhh5y5czee5VUf466TudGjhSYnByLfHU3lp7XGKDEb2x2eQnmPqRnlqSAcau7vy5ubTLG4Tz2ODAq+5HV1eUQnvbYsh3K8pi2eEM/rfO3l+9RE+uK8nL6w5aqnX3KMeu2LTeXxwIH0Cy8//8pfx3XjyjvbM/CyS+LRcHuzvj7P5db57dhDOSt1UQg73a0oXn0Z8vjeR30e0RSnFvvgMywVOXXw8b+XQOTRJ7kLYuUU744lMuMJb35/lk+m9mLv2GBsOp9DOqwGv/a4rz648QmN3V/bOG0aDei5ordkVm86HO+JY/Es8f7yzE1PC2/LTmct8GXmOtOxCPrq/J26uznw8vRcPf3aAyR/vtbze7yPa8crYzuQVGWjZqOLJulycnWjr1YBvZg/kal6xJbEDli6cm6GUYmY/P15ef5wDiZm4OCt+v3gfWsPdYb5Vjrj5rVI3Oq1mXQoPD9dRUVHWDkMIm/eHVUe4mlfEvLFdyCsysPXUJZbuTsDFSfFrQQmPDgzg010JzB7WgWeHB+Hi7ERcWg75RQa6+za27OdA4hWmmBO2R30XxnRvxZqDySgFf5nQnel9/Sx1z2XkMfjt7QDEvDkGFyeFUrf/5tP5RQb6/HUrAzo0JyE9l6z8Yj6dGU5Xn0ZWiccWKKUOaq3DK1wnyV0I2/b+thguZRfwxB3tGfgPU5L1beLOhax8nJQitG0T/jKhGw98up/MvGK6tW7E188MvKa7pSKbjl6gWcN6PP3fQ1zNK2ZaeFteHN2pwitFkzJyKTFq2nt71Ml7vFF/++40n/wcD8DCe0OZEOpr1XisrarkLt0yQtiIrLxiXl5/jDMXs1n2UG/8mjWk2GDk010JZOUXk1toGlo4d1Qn3v7hLL5N3Pl2zkDLOPEnh7Tnr5vP8Mc7O1Wb2AHGh7QG4JcXh7InLoMhnbwrvVLUr1nDWnqXNfNgf3+W/JJAq8Zu3BXsY+1wbJokdyFsxMoD5/juxEU867vw+BcH2fjMAA4kZJKVX4xSsOFwCp1aevLUkPYYjZrBHb2vuQDokYGB9A1sRo82TW7qdUuHGtoDn8buvDMlhFaN3XBxlsF+VZHkLoSVFRQb+CUmnV2x6QR6N+RPY7vwyPIoPtwex+XsQtxdnVkyM5wd0WkM7dQCpRSzhweV24+zk7rpxG6PJob9trtiblS1yV0ptRQYB1zWWnc3l3kBqwB/IBGYqrXOVKazGguBsUAe8KDW+lDdhC6EfXl53TEiE67wzLAOTAz15d0fzzI4yJv5X50g5nIOAPf1acfwLi2ZGNqaD3fE4uykuCu4Nf07NKd/h+ZWfgfCntzI95plwOjryl4Gtmmtg4Bt5ucAY4Ag82MW8FHthCmEfTuXkcfKA+dJzSrgzxtPsiP6Mh9sj2Paon0kpOdarrCM8PcCTBf9eNR3oaDYyCMDA6ratRAVqja5a613AleuK54ALDcvLwcmlin/XJvsA5oopeSsh/jNW3fINMzwk+m9yCsq4fnV/7sg6JFBAWx6ZgD39GzD8C6m8drNPOrz0QO9mH9XF7q2bmStsIUdu9U+95Za61Tz8kWgdPYfX+B8mXrJ5rJUrqOUmoWpdU+7dtXPLyGEvfjpzCXmrT+Ot2d9vni4D43dXVl7MJmBHZozuKM303q35cvI84S2bcI7U3oQ0NwDZyfFu1NDrtlP38Bm9K3g6k8hbkSNTzdr00D5mx4sr7VepLUO11qHe3t71zQMIWzGf/efp6jESPTFHJ5ffYS98RmkXM1ncq82ADw3oiOe5guHOrTwvObqTSFqy6223C8ppXy01qnmbpfL5vIUoG2Zem3MZUL8JhQUG9gdm87kXm0IaunBnzee5HhKFp71XSzDDVs2cmP3vGE0rCeD1UTdudWW+yZgpnl5JrCxTPkMZdIXyCrTfSOE3bqSW0ROFdPkltobl0F+sYGhnb2Z3tePyb3a4F7Pmf83qfs19/hs5OYqLXZRp25kKOSXwBCguVIqGXgN+DuwWin1CJAETDVX34xpGGQspqGQD9VBzELcVgXFBiZ8sAtnpfjTXV3x9qxPSJvGKKU4kHiF//splvl3daGRuyvz1h+nZaP69AtsjlKKd6aEVP8CQtSBapO71vr3lawaXkFdDTxd06CEsCX/2ZfE+Sv5OCl47HPTHEgPDwjgoQH+3Ld4H8UGTX6xgbB2TUjLKeTrZwbiXq/iy/iFuF2k00+IKlz6tYCFW2O4o6M3T9zRnoISAz+cuMjS3Qmcu5JHsUHz+OBAPjFPuzuscwsZuihsgiR3Iarw7pazFBqMvDG+G/7NTZNn9Qtsxr74DLaevkTHlh68NLoze+IyOJ6SxSS5NF7YCJl5R4jrrDuYzKQPd7M/PoPvTlxkfEhrS2IHcHN15uUxnQEY0aUlTk6K/zzah7cn92BMd/uYgEs4Pmm5i9+M709cxLeJO8FtGlda52JWAa9tOkluUQnTFu0DqDBhj+rWin9PC+WOjqZrNBq7uzIlvG25ekJYi7TcxW9CWnYhc748zJyVhykxGCutt3xvIvnFBtY+0d9SNqCCCbuUUkwM85UbMwubJcld/CasjDxHkWSfKG0AABSGSURBVMFIQnouG4+YbqqsteaXmDTyiwyWej+eukSfAC96+TVl2wt3sHJW32vGpwthL6RbRji897fF8M+t0Qzu6E1GTiELt8XQ1qsBGTmFPLniEON6+PD+78NIzMgj9nIO9/cxzXXU3tvD6reVE+JWSXIXDi2nsIQPdsQytFML/jUtlAMJV3j08yimfrIXZyeFk4JvjqUyrHMLMnKKABjZtWU1exXC9klyFw7t22MXKCg28vTQ9jR2d2V4lxY8PbQ9qVcLWH84hccHB3IwKZM3vj5F6ybudPFpRJumDawdthA1JsldOJwLV/P5aEcce+MziEvLob13Q3q2awqYToTOHdUZrTXjQnzoF9iclKv5jH3vF06n/sqcCm5fJ4Q9khOqwioKig28tvEE5zLyanW/+UUGHvrsAKujztPOqwEvjOzIfx7tg+kOkP+jlGJY55a413OmQwsPnhthSuqj7eRG0UJUR1ruwiq+PnqB5XuTSLqSx7KHImq8v6SMXNo2bcB7P8Vw9lI2yx+OsIxBvxFP3tGesd19rrlYSQh7JsldWMWXkedwdlLsOJvGzug0Bt9EIr7e9ycu8sR/DtKyUX2u5hUzKcz3phI7mFryktiFI5FuGXHb7YpJ59C5q/zxzk609XLnr5tPYzBeezOvw+cyWX3gfCV7uNaK/UkAtGnagHrOTjw/smOtxyyEvZGWu7itCksM/HnjCfyaNeChAf6082rA0/89xIbDKZbb0AG89f1Z9sZn0LV1I7r7Vj5dwOnUX9kVm86c4UH8YUQQJUaNq7O0WYSQvwJxWy3eGU98ei5vjO+Gm6szY4Nb0a11I/71YzSrD5ynxGAkK7+YA4lXAFjwzSkMRk12QTEFxf+7kjQrr5ghb29n0oe78faoz/192qGUksQuhJm03MVtk5VXzAfb4xjTvRVDOrUATH3dr4ztwqPLo3hx3TGW7k7gzMVsAKaFt2VV1Hn+vTWazcdTyS4o4Y3x3RjdvRU7oi+TmJFHuF9T3p4SQstGbtZ8a0LYHEnu4rbYdPQCX+w1Tcr1zLAO16wb0KE5R1+7k68Op/DFviTaernj6uTEX+8OprDEwPs/xQLQqpEbT644xH192lFQbKBJA1dWPd5P7kUqRAUkuYs6l5VfzJwvDwPQ1acR3VqX70Ov5+LE1N5tmdq7Laa7Nf6vVb/l1CUaubmyY+4Q/vVjNJ/sjAdgXA8fSexCVEKSu6hzm4+nAjC9rx/3921Xbf2yFxy1aOTGounh1Hd1stwkw69ZQz7bnSDzpwtRBVXaSrKm8PBwHRUVZe0wRB342+bTfLY7kXbNGvDjHwaXu1JUCHHrlFIHtdbhFa2TlruoM4fOZfLJznhGdGnBi6M7S2IX4jaS5C7qzIfbY2nuUZ9/3xuGR335VRPidpJBwaJO5BWVsDMmnd+F+EhiF8IKJLmLclKz8okyX0RUatHOOLaeunTD+/glJp2iEiMju8iNL4SwBmlSiXLe/PY035+4yJY/DCbQ24NzGXn87bszNPeoz8Cg5lXeUzQuLYed0WkcPncVTzcXegd43cbIhRClpOUurmEwan6JSafEqPnbd2cA+GxPAgBp2YX8Z1/SNfUT03O5+8PdHEzKBGD+hhO88fUpNh29wLTwtjIdgBBWIi13cY1jyVfJyi8mpG0Tfjx1iUeWHWDbmcvc3dOXtGzTzaVTruaz42wac0d1IuZSDofOXeWej/bw0f092RufgburMwaj5tFBgdZ+O0L8ZklyF9fYGZ2OUvDxAz2Z/NFetp25zON3BPLCyE6cz8xjzL9/YcW+c3g1rMcf1xylVWM3PN1cMBo1T644hFfDemx6ZgDZBSW0aizzvQhhLZLchYXBqNkZk0awb2N8GrvzxSMR/FpQQmjbJgC09/bgx+cH09jdlfxiA3e9t4v4tFyeHNKegGYN+dt3p1k8I1xuMC2EDahRcldKJQLZgAEo0VqHK6W8gFWAP5AITNVaZ9YsTFHXvjqcwty1Ryk2aJ4ZaprYK9Dbo1w9v2amuxU1ARbeG8oTXxxkdLdWhLRtwuRebXCSuV6EsAm1cbZrqNY6tMwlsC8D27TWQcA283Nhw4xGzXvbYig2mKaiuNFb3g0K8ub466MIMbfsJbELYTvqYijDBGC5eXk5MLEOXkPUQHZBMYt2xpGalQ/AhsMpxKfnMj6kNf0CmxHWrskN70sSuhC2qaZ97hrYopTSwCda60VAS611qnn9RaDCq1iUUrOAWQDt2lU/U6AwMRg1y/ckMqJLS9o1u/m+7YJiA+P/bzcJ6bmsjDzPgondmbfhOBH+Xrw7NUSGLgrhIGr6lzxQa90TGAM8rZQaXHalNk05WeG0k1rrRVrrcK11uLf3zd2p/rds1YHz/OWbU9y/ZB9p2YU3vf0vMekkpOfy9ND2JF/N54El+2nk5sLH03tJYhfCgdTor1lrnWL+eRnYAEQAl5RSPgDmn5drGqQwKSg28O6Ws3Ru5UladiGPLj9AfpGh+g3L+OHkRTzdXHh2eEf+Mr4bWsOr47ri1bBeHUUthLCGW07uSqmGSinP0mXgTuAEsAmYaa42E9hY0yCFyc/RaWTkFjFvbBfeuzeMo8lZvP3D2Wq32xefwZHzVykoNrDt9CWGd25BPRcn7o1ox8H5I5gQ6nsbohdC3E416XNvCWwwz9HtAvxXa/29UuoAsFop9QiQBEyteZgC4JtjqXg1rMeA9s1wcXZiRj8/PtuTwMSw1vRoU/FJ0Cu5RTz02QGKDEb6t29GZl4xU3v/7w5GzTzq367whRC30S233LXW8VrrEPOjm9b6TXN5htZ6uNY6SGs9Qmt9pbp9iepdyS1i66lLjO7eChdz3/jcUZ1o2qAe//j+TKXbfbY7gYISAwM7NOeXmHQiArzoF9jsdoUthLASuULVTizdZUrSD/X3t5R5urny9NAOLPjmlKm75brpdY1GzaoD5xneuSWLZ/Tih5MXCW7TRO6IJMRvgAyPsAMGo+Y/+5MY1bUVQS09r1k3va8fHVp48OeNJ8kpLLGUn8vI49Nd8VzOLuR3IT4opRjd3QffJu63O3whhBVIcrcDp1N/5WpeMWOCW5VbV8/Fib/dHUxqVj7z1h+n9Ibns788xF83m7prhnVucVvjFUJYnyR3O7AnLh2g0r7y3v5ePD+yI18fvcCK/ecwGjWnU7MBmDU4EE8319sWqxDCNkifu4364eRFLmYVMLO/P3viMmjv3ZAWjSqfQvepIR2ISsrkja9N3TNFBiP/nBrC3T3b3MaohRC2QlruNupfP0bz+tcnOZB4hX3xGQzs0LzK+k5OioX3htGplSd/N99BqWe7prcjVCGEDZLkboPSsgs5czEbrWHW51EUFBsZdgM3mm7s7soXD/ehcytPfJu443cLc88IIRyDdMvYoNI+9ul9/fhiXxIN6znTN/DGbjTdtGE9vnp6ALmFJTLkUYjfMEnuNmj7mct4urnw5991JfZyDv7NG1DfxfmGt3dzdcbN9cbrCyEcjyR3G3M5u4DNxy/y+4i2uDo78d/H+kgLXAhx06TP3cYs35NIsdHIQwMCACSxCyFuiSR3G5KZW8Sy3YmM7e6Df/OG1g5HCGHHJLnbCK01/+/b0+QVG3h2RJC1wxFC2DlJ7jYgv8jAS+uOse5QMrOHdqDjdfPHCCHEzZLkbgM+/jmO1VHJPHFHe54b0dHa4QghHICMlrGyohIj/408x9BO3rw8prO1wxFCOAhpuVvZ9rOXScsuZEY/f2uHIoRwIJLcrezn6DQ86rswMKjquWOEEOJmSHK3Iq01O6PT6Ne+Ga7O8lEIIWqPZBQrSsrIIzkzn8HSahdC1DJJ7nVIa83BpEz+uOYom45eKLd+b3wGAP2rmc5XCCFuloyWqUMvrD7K+sMpAPx46hJDOnnTqMxdkSITrtDcoz6BcjWqEKKWScu9jmw7fYn1h1N4ZGAAa57oR1Z+MZ/ujL+mTmTCFfoEeMn8MUKIWifJvQ7Ep+Uwd+0xOrTw4KXRnent78VdPXz4dFcC6TmFACRn5pFyNZ+IgBubp10IIW6GJPc68PK64wAsnhFOPRfTIX5hZEcKS4yWW+BFJlwBkOQuhKgTktxrWWTCFSITrzBnWAcCyvSlB3p78NSQ9qw9mMw3xy4QmXCFRm4udJJ5ZIQQdUBOqNYCo1GTnltIC083luyKp2kDV6b1bleu3pzhQeyKTWfe+uNkF5QwoksLnJykv10IUfuk5V4LPt+bSP+//cSWkxfZevoyU8Pb4l6v/G3uXJ2dWDgtjBKDBqC3v3TJCCHqhrTca8ho1Czfm0SJUfPkikMYjJp7I8q32ku1a9aA7X8cwhf7EpkS3vY2RiqE+C2R5H4LMnIKeXblEQK9GzKgQ3MS0nO5r0870rILGdO91TV97RVp1diNuaNkBkghRN2ps+SulBoNLAScgU+11n+vq9eqzMYjKZQYNHf39L3hseQ/R6dxLiOXB/r6VbrNiv3n2BWbzq7YdNZEJdO0gSt/HtcVN9fyXTFCCGENdZLclVLOwAfASCAZOKCU2qS1PlWbr6O1RinFkl0JLN4Zz+zhHTAYNSUGzYAOzZm75hhFBiPx6TlVtpS11qRlF/Lj6Uv8acMJUxkwIcSXx/8TRffWjZk/risABcUGVuxPYnBHb+o5K7aevsz0fn6S2IUQNqWuWu4RQKzWOh5AKbUSmADUanLfevoyf1h1hNyiEhrWc7Ek5lLOTorhnVvw8c/xTAz1JaiSYYef/pLAm5tP46RgWOcWKOCNr0+xbHci8em57Iu/wriQ1oS2bcIbX5/k0q+F/GtqII3cXUnKyGN6X7/afFtCCFFjdZXcfYHzZZ4nA33KVlBKzQJmAbRrV/kJyKr4NHZjcq82uDorZg8PIiUzH4/6LuQUlvD53kT8mzVkcq82DH1nB3/55hSfPxxBcmY+93y0h15+TVkwsTsJ6bl89HMcACO7tuTdqaEATP5oD5ezC/no/p68tukkD34WyUP9A/gy8jxP3NHeMtnXj8/fcUuxCyFEXVJa69rfqVKTgdFa60fNz6cDfbTWz1RUPzw8XEdFRdV6HKWW7krgL9+cYuG9oUQmXGHF/nMAeDWsx5XcIsB0NenIri0t2+QXGTBqTcP6LiSm5zLriyiiL+XQurEbP/1xiHTDCCGsTil1UGsdXtG6umq5pwBlx/m1MZdZxfR+fmw8eoE/rDqCBu7r0w4/rwb87bszTAtvS3Cbxgzv3OKabcqOU/dv3pCVs/oxb/0x7u3dThK7EMLm1VXL3QWIBoZjSuoHgPu01icrql/XLXeA3MIS3v8pFjdXJx4ZGIBHfRdOXviVrj6N5CpRIYRduu0td611iVLqGeAHTEMhl1aW2G+XhvVdeHnMtSNmuvs2tlI0QghRt+psnLvWejOwua72L4QQonIyt4wQQjggSe5CCOGAJLkLIYQDkuQuhBAOSJK7EEI4IEnuQgjhgCS5CyGEA6qTK1RvOgil0oCkW9y8OZBei+HcbvYcv8RuHRK7ddhi7H5aa++KVthEcq8JpVRUZZff2gN7jl9itw6J3TrsLXbplhFCCAckyV0IIRyQIyT3RdYOoIbsOX6J3Tokduuwq9jtvs9dCCFEeY7QchdCCHEdSe5CCOGA7Dq5K6VGK6XOKqVilVIvWzue6iilEpVSx5VSR5RSUeYyL6XUj0qpGPPPptaOE0AptVQpdVkpdaJMWYWxKpP3zJ/DMaVUT+tFXmnsryulUszH/ohSamyZdfPMsZ9VSo2yTtSWWNoqpbYrpU4ppU4qpZ41l9v8sa8idps/9kopN6VUpFLqqDn2N8zlAUqp/eYYVyml6pnL65ufx5rX+1sr9kppre3ygekOT3FAIFAPOAp0tXZc1cScCDS/ruwt4GXz8svAP6wdpzmWwUBP4ER1sQJjge8ABfQF9ttg7K8Df6ygblfz7059IMD8O+Vsxdh9gJ7mZU9Mt6vsag/HvorYbf7Ym4+fh3nZFdhvPp6rgXvN5R8DT5qXnwI+Ni/fC6yy1nGv7GHPLfcIIFZrHa+1LgJWAhOsHNOtmAAsNy8vByZaMRYLrfVO4Mp1xZXFOgH4XJvsA5oopXxuT6TlVRJ7ZSYAK7XWhVrrBCAW0++WVWitU7XWh8zL2cBpwBc7OPZVxF4Zmzn25uOXY37qan5oYBiw1lx+/XEv/TzWAsOVUjZ1M2Z7Tu6+wPkyz5Op+hfJFmhgi1LqoFJqlrmspdY61bx8EWhpndBuSGWx2stn8Yy562Jpme4vm43d/FU/DFMr0q6O/XWxgx0ce6WUs1LqCHAZ+BHTN4mrWuuSCuKzxG5enwU0u70RV82ek7s9Gqi17gmMAZ5WSg0uu1KbvuPZxdhUe4rV7COgPRAKpALvWjecqimlPIB1wHNa61/LrrP1Y19B7HZx7LXWBq11KNAG0zeIzlYOqUbsObmnAG3LPG9jLrNZWusU88/LwAZMv0CXSr9Gm39etl6E1aosVpv/LLTWl8x/vEZgMf/7+m9zsSulXDElxxVa6/XmYrs49hXFbk/HHkBrfRXYDvTD1M3lYl5VNj5L7Ob1jYGM2xxqlew5uR8Agsxns+thOqmxycoxVUop1VAp5Vm6DNwJnMAU80xztZnARutEeEMqi3UTMMM8cqMvkFWmC8EmXNcPPQnTsQdT7PeaRz8EAEFA5O2Or5S533YJcFpr/c8yq2z+2FcWuz0ce6WUt1KqiXnZHRiJ6ZzBdmCyudr1x73085gM/GT+RmU7rH1GtyYPTCMFojH1jf3J2vFUE2sgppEBR4GTpfFi6qfbBsQAWwEva8dqjutLTF+hizH1NT5SWayYRhp8YP4cjgPhNhj7F+bYjmH6w/QpU/9P5tjPAmOsHPtATF0ux4Aj5sdYezj2VcRu88ce6AEcNsd4AvizuTwQ0z+cWGANUN9c7mZ+HmteH2jN35uKHjL9gBBCOCB77pYRQghRCUnuQgjhgCS5CyGEA5LkLoQQDkiSuxBCOCBJ7kII4YAkuQshhAP6/wvOnLuAPqVAAAAAAElFTkSuQmCC\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "plt.plot(score_arr2, label ='accumulated scores in one game play 3')\n",
        "plt.legend(loc='upper left')"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}