makemore4.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rToK0Tku8PPn"
      },
      "source": [
        "## makemore: becoming a backprop ninja"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "ChBbac4y8PPq"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "import torch.nn.functional as F\n",
        "import matplotlib.pyplot as plt # for making figures\n",
        "%matplotlib inline"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "klmu3ZG08PPr"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "32033\n",
            "15\n",
            "['emma', 'olivia', 'ava', 'isabella', 'sophia', 'charlotte', 'mia', 'amelia']\n"
          ]
        }
      ],
      "source": [
        "# read in all the words\n",
        "words = open('names.txt', 'r').read().splitlines()\n",
        "print(len(words))\n",
        "print(max(len(w) for w in words))\n",
        "print(words[:8])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "BCQomLE_8PPs"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{1: 'a', 2: 'b', 3: 'c', 4: 'd', 5: 'e', 6: 'f', 7: 'g', 8: 'h', 9: 'i', 10: 'j', 11: 'k', 12: 'l', 13: 'm', 14: 'n', 15: 'o', 16: 'p', 17: 'q', 18: 'r', 19: 's', 20: 't', 21: 'u', 22: 'v', 23: 'w', 24: 'x', 25: 'y', 26: 'z', 0: '.'}\n",
            "27\n"
          ]
        }
      ],
      "source": [
        "# build the vocabulary of characters and mappings to/from integers\n",
        "chars = sorted(list(set(''.join(words))))\n",
        "stoi = {s:i+1 for i,s in enumerate(chars)}\n",
        "stoi['.'] = 0\n",
        "itos = {i:s for s,i in stoi.items()}\n",
        "vocab_size = len(itos)\n",
        "print(itos)\n",
        "print(vocab_size)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "V_zt2QHr8PPs"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "torch.Size([182625, 3]) torch.Size([182625])\n",
            "torch.Size([22655, 3]) torch.Size([22655])\n",
            "torch.Size([22866, 3]) torch.Size([22866])\n"
          ]
        }
      ],
      "source": [
        "# build the dataset\n",
        "block_size = 3 # context length: how many characters do we take to predict the next one?\n",
        "\n",
        "def build_dataset(words):\n",
        "  X, Y = [], []\n",
        "\n",
        "  for w in words:\n",
        "    context = [0] * block_size\n",
        "    for ch in w + '.':\n",
        "      ix = stoi[ch]\n",
        "      X.append(context)\n",
        "      Y.append(ix)\n",
        "      context = context[1:] + [ix] # crop and append\n",
        "\n",
        "  X = torch.tensor(X)\n",
        "  Y = torch.tensor(Y)\n",
        "  print(X.shape, Y.shape)\n",
        "  return X, Y\n",
        "\n",
        "import random\n",
        "random.seed(42)\n",
        "random.shuffle(words)\n",
        "n1 = int(0.8*len(words))\n",
        "n2 = int(0.9*len(words))\n",
        "\n",
        "Xtr,  Ytr  = build_dataset(words[:n1])     # 80%\n",
        "Xdev, Ydev = build_dataset(words[n1:n2])   # 10%\n",
        "Xte,  Yte  = build_dataset(words[n2:])     # 10%"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "MJPU8HT08PPu"
      },
      "outputs": [],
      "source": [
        "# utility function we will use later when comparing manual gradients to PyTorch gradients\n",
        "def cmp(s, dt, t):\n",
        "  ex = torch.all(dt == t.grad).item()\n",
        "  app = torch.allclose(dt, t.grad)\n",
        "  maxdiff = (dt - t.grad).abs().max().item()\n",
        "  print(f'{s:15s} | exact: {str(ex):5s} | approximate: {str(app):5s} | maxdiff: {maxdiff}')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "id": "ZlFLjQyT8PPu"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "4137\n"
          ]
        }
      ],
      "source": [
        "n_embd = 10 # the dimensionality of the character embedding vectors\n",
        "n_hidden = 64 # the number of neurons in the hidden layer of the MLP\n",
        "\n",
        "g = torch.Generator().manual_seed(2147483647) # for reproducibility\n",
        "C  = torch.randn((vocab_size, n_embd),            generator=g)\n",
        "# Layer 1\n",
        "W1 = torch.randn((n_embd * block_size, n_hidden), generator=g) * (5/3)/((n_embd * block_size)**0.5)\n",
        "b1 = torch.randn(n_hidden,                        generator=g) * 0.1 # using b1 just for fun, it's useless because of BN\n",
        "# Layer 2\n",
        "W2 = torch.randn((n_hidden, vocab_size),          generator=g) * 0.1\n",
        "b2 = torch.randn(vocab_size,                      generator=g) * 0.1\n",
        "# BatchNorm parameters\n",
        "bngain = torch.randn((1, n_hidden))*0.1 + 1.0\n",
        "bnbias = torch.randn((1, n_hidden))*0.1\n",
        "\n",
        "# Note: I am initializating many of these parameters in non-standard ways\n",
        "# because sometimes initializating with e.g. all zeros could mask an incorrect\n",
        "# implementation of the backward pass.\n",
        "\n",
        "parameters = [C, W1, b1, W2, b2, bngain, bnbias]\n",
        "print(sum(p.nelement() for p in parameters)) # number of parameters in total\n",
        "for p in parameters:\n",
        "  p.requires_grad = True"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "QY-y96Y48PPv"
      },
      "outputs": [],
      "source": [
        "batch_size = 32\n",
        "n = batch_size # a shorter variable also, for convenience\n",
        "# construct a minibatch\n",
        "ix = torch.randint(0, Xtr.shape[0], (batch_size,), generator=g)\n",
        "Xb, Yb = Xtr[ix], Ytr[ix] # batch X,Y"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "id": "8ofj1s6d8PPv"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "tensor(3.3309, grad_fn=<NegBackward0>)"
            ]
          },
          "execution_count": 10,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# forward pass, \"chunkated\" into smaller steps that are possible to backward one at a time\n",
        "\n",
        "emb = C[Xb] # embed the characters into vectors\n",
        "embcat = emb.view(emb.shape[0], -1) # concatenate the vectors\n",
        "# Linear layer 1\n",
        "hprebn = embcat @ W1 + b1 # hidden layer pre-activation\n",
        "# BatchNorm layer\n",
        "bnmeani = 1/n*hprebn.sum(0, keepdim=True)\n",
        "bndiff = hprebn - bnmeani\n",
        "bndiff2 = bndiff**2\n",
        "bnvar = 1/(n-1)*(bndiff2).sum(0, keepdim=True) # note: Bessel's correction (dividing by n-1, not n)\n",
        "bnvar_inv = (bnvar + 1e-5)**-0.5\n",
        "bnraw = bndiff * bnvar_inv\n",
        "hpreact = bngain * bnraw + bnbias\n",
        "# Non-linearity\n",
        "h = torch.tanh(hpreact) # hidden layer\n",
        "# Linear layer 2\n",
        "logits = h @ W2 + b2 # output layer\n",
        "# cross entropy loss (same as F.cross_entropy(logits, Yb))\n",
        "logit_maxes = logits.max(1, keepdim=True).values\n",
        "norm_logits = logits - logit_maxes # subtract max for numerical stability\n",
        "counts = norm_logits.exp()\n",
        "counts_sum = counts.sum(1, keepdims=True)\n",
        "counts_sum_inv = counts_sum**-1 # if I use (1.0 / counts_sum) instead then I can't get backprop to be bit exact...\n",
        "probs = counts * counts_sum_inv\n",
        "logprobs = probs.log()\n",
        "loss = -logprobs[range(n), Yb].mean()\n",
        "\n",
        "# PyTorch backward pass\n",
        "for p in parameters:\n",
        "  p.grad = None\n",
        "for t in [logprobs, probs, counts, counts_sum, counts_sum_inv, # afaik there is no cleaner way\n",
        "          norm_logits, logit_maxes, logits, h, hpreact, bnraw,\n",
        "         bnvar_inv, bnvar, bndiff2, bndiff, hprebn, bnmeani,\n",
        "         embcat, emb]:\n",
        "  t.retain_grad()\n",
        "loss.backward()\n",
        "loss"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "id": "mO-8aqxK8PPw"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "logprobs        | exact: True  | approximate: True  | maxdiff: 0.0\n",
            "probs           | exact: True  | approximate: True  | maxdiff: 0.0\n",
            "counts_sum_inv  | exact: True  | approximate: True  | maxdiff: 0.0\n",
            "counts_sum      | exact: True  | approximate: True  | maxdiff: 0.0\n",
            "counts          | exact: True  | approximate: True  | maxdiff: 0.0\n",
            "norm_logits     | exact: True  | approximate: True  | maxdiff: 0.0\n",
            "logit_maxes     | exact: True  | approximate: True  | maxdiff: 0.0\n",
            "logits          | exact: True  | approximate: True  | maxdiff: 0.0\n",
            "h               | exact: True  | approximate: True  | maxdiff: 0.0\n",
            "W2              | exact: True  | approximate: True  | maxdiff: 0.0\n",
            "b2              | exact: True  | approximate: True  | maxdiff: 0.0\n",
            "hpreact         | exact: True  | approximate: True  | maxdiff: 0.0\n",
            "bngain          | exact: True  | approximate: True  | maxdiff: 0.0\n",
            "bnbias          | exact: True  | approximate: True  | maxdiff: 0.0\n",
            "bnraw           | exact: True  | approximate: True  | maxdiff: 0.0\n",
            "bnvar_inv       | exact: True  | approximate: True  | maxdiff: 0.0\n",
            "bnvar           | exact: True  | approximate: True  | maxdiff: 0.0\n",
            "bndiff2         | exact: True  | approximate: True  | maxdiff: 0.0\n",
            "bndiff          | exact: True  | approximate: True  | maxdiff: 0.0\n",
            "bnmeani         | exact: True  | approximate: True  | maxdiff: 0.0\n",
            "hprebn          | exact: True  | approximate: True  | maxdiff: 0.0\n",
            "embcat          | exact: True  | approximate: True  | maxdiff: 0.0\n",
            "W1              | exact: True  | approximate: True  | maxdiff: 0.0\n",
            "b1              | exact: True  | approximate: True  | maxdiff: 0.0\n",
            "emb             | exact: True  | approximate: True  | maxdiff: 0.0\n",
            "C               | exact: True  | approximate: True  | maxdiff: 0.0\n"
          ]
        }
      ],
      "source": [
        "# Exercise 1: backprop through the whole thing manually,\n",
        "# backpropagating through exactly all of the variables\n",
        "# as they are defined in the forward pass above, one by one\n",
        "\n",
        "# loss = -logprobs[range(n), Yb].mean()\n",
        "# what is the derivative of mean? --> mean is just addition and division, derivative of divide is just 1/x, where x is the count\n",
        "# why do i need to multiply by the one hot to change shape though? --> because of logprobs[range(n), Yb] is picking out row 1-2 at column Yb\n",
        "dlogprobs = -F.one_hot(Yb, num_classes=27).float() * torch.tensor(1.0 / len(logprobs))\n",
        "cmp('logprobs', dlogprobs, logprobs)\n",
        "\n",
        "# logprobs = probs.log()\n",
        "# derivative of log x is 1/x\n",
        "dprobs = 1/probs * dlogprobs\n",
        "cmp('probs', dprobs, probs)\n",
        "\n",
        "# probs = counts * counts_sum_inv\n",
        "# derivative of multiply is just the other thing\n",
        "# why do i need to sum to change the shape though? probably because of the broadcasting rules?\n",
        "dcounts_sum_inv = (dprobs * counts).sum(1, keepdims=True)\n",
        "cmp('counts_sum_inv', dcounts_sum_inv, counts_sum_inv)\n",
        "\n",
        "# counts_sum_inv = counts_sum**-1\n",
        "# derivative of x^-1 is -1/x^2\n",
        "dcounts_sum = (-counts_sum**-2) * dcounts_sum_inv\n",
        "cmp('counts_sum', dcounts_sum, counts_sum)\n",
        "\n",
        "# counts_sum = counts.sum(1, keepdims=True)\n",
        "# derivative of sum is just 1\n",
        "# probs = counts * counts_sum_inv\n",
        "# derivative of multiply is just the other thing\n",
        "# since counts is used twice, we need to add the gradients together\n",
        "dcounts = torch.ones_like(counts) * dcounts_sum\n",
        "dcounts += dprobs * counts_sum_inv\n",
        "cmp('counts', dcounts, counts)\n",
        "\n",
        "# counts = norm_logits.exp()\n",
        "# derivative of exp is exp\n",
        "dnorm_logits = norm_logits.exp() * dcounts\n",
        "cmp('norm_logits', dnorm_logits, norm_logits)\n",
        "\n",
        "# norm_logits = logits - logit_maxes\n",
        "# derivative of subtract is just -1\n",
        "dlogit_maxes = -dnorm_logits.sum(1, keepdims=True)\n",
        "cmp('logit_maxes', dlogit_maxes, logit_maxes)\n",
        "\n",
        "# logit_maxes = logits.max(1, keepdim=True).values\n",
        "# norm_logits = logits - logit_maxes\n",
        "dlogits = F.one_hot(logits.max(1).indices, num_classes=27).float() * dlogit_maxes\n",
        "dlogits += dnorm_logits\n",
        "cmp('logits', dlogits, logits)\n",
        "\n",
        "# logits = h @ W2 + b2\n",
        "dh = dlogits @ W2.T\n",
        "cmp('h', dh, h)\n",
        "dW2 = h.T @ dlogits\n",
        "cmp('W2', dW2, W2)\n",
        "db2 = dlogits.sum(0)\n",
        "cmp('b2', db2, b2)\n",
        "\n",
        "# h = torch.tanh(hpreact)\n",
        "# derivative of tanh is 1 - tanh^2\n",
        "dhpreact = (1 - h**2) * dh\n",
        "cmp('hpreact', dhpreact, hpreact)\n",
        "\n",
        "# hpreact = bngain * bnraw + bnbias\n",
        "# this is element wise multiplication, not matrix multiplication!\n",
        "dbngain = (bnraw * dhpreact).sum(0, keepdims=True)\n",
        "dbnraw = bngain * dhpreact\n",
        "dbnbias = dhpreact.sum(0, keepdims=True)\n",
        "cmp('bngain', dbngain, bngain)\n",
        "cmp('bnbias', dbnbias, bnbias)\n",
        "cmp('bnraw', dbnraw, bnraw)\n",
        "\n",
        "dbnvar_inv = (bndiff * dbnraw).sum(0, keepdims=True)\n",
        "cmp('bnvar_inv', dbnvar_inv, bnvar_inv)\n",
        "\n",
        "# bnvar_inv = (bnvar + 1e-5)**-0.5\n",
        "# derivative of x^-0.5 is -0.5*x^-1.5\n",
        "dbnvar = (-0.5*(bnvar + 1e-5)**-1.5) * dbnvar_inv\n",
        "cmp('bnvar', dbnvar, bnvar)\n",
        "\n",
        "# bnvar = 1/(n-1)*(bndiff2).sum(0, keepdim=True)\n",
        "dbndiff2 = torch.ones_like(bndiff2) * (1/(n-1)) * dbnvar\n",
        "cmp('bndiff2', dbndiff2, bndiff2)\n",
        "\n",
        "# bnraw = bndiff * bnvar_inv\n",
        "# again, element wise multiplication\n",
        "dbndiff = bnvar_inv * dbnraw\n",
        "dbndiff += 2 * bndiff * dbndiff2\n",
        "cmp('bndiff', dbndiff, bndiff)\n",
        "\n",
        "# bndiff = hprebn - bnmeani\n",
        "# bnmeani = 1/n*hprebn.sum(0, keepdim=True)\n",
        "dbnmeani = -dbndiff.sum(0, keepdims=True)\n",
        "dhprebn = dbndiff.clone()\n",
        "dhprebn += 1/n * dbnmeani\n",
        "cmp('bnmeani', dbnmeani, bnmeani)\n",
        "cmp('hprebn', dhprebn, hprebn)\n",
        "\n",
        "# hprebn = embcat @ W1 + b1\n",
        "dembcat = dhprebn @ W1.T\n",
        "cmp('embcat', dembcat, embcat)\n",
        "dW1 = embcat.T @ dhprebn\n",
        "cmp('W1', dW1, W1)\n",
        "db1 = dhprebn.sum(0)\n",
        "cmp('b1', db1, b1)\n",
        "\n",
        "# embcat = emb.view(emb.shape[0], -1)\n",
        "demb = dembcat.view(emb.shape)\n",
        "cmp('emb', demb, emb)\n",
        "\n",
        "# emb = C[Xb]\n",
        "dC = torch.zeros_like(C)\n",
        "for k in range(Xb.shape[0]):\n",
        "    for j in range(Xb.shape[1]):\n",
        "        dC[Xb[k,j]] += demb[k,j]\n",
        "cmp('C', dC, C)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "ebLtYji_8PPw"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "3.330932378768921 diff: 0.0\n"
          ]
        }
      ],
      "source": [
        "# Exercise 2: backprop through cross_entropy but all in one go\n",
        "# to complete this challenge look at the mathematical expression of the loss,\n",
        "# take the derivative, simplify the expression, and just write it out\n",
        "\n",
        "# forward pass\n",
        "\n",
        "# before:\n",
        "# logit_maxes = logits.max(1, keepdim=True).values\n",
        "# norm_logits = logits - logit_maxes # subtract max for numerical stability\n",
        "# counts = norm_logits.exp()\n",
        "# counts_sum = counts.sum(1, keepdims=True)\n",
        "# counts_sum_inv = counts_sum**-1 # if I use (1.0 / counts_sum) instead then I can't get backprop to be bit exact...\n",
        "# probs = counts * counts_sum_inv\n",
        "# logprobs = probs.log()\n",
        "# loss = -logprobs[range(n), Yb].mean()\n",
        "\n",
        "# now:\n",
        "loss_fast = F.cross_entropy(logits, Yb)\n",
        "print(loss_fast.item(), 'diff:', (loss_fast - loss).item())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "id": "-gCXbB4C8PPx"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "logits          | exact: False | approximate: True  | maxdiff: 4.889443516731262e-09\n"
          ]
        }
      ],
      "source": [
        "# backward pass\n",
        "\n",
        "# -----------------\n",
        "# YOUR CODE HERE :)\n",
        "dlogits = F.softmax(logits, 1) # TODO. my solution is 3 lines\n",
        "dlogits[range(n), Yb] -= 1\n",
        "dlogits /= n\n",
        "# -----------------\n",
        "\n",
        "cmp('logits', dlogits, logits) # I can only get approximate to be true, my maxdiff is 6e-9"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "<matplotlib.image.AxesImage at 0x1490d0950>"
            ]
          },
          "execution_count": 14,
          "metadata": {},
          "output_type": "execute_result"
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAKTCAYAAADlpSlWAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAxfElEQVR4nO3df4zcdZ0/8Nfs7O7stt0uLtBuKy0UEKpAewlCbVSOkx6lJkSkJvgjOTAEo1fIQeNpelERz6R3mKjnBfGfOzgTqx4XwWhyGK1SYq7AWY8gctS2wFGutBW03Xbb/TUz3z/6Zc+VFtj2VWZ59/FIJunOTJ/7ms98Pp957mdnP1NpNpvNAAAoRFurBwAAyKTcAABFUW4AgKIoNwBAUZQbAKAoyg0AUBTlBgAoSnurB/hjjUYjduzYET09PVGpVFo9DgAwBTSbzdi3b1/MnTs32tpe+djMlCs3O3bsiHnz5rV6DABgCtq+fXucdtppr3ifKVduenp6IiLiv/7rv8b/fSza2/Me4p49e9KyIiI6OzvTskZGRtKyent707IiIgYGBtKyqtVqWtb555+flvWrX/0qLSsiUo9aZmY1Go20rOyTo2c+zrGxsbSszMf5aj+tTlbmbN3d3WlZmetZ5r4xIvc5mDZtWlpW5jIbGhpKy4rIm23//v3xrne96zV1gylXbl7aQfX09KSUm46OjmPOeEm9Xk/Lipi65SZjuf+hzB1oZrnJlL3MlJvJU24mT7mZvBOh3GS+bkbkzhbx2rZ1bygGAIqi3AAARVFuAICiHLdyc8cdd8QZZ5wRXV1dsWTJknjkkUeO17cCABh3XMrNd7/73Vi9enXceuut8ctf/jIWL14cy5cvj927dx+PbwcAMO64lJsvf/nLccMNN8RHP/rReNvb3hbf+MY3Ytq0afHP//zPx+PbAQCMSy83IyMjsWnTpli2bNn/fZO2tli2bFls3LjxZfcfHh6OgYGBCRcAgKOVXm5eeOGFqNfrMXv27AnXz549O3bu3Pmy+69duzZ6e3vHL85ODAAci5b/tdSaNWti796945ft27e3eiQA4A0s/QzFp5xySlSr1di1a9eE63ft2hX9/f0vu3+tVotarZY9BgBwgko/ctPZ2RkXXnhhrF+/fvy6RqMR69evj6VLl2Z/OwCACY7LZ0utXr06rr322nj7298eF198cXz1q1+NwcHB+OhHP3o8vh0AwLjjUm6uueaa+O1vfxuf+9znYufOnfEnf/Incf/997/sTcYAANmO26eC33jjjXHjjTcer3gAgMNq+V9LAQBkUm4AgKIct19LHat6vR71ej0lJ8tJJ52UlhURMTQ0lJbV3p73VB44cCAtKyKi2WymZbW15fXxbdu2pWU1Go20rIiIjo6OtKypuvwzt82IiHPOOScta8uWLWlZmetG9nqW+XxmzjYyMpKWlS1zvc1cZpmvJ5nrRUTePqhSqbzm+zpyAwAURbkBAIqi3AAARVFuAICiKDcAQFGUGwCgKMoNAFAU5QYAKIpyAwAURbkBAIqi3AAARVFuAICiKDcAQFGUGwCgKMoNAFAU5QYAKIpyAwAURbkBAIrS3uoBjmRoaCg6OjqOOadSqSRMc8iBAwfSsrK1teX11Pb23NWiq6srLSvz+ezu7k7LGhkZScvKzqtWq1Myq7OzMy0rIuLJJ59My1qwYEFa1m9+85u0rOxts9lspmX19vamZWXua4eHh9OyInL3taOjo2lZmXPV6/W0rIi8/fZkchy5AQCKotwAAEVRbgCAoig3AEBRlBsAoCjKDQBQFOUGACiKcgMAFEW5AQCKotwAAEVRbgCAoig3AEBRlBsAoCjKDQBQFOUGACiKcgMAFEW5AQCKotwAAEVRbgCAorS3eoAjqVarUa1Wjzmn0WgkTHNIrVZLy8rW0dGRljU8PJyWFRExNjaWllWpVNKyMtXr9dS89va8TTNzG8h8nM1mMy0rIqK7uzst63//93/TsoaGhtKyMp/LiNznYN++fWlZmcss28KFC9OyNm/enJaVuW/MfD3JNJn9oiM3AEBRlBsAoCjKDQBQFOUGACiKcgMAFEW5AQCKotwAAEVRbgCAoig3AEBRlBsAoCjKDQBQFOUGACiKcgMAFEW5AQCKotwAAEVRbgCAoig3AEBRlBsAoCjtrR7gSM4///yUnG3btqXkRESMjY2lZUVENJvNtKyRkZG0rM7OzrSsiNzlVq/X07JqtVpaVnt77qaUuW5kLrOOjo60rOztqVKppGWddtppaVlPPfVUWlb2tpmprS3vZ+XM9Sxz3xgRsXnz5rSszO08c5mNjo6mZUXkzvZaOXIDABRFuQEAiqLcAABFUW4AgKIoNwBAUZQbAKAoyg0AUBTlBgAoinIDABRFuQEAiqLcAABFUW4AgKIoNwBAUZQbAKAoyg0AUBTlBgAoinIDABRFuQEAitLe6gGO5PHHH4+enp5WjzFBe3vu4qpUKmlZ1Wo1LevAgQNpWdmmTZuWljU8PJyWVa/X07IiImq1WlpW5myZWdnbU0dHR1rWc889l5aVaXR0NDUv8/l861vfmpa1devWtKzMfWNE7no7MjIyJbOyX3uHhoZScprN5mu+ryM3AEBRlBsAoCjKDQBQFOUGACiKcgMAFCW93Hz+85+PSqUy4bJw4cLsbwMAcFjH5U/BzzvvvPjJT37yf98k+U8+AQCO5Li0jvb29ujv7z8e0QAAr+i4vOdmy5YtMXfu3DjzzDPjIx/5SDz77LNHvO/w8HAMDAxMuAAAHK30crNkyZK4++674/77748777wznn766Xj3u98d+/btO+z9165dG729veOXefPmZY8EAJxAKs3JnM/4KOzZsydOP/30+PKXvxzXX3/9y24fHh6ecBr8gYGBmDdvno9fmKQT5eMXMj+W4ET5+IXM07JnrrPZ21NnZ2daVubHHEzV5R9xYnz8QvZL3FT9+IVMU/XjF/bt2xfnnXde7N27N2bOnPmK9z3u7/Q96aST4pxzzjniylqr1VJ35ADAie24n+dm//79sW3btpgzZ87x/lYAAPnl5pOf/GRs2LAhnnnmmfiP//iPeP/73x/VajU+9KEPZX8rAICXSf+11HPPPRcf+tCH4sUXX4xTTz013vWud8VDDz0Up556ava3AgB4mfRy853vfCc7EgDgNfPZUgBAUZQbAKAoU/ZDnzo6OqKjo+OYczLP2ZL9J+v79+9Py8o890L2eSG6urrSsjJny1xm55xzTlpWRMSTTz6ZlpX5OBuNRlpW5rlksvOmTZuWltXd3Z2WlX0Oqszz3Gzbti0tK3M9m8qfbThVzxs1ODiYlhWR9zgns144cgMAFEW5AQCKotwAAEVRbgCAoig3AEBRlBsAoCjKDQBQFOUGACiKcgMAFEW5AQCKotwAAEVRbgCAoig3AEBRlBsAoCjKDQBQFOUGACiKcgMAFEW5AQCKotwAAEVpb/UAR1Kv16Nerx9zTnt73kM8cOBAWlZExKxZs9KyXnzxxbSsWq2WlhURMTw8nJbV09OTljU4OJiW9cQTT6RlRUS0teX93DE6OpqWValU0rK6urrSsiIi3vzmN6dlbd26NS1rKst8PmfOnJmWtXfv3rSsbGNjY2lZ1Wo1LSvj9fIlma+bERGNRiMlZzLrqyM3AEBRlBsAoCjKDQBQFOUGACiKcgMAFEW5AQCKotwAAEVRbgCAoig3AEBRlBsAoCjKDQBQFOUGACiKcgMAFEW5AQCKotwAAEVRbgCAoig3AEBRlBsAoCjtrR7gjaTZbKbm/f73v0/LGhsbS8s699xz07IiIrZt25aWValU0rIajUZaVlvb1P05ob09bzPPfJwjIyNpWRERW7ZsScvKXM8yZT6XERH1ej01L0vm8u/q6krLiogYGhpKzcsylbfNarWakjOZ1+Cpu0cGADgKyg0AUBTlBgAoinIDABRFuQEAiqLcAABFUW4AgKIoNwBAUZQbAKAoyg0AUBTlBgAoinIDABRFuQEAiqLcAABFUW4AgKIoNwBAUZQbAKAoyg0AUJT2Vg9wJPV6Per1+jHnnHHGGcc+zP/3zDPPpGVFRIyNjaVldXZ2pmVt3bo1LSsi93EODAykZfX09KRljYyMpGVFRBw4cCAtq6OjIy2r2WymZVWr1bSsiNzZ2tryfu7r6upKyxodHU3Lish9nHv27EnL6u7uTsvat29fWlZExLRp09KyMrfzzO2pvT23GmS9BkymEzhyAwAURbkBAIqi3AAARVFuAICiKDcAQFGUGwCgKMoNAFAU5QYAKIpyAwAURbkBAIqi3AAARVFuAICiKDcAQFGUGwCgKMoNAFAU5QYAKIpyAwAURbkBAIrS3uoBjqRer0e9Xj/mnC1btiRMc0ilUknLiojo6OhIy2o0GmlZ2TJny1gnXjI4OJiW1daW+3NCZt7Y2FhaVldXV1rWyMhIWlZERHt73u5s9uzZaVkvvPBCWla1Wk3LishdZpnb+fz589Oyfv3rX6dlRUTs378/LStzO898fcrcz0bkzTaZHEduAICiKDcAQFGUGwCgKMoNAFAU5QYAKIpyAwAUZdLl5sEHH4wrr7wy5s6dG5VKJe67774Jtzebzfjc5z4Xc+bMie7u7li2bFnqn2MDALySSZebwcHBWLx4cdxxxx2Hvf3222+Pr33ta/GNb3wjHn744Zg+fXosX748hoaGjnlYAIBXM+kzOK1YsSJWrFhx2NuazWZ89atfjc985jPxvve9LyIivvnNb8bs2bPjvvvuiw9+8IMv+z/Dw8MxPDw8/vXAwMBkRwIAGJf6npunn346du7cGcuWLRu/rre3N5YsWRIbN2487P9Zu3Zt9Pb2jl/mzZuXORIAcIJJLTc7d+6MiJefunz27Nnjt/2xNWvWxN69e8cv27dvzxwJADjBtPyzpWq1WtRqtVaPAQAUIvXITX9/f0RE7Nq1a8L1u3btGr8NAOB4Si03CxYsiP7+/li/fv34dQMDA/Hwww/H0qVLM78VAMBhTfrXUvv374+tW7eOf/3000/Ho48+Gn19fTF//vy4+eab44tf/GK85S1viQULFsRnP/vZmDt3blx11VWZcwMAHNaky80vfvGL+LM/+7Pxr1evXh0REddee23cfffd8alPfSoGBwfjYx/7WOzZsyfe9a53xf333x9dXV15UwMAHMGky82ll14azWbziLdXKpX4whe+EF/4wheOaTAAgKPhs6UAgKIoNwBAUVp+npsjqVQqUalUjjmno6MjYZpD6vV6WlZExOWXX56Wdf/996dlTZ8+PS0rYuo+B41GIy0re93InC1jO3pJ5mfEtbXl/myVOduzzz6bllWtVqdkVkTEwYMH07K6u7vTsp5++um0rLGxsbSs7LzM5zNzO29vz60GIyMjKTmv9JaYP+bIDQBQFOUGACiKcgMAFEW5AQCKotwAAEVRbgCAoig3AEBRlBsAoCjKDQBQFOUGACiKcgMAFEW5AQCKotwAAEVRbgCAoig3AEBRlBsAoCjKDQBQFOUGAChKe6sHON7GxsbSsmq1WlpWRMT999+fllWtVtOyDhw4kJYVETFjxoy0rNHR0bSshQsXpmX95je/ScuKyF1v29vzNvNKpZKWVa/X07IiItra8n5W6+zsTMvq7u5OyxoaGkrLiojo6OhIyxoeHk7Lylxns51yyilpWbt3707LynwNyNzOM/Mmk+PIDQBQFOUGACiKcgMAFEW5AQCKotwAAEVRbgCAoig3AEBRlBsAoCjKDQBQFOUGACiKcgMAFEW5AQCKotwAAEVRbgCAoig3AEBRlBsAoCjKDQBQFOUGAChKe6sHON6q1WpaVltbbhesVCppWY1GIy1r5syZaVkREfv27UvLynycTzzxRFpWtsz1ttlspmV1dnamZWXOFRGxcOHCtKytW7emZQ0ODqZlZe4zIiKmTZuWlrV37960rFqtlpY1PDyclhUR8cILL6RlZW5P2etGpqxtfTL7RUduAICiKDcAQFGUGwCgKMoNAFAU5QYAKIpyAwAURbkBAIqi3AAARVFuAICiKDcAQFGUGwCgKMoNAFAU5QYAKIpyAwAURbkBAIqi3AAARVFuAICiKDcAQFHaWz3AkXR0dERHR8cx54yNjSVMc8jIyEhaVkREV1dXWtbQ0FBa1sGDB9OyIiIqlUpa1rRp09Ky6vV6Wla2zGVWrVbTss4444y0rC1btqRlRUQ8+eSTaVmjo6NpWZlqtVpq3v79+9Oyuru707KazWZaVvYyy3xNyZS5P8tc/pkm8xgduQEAiqLcAABFUW4AgKIoNwBAUZQbAKAoyg0AUBTlBgAoinIDABRFuQEAiqLcAABFUW4AgKIoNwBAUZQbAKAoyg0AUBTlBgAoinIDABRFuQEAiqLcAABFUW4AgKK0t3qAI1m8eHFUKpVjznnmmWeOfZj/b2RkJC0rImJoaCg1L0tPT09q3p49e9Kypuoy6+joaPUIr4unnnoqLevgwYNpWRER1Wo1LavZbKZldXZ2pmVlr//d3d1pWZnPZ3t73ktTvV5Py4qIaGvLOyZQq9XSsjLX2ez1rNFopORM5jE6cgMAFEW5AQCKotwAAEVRbgCAoig3AEBRJl1uHnzwwbjyyitj7ty5UalU4r777ptw+3XXXReVSmXC5YorrsiaFwDgFU263AwODsbixYvjjjvuOOJ9rrjiinj++efHL9/+9rePaUgAgNdq0icTWLFiRaxYseIV71Or1aK/v/+ohwIAOFrH5T03DzzwQMyaNSvOPffc+MQnPhEvvvjiEe87PDwcAwMDEy4AAEcrvdxcccUV8c1vfjPWr18ff//3fx8bNmyIFStWHPEskWvXro3e3t7xy7x587JHAgBOIOkfv/DBD35w/N8XXHBBLFq0KM4666x44IEH4rLLLnvZ/desWROrV68e/3pgYEDBAQCO2nH/U/AzzzwzTjnllNi6dethb6/VajFz5swJFwCAo3Xcy81zzz0XL774YsyZM+d4fysAgMn/Wmr//v0TjsI8/fTT8eijj0ZfX1/09fXFbbfdFitXroz+/v7Ytm1bfOpTn4qzzz47li9fnjo4AMDhTLrc/OIXv4g/+7M/G//6pffLXHvttXHnnXfGY489Fv/yL/8Se/bsiblz58bll18ef/u3f5v60e4AAEcy6XJz6aWXRrPZPOLtP/rRj45pIACAY+GzpQCAoig3AEBR0s9zk2XTpk3R09NzzDlDQ0MJ0xwyffr0tKyIQ2dnztLZ2ZmWdfDgwbSsiDjiCRyPRrVaTctqNBppWSMjI2lZEbnP5/z589OynnnmmbSs7u7utKyIqbtuZG9PmTL3j5nr7NjYWFpW5nMZMXX3Z5nLrL09txq0teUcRxkdHX3t3zPlOwIATBHKDQBQFOUGACiKcgMAFEW5AQCKotwAAEVRbgCAoig3AEBRlBsAoCjKDQBQFOUGACiKcgMAFEW5AQCKotwAAEVRbgCAoig3AEBRlBsAoCjKDQBQlPZWD3AkF110UVQqlWPO2b59e8I0hwwNDaVlRUTK43vJyMhIWlaz2UzLish9nNOnT0/L2r9/f1pWo9FIy4qI6OjoSMvasmVLWla9Xk/LGhsbS8uKyF1mmY8zc3uqVqtpWRG56217e97LSeb+rLOzMy0rInfdyHycbW15xyqm6mvAZHIcuQEAiqLcAABFUW4AgKIoNwBAUZQbAKAoyg0AUBTlBgAoinIDABRFuQEAiqLcAABFUW4AgKIoNwBAUZQbAKAoyg0AUBTlBgAoinIDABRFuQEAiqLcAABFaW/1AEfyyCOPRE9PzzHn7N27N2GaQ2q1WlpWRMTw8HBaVrVaTctqNBppWRERM2bMSMs6ePBgWlbm85m9zPbv35+W1dHRkZZVqVTSsur1elpWRMTIyEhaVmdnZ1pW5vo/NDSUlhUR0Ww207IyZ8tc/tnbZl9fX1rW7t2707Km8rZ5xhlnpORMZn115AYAKIpyAwAURbkBAIqi3AAARVFuAICiKDcAQFGUGwCgKMoNAFAU5QYAKIpyAwAURbkBAIqi3AAARVFuAICiKDcAQFGUGwCgKMoNAFAU5QYAKIpyAwAURbkBAIrS3uoBjre2trz+1mg00rIiIiqVSlpW5uPMzIrIXW6Zs42OjqZlnX322WlZERFbtmxJy8pcz9rb83YZ1Wo1LSsi4uDBg2lZY2NjaVmDg4NpWfV6PS0rInd76u3tTcvKXGbNZjMtKyJiYGAgLaurqystK3M/m72ebd26NSVn3759sXjx4td0X0duAICiKDcAQFGUGwCgKMoNAFAU5QYAKIpyAwAURbkBAIqi3AAARVFuAICiKDcAQFGUGwCgKMoNAFAU5QYAKIpyAwAURbkBAIqi3AAARVFuAICiKDcAQFHaWz3AkXR3d0d3d/cx5wwODiZMc0iz2UzLiohob89b/JmzVavVtKyIiIMHD6ZlZc6WmfWb3/wmLSsioqurKy1rZGQkLStzPRseHk7Lioio1WppWZ2dnWlZ+/fvT8uqVCppWRERbW15P99mbueZ62zmY4yIGBsbS83Lkvk4zzvvvLSsiIjNmzen5EzmNdORGwCgKMoNAFAU5QYAKIpyAwAURbkBAIoyqXKzdu3auOiii6KnpydmzZoVV1111cveBT00NBSrVq2Kk08+OWbMmBErV66MXbt2pQ4NAHAkkyo3GzZsiFWrVsVDDz0UP/7xj2N0dDQuv/zyCX9ufcstt8QPfvCDuOeee2LDhg2xY8eOuPrqq9MHBwA4nEmdaOX++++f8PXdd98ds2bNik2bNsUll1wSe/fujX/6p3+KdevWxXve856IiLjrrrvirW99azz00EPxjne8I29yAIDDOKb33OzduzciIvr6+iIiYtOmTTE6OhrLli0bv8/ChQtj/vz5sXHjxsNmDA8Px8DAwIQLAMDROupy02g04uabb453vvOdcf7550dExM6dO6OzszNOOumkCfedPXt27Ny587A5a9eujd7e3vHLvHnzjnYkAICjLzerVq2Kxx9/PL7zne8c0wBr1qyJvXv3jl+2b99+THkAwIntqD7c6MYbb4wf/vCH8eCDD8Zpp502fn1/f3+MjIzEnj17Jhy92bVrV/T39x82q1arpX4mDABwYpvUkZtmsxk33nhj3HvvvfHTn/40FixYMOH2Cy+8MDo6OmL9+vXj123evDmeffbZWLp0ac7EAACvYFJHblatWhXr1q2L73//+9HT0zP+Ppre3t7o7u6O3t7euP7662P16tXR19cXM2fOjJtuuimWLl3qL6UAgNfFpMrNnXfeGRERl1566YTr77rrrrjuuusiIuIrX/lKtLW1xcqVK2N4eDiWL18eX//611OGBQB4NZMqN81m81Xv09XVFXfccUfccccdRz0UAMDR8tlSAEBRlBsAoChH9afgr4cLLrggKpXKMef8z//8T8I0h4yMjKRlZRsbG0vL6ujoSMuKyJ2trS2vj4+OjqZlvZZf2U5G5jJrNBppWcPDw2lZmc9lRO5zkPk4M/ZjL6lWq2lZEbnrWW9vb1rWwYMH07Ky1ev1tKz29qn5EvzEE0+k5mXtgyaT48gNAFAU5QYAKIpyAwAURbkBAIqi3AAARVFuAICiKDcAQFGUGwCgKMoNAFAU5QYAKIpyAwAURbkBAIqi3AAARVFuAICiKDcAQFGUGwCgKMoNAFAU5QYAKEp7qwc4kkceeSR6enqOOWf27NkJ0xyyffv2tKyIiOHh4bSsarWaljU4OJiWFRExY8aMtKyhoaG0rFqtlpbVaDTSsiJy142Ojo60rEqlkpZVr9fTsiIiRkZG0rI6OzvTsjL2Yy/JXP8jcteN3//+92lZU3nb7OvrS8vavXt3Wlbma0Dmdh6R9xxMJseRGwCgKMoNAFAU5QYAKIpyAwAURbkBAIqi3AAARVFuAICiKDcAQFGUGwCgKMoNAFAU5QYAKIpyAwAURbkBAIqi3AAARVFuAICiKDcAQFGUGwCgKMoNAFAU5QYAKEp7qwc4ko6Ojujo6DjmnEqlkjDNIWNjY2lZ2Wq1WlrWyMhIWlZE7nJrNBppWcPDw2lZ7e25m1J2XpbM5Z+5bUZEdHZ2pmVlztZsNtOyRkdH07IiIqrValpW5rqRvQ/KlLlutLXlHV/o6upKy8pe/lnrxmS2JUduAICiKDcAQFGUGwCgKMoNAFAU5QYAKIpyAwAURbkBAIqi3AAARVFuAICiKDcAQFGUGwCgKMoNAFAU5QYAKIpyAwAURbkBAIqi3AAARVFuAICiKDcAQFHaWz3AkTQajWg0Gsec89vf/jZhmkMGBgbSsiIiarVaWtbw8HBa1vTp09OyIiL279+flvWWt7wlLWvLli1pWfV6PS0rIuKkk05Ky/rd736XltXWlvfzUPYy6+joSMsaHR1NyxoZGUnLajabaVkRkbKPfUl7e97LydjYWFpW5jobEbF79+60rPnz56dlZb7WZa9nnZ2dr3uOIzcAQFGUGwCgKMoNAFAU5QYAKIpyAwAURbkBAIqi3AAARVFuAICiKDcAQFGUGwCgKMoNAFAU5QYAKIpyAwAURbkBAIqi3AAARVFuAICiKDcAQFGUGwCgKO2tHuBIarVa1Gq1Y87Zt29fwjSHNBqNtKyIiJGRkbSsarWaltXenrtaZM62bdu2tKxms5mWValU0rIiIvbs2ZOW1dnZmZbV1pb389Do6GhaVkTE2NhYWlbmupG5PWXvg84///y0rMceeywtK3Ofkb3Menp60rJ27dqVlpW5nWcvs6Ghodc9x5EbAKAoyg0AUBTlBgAoinIDABRFuQEAijKpcrN27dq46KKLoqenJ2bNmhVXXXVVbN68ecJ9Lr300qhUKhMuH//4x1OHBgA4kkmVmw0bNsSqVavioYceih//+McxOjoal19+eQwODk643w033BDPP//8+OX2229PHRoA4EgmdQKG+++/f8LXd999d8yaNSs2bdoUl1xyyfj106ZNi/7+/pwJAQAm4Zjec7N3796IiOjr65tw/be+9a045ZRT4vzzz481a9bEgQMHjpgxPDwcAwMDEy4AAEfrqE+d2Wg04uabb453vvOdE85y+eEPfzhOP/30mDt3bjz22GPx6U9/OjZv3hzf+973Dpuzdu3auO222452DACACY663KxatSoef/zx+PnPfz7h+o997GPj/77gggtizpw5cdlll8W2bdvirLPOelnOmjVrYvXq1eNfDwwMxLx58452LADgBHdU5ebGG2+MH/7wh/Hggw/Gaaed9or3XbJkSUREbN269bDlJuszpAAAIiZZbprNZtx0001x7733xgMPPBALFix41f/z6KOPRkTEnDlzjmpAAIDJmFS5WbVqVaxbty6+//3vR09PT+zcuTMiInp7e6O7uzu2bdsW69ati/e+971x8sknx2OPPRa33HJLXHLJJbFo0aLj8gAAAP7QpMrNnXfeGRGHTtT3h+6666647rrrorOzM37yk5/EV7/61RgcHIx58+bFypUr4zOf+UzawAAAr2TSv5Z6JfPmzYsNGzYc00AAAMfCZ0sBAEVRbgCAohz1eW6Ot9HR0RgdHW31GBO0teV2wXq9npbV2dmZlrVnz560rIiIGTNmpGUdPHgwLevVfs06Geeee25aVkTEE088kZqXpVKptHqEI8rcPhuNRlpW5qkuhoaG0rIictezzHUjc99YrVbTsiIiZs6cmZa1Y8eOtKzM/Vnm8m8VR24AgKIoNwBAUZQbAKAoyg0AUBTlBgAoinIDABRFuQEAiqLcAABFUW4AgKIoNwBAUZQbAKAoyg0AUBTlBgAoinIDABRFuQEAiqLcAABFUW4AgKIoNwBAUZQbAKAo7a0e4Ejq9XrU6/VWjzFBR0dHat5ZZ52VlvXMM8+kZVUqlbSsiIgDBw6kZTWbzbSstra8bv/UU0+lZUVEDA0NpWWNjY2lZTUajbSsbO3tebuzarWalpW5/DMfY7aRkZG0rDe96U1pWb/73e/SsiIiXnjhhbSszP3Z6OhoWlb2etbV1ZWSM5nH6MgNAFAU5QYAKIpyAwAURbkBAIqi3AAARVFuAICiKDcAQFGUGwCgKMoNAFAU5QYAKIpyAwAURbkBAIqi3AAARVFuAICiKDcAQFGUGwCgKMoNAFAU5QYAKEp7qwc4klqtFrVa7ZhzRkdHE6Y5ZGRkJC0rImLbtm2peVkuuOCC1Lxf//rXaVmVSiUtK/P5bDQaaVkRER0dHWlZ9Xp9SmZlazabaVmZj3PatGlpWYODg2lZEZGyj31JW1vez8oDAwNpWdVqNS0rInc9mz59elpW5j5jz549aVkRefvtyeyzHbkBAIqi3AAARVFuAICiKDcAQFGUGwCgKMoNAFAU5QYAKIpyAwAURbkBAIqi3AAARVFuAICiKDcAQFGUGwCgKMoNAFAU5QYAKIpyAwAURbkBAIqi3AAARWlv9QBHcvDgwWhvP/bxms1mwjSHVKvVtKxsGcvqJb/61a/SsiJyZxsdHU3LmjFjRlrWvHnz0rIiIrZs2ZKW1daW9zNM5vZUqVTSsiJyZ+vq6krLOnDgQFpWtsztKfP5zFxnx8bG0rIicvdng4ODaVkdHR1pWd3d3WlZEXnPwWSWvSM3AEBRlBsAoCjKDQBQFOUGACiKcgMAFEW5AQCKotwAAEVRbgCAoig3AEBRlBsAoCjKDQBQFOUGACiKcgMAFEW5AQCKotwAAEVRbgCAoig3AEBRlBsAoCjtrR7gSC688MKoVCrHnPPUU08lTHPI2NhYWlZERFdXV1pW5mydnZ1pWRERIyMjaVkZ68RLDh48mJa1efPmtKyI3MdZr9fTsprNZlpWW1vuz1aZj3NoaCgtK/NxZj7GiNz1rL19ar6cZC+zzHWjp6cnLStzPdu3b19aVkTeetZoNF7zfR25AQCKotwAAEVRbgCAoig3AEBRlBsAoCjKDQBQlEmVmzvvvDMWLVoUM2fOjJkzZ8bSpUvj3//938dvHxoailWrVsXJJ58cM2bMiJUrV8auXbvShwYAOJJJlZvTTjst/u7v/i42bdoUv/jFL+I973lPvO9974tf//rXERFxyy23xA9+8IO45557YsOGDbFjx464+uqrj8vgAACHU2ke41m5+vr64ktf+lJ84AMfiFNPPTXWrVsXH/jAByIi4sknn4y3vvWtsXHjxnjHO95x2P8/PDwcw8PD418PDAzEvHnzor293Un8JiFztuyTq03Vk/hN5oRQrybz5HYREdVqNS1rqj7OqXwSv8wT0k3lk/hlrmeZWZky9z8RudvA9OnT07JOhJP47du3LxYtWhR79+6NmTNnvuJ9j3pp1Ov1+M53vhODg4OxdOnS2LRpU4yOjsayZcvG77Nw4cKYP39+bNy48Yg5a9eujd7e3vHLvHnzjnYkAIDJl5tf/epXMWPGjKjVavHxj3887r333njb294WO3fujM7OzjjppJMm3H/27Nmxc+fOI+atWbMm9u7dO37Zvn37pB8EAMBLJn3s9dxzz41HH3009u7dG//2b/8W1157bWzYsOGoB6jValGr1Y76/wMA/KFJl5vOzs44++yzI+LQh1v+53/+Z/zDP/xDXHPNNTEyMhJ79uyZcPRm165d0d/fnzYwAMArOeZ3IDUajRgeHo4LL7wwOjo6Yv369eO3bd68OZ599tlYunTpsX4bAIDXZFJHbtasWRMrVqyI+fPnx759+2LdunXxwAMPxI9+9KPo7e2N66+/PlavXh19fX0xc+bMuOmmm2Lp0qVH/EspAIBskyo3u3fvjr/4i7+I559/Pnp7e2PRokXxox/9KP78z/88IiK+8pWvRFtbW6xcuTKGh4dj+fLl8fWvf/24DA4AcDjHfJ6bbAMDA9Hb2+s8N5PkPDeTN1XP/xLhPDdHw3luJs95bibPeW4m7w11nhsAgKlIuQEAipJ37DXZY489Fj09PcecMzo6mjDNId3d3WlZEREHDhxIy3q1Q3STMTg4mJYVkXsoPfPQa+ava7LXjcxD6dm//smSvcz+8GNcppKp+uuaiBg/rUeGJ554Ii0rc93IfjvBjBkz0rKy97VZstfZrOdgMvvsqbnXAwA4SsoNAFAU5QYAKIpyAwAURbkBAIqi3AAARVFuAICiKDcAQFGUGwCgKMoNAFAU5QYAKIpyAwAURbkBAIqi3AAARVFuAICiKDcAQFGUGwCgKO2tHuCPNZvNiIjYv39/St7o6GhKTkTE2NhYWlZExIEDB9KyKpVKWtbg4GBaVkTucmtry+vjjUYjLSt73RgZGUnLylw3MmUvs+Hh4dS8LNVqNS2rXq+nZUX83/42w759+9KyMteNzP1sRO4yO3jwYFpWpsx1NiLv+XypF7yW56DSzHymEjz33HMxb968Vo8BAExB27dvj9NOO+0V7zPlyk2j0YgdO3ZET0/PK/7EOTAwEPPmzYvt27fHzJkzX8cJibD8W83ybz3PQWtZ/q3ViuXfbDZj3759MXfu3Fc9ij/lfi3V1tb2qo3sD82cOdOK3UKWf2tZ/q3nOWgty7+1Xu/l39vb+5ru5w3FAEBRlBsAoChv2HJTq9Xi1ltvjVqt1upRTkiWf2tZ/q3nOWgty7+1pvryn3JvKAYAOBZv2CM3AACHo9wAAEVRbgCAoig3AEBRlBsAoChvyHJzxx13xBlnnBFdXV2xZMmSeOSRR1o90gnj85//fFQqlQmXhQsXtnqsYj344INx5ZVXxty5c6NSqcR999034fZmsxmf+9znYs6cOdHd3R3Lli2LLVu2tGbYAr3a8r/uuutetj1cccUVrRm2QGvXro2LLrooenp6YtasWXHVVVfF5s2bJ9xnaGgoVq1aFSeffHLMmDEjVq5cGbt27WrRxGV5Lcv/0ksvfdk28PGPf7xFE/+fN1y5+e53vxurV6+OW2+9NX75y1/G4sWLY/ny5bF79+5Wj3bCOO+88+L5558fv/z85z9v9UjFGhwcjMWLF8cdd9xx2Ntvv/32+NrXvhbf+MY34uGHH47p06fH8uXLY2ho6HWetEyvtvwjIq644ooJ28O3v/3t13HCsm3YsCFWrVoVDz30UPz4xz+O0dHRuPzyy2NwcHD8Prfcckv84Ac/iHvuuSc2bNgQO3bsiKuvvrqFU5fjtSz/iIgbbrhhwjZw++23t2jiP9B8g7n44oubq1atGv+6Xq83586d21y7dm0Lpzpx3Hrrrc3Fixe3eowTUkQ077333vGvG41Gs7+/v/mlL31p/Lo9e/Y0a7Va89vf/nYLJizbHy//ZrPZvPbaa5vve9/7WjLPiWj37t3NiGhu2LCh2WweWt87Ojqa99xzz/h9/vu//7sZEc2NGze2asxi/fHybzabzT/90z9t/tVf/VXrhjqCN9SRm5GRkdi0aVMsW7Zs/Lq2trZYtmxZbNy4sYWTnVi2bNkSc+fOjTPPPDM+8pGPxLPPPtvqkU5ITz/9dOzcuXPC9tDb2xtLliyxPbyOHnjggZg1a1ace+658YlPfCJefPHFVo9UrL1790ZERF9fX0REbNq0KUZHRydsAwsXLoz58+fbBo6DP17+L/nWt74Vp5xySpx//vmxZs2aOHDgQCvGm2DKfSr4K3nhhReiXq/H7NmzJ1w/e/bsePLJJ1s01YllyZIlcffdd8e5554bzz//fNx2223x7ne/Ox5//PHo6elp9XgnlJ07d0ZEHHZ7eOk2jq8rrrgirr766liwYEFs27Yt/uZv/iZWrFgRGzdujGq12urxitJoNOLmm2+Od77znXH++edHxKFtoLOzM0466aQJ97UN5Dvc8o+I+PCHPxynn356zJ07Nx577LH49Kc/HZs3b47vfe97LZz2DVZuaL0VK1aM/3vRokWxZMmSOP300+Nf//Vf4/rrr2/hZPD6++AHPzj+7wsuuCAWLVoUZ511VjzwwANx2WWXtXCy8qxatSoef/xx7/FrkSMt/4997GPj/77gggtizpw5cdlll8W2bdvirLPOer3HHPeG+rXUKaecEtVq9WXvhN+1a1f09/e3aKoT20knnRTnnHNObN26tdWjnHBeWudtD1PHmWeeGaeccortIdmNN94YP/zhD+NnP/tZnHbaaePX9/f3x8jISOzZs2fC/W0DuY60/A9nyZIlEREt3wbeUOWms7MzLrzwwli/fv34dY1GI9avXx9Lly5t4WQnrv3798e2bdtizpw5rR7lhLNgwYLo7++fsD0MDAzEww8/bHtokeeeey5efPFF20OSZrMZN954Y9x7773x05/+NBYsWDDh9gsvvDA6OjombAObN2+OZ5991jaQ4NWW/+E8+uijEREt3wbecL+WWr16dVx77bXx9re/PS6++OL46le/GoODg/HRj3601aOdED75yU/GlVdeGaeffnrs2LEjbr311qhWq/GhD32o1aMVaf/+/RN+Anr66afj0Ucfjb6+vpg/f37cfPPN8cUvfjHe8pa3xIIFC+Kzn/1szJ07N6666qrWDV2QV1r+fX19cdttt8XKlSujv78/tm3bFp/61Kfi7LPPjuXLl7dw6nKsWrUq1q1bF9///vejp6dn/H00vb290d3dHb29vXH99dfH6tWro6+vL2bOnBk33XRTLF26NN7xjne0ePo3vldb/tu2bYt169bFe9/73jj55JPjsccei1tuuSUuueSSWLRoUWuHb/Wfax2Nf/zHf2zOnz+/2dnZ2bz44oubDz30UKtHOmFcc801zTlz5jQ7Ozubb37zm5vXXHNNc+vWra0eq1g/+9nPmhHxssu1117bbDYP/Tn4Zz/72ebs2bObtVqtedlllzU3b97c2qEL8krL/8CBA83LL7+8eeqppzY7Ojqap59+evOGG25o7ty5s9VjF+Nwyz4imnfdddf4fQ4ePNj8y7/8y+ab3vSm5rRp05rvf//7m88//3zrhi7Iqy3/Z599tnnJJZc0+/r6mrVarXn22Wc3//qv/7q5d+/e1g7ebDYrzWaz+XqWKQCA4+kN9Z4bAIBXo9wAAEVRbgCAoig3AEBRlBsAoCjKDQBQFOUGACiKcgMAFEW5AQCKotwAAEVRbgCAovw/waV15qnRLhQAAAAASUVORK5CYII=",
            "text/plain": [
              "<Figure size 800x800 with 1 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "plt.figure(figsize=(8,8))\n",
        "plt.imshow(dlogits.detach(), cmap='gray')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "id": "hd-MkhB68PPy"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "max diff: tensor(7.1526e-07, grad_fn=<MaxBackward1>)\n"
          ]
        }
      ],
      "source": [
        "# Exercise 3: backprop through batchnorm but all in one go\n",
        "# to complete this challenge look at the mathematical expression of the output of batchnorm,\n",
        "# take the derivative w.r.t. its input, simplify the expression, and just write it out\n",
        "# BatchNorm paper: https://arxiv.org/abs/1502.03167\n",
        "\n",
        "# forward pass\n",
        "\n",
        "# before:\n",
        "# bnmeani = 1/n*hprebn.sum(0, keepdim=True)\n",
        "# bndiff = hprebn - bnmeani\n",
        "# bndiff2 = bndiff**2\n",
        "# bnvar = 1/(n-1)*(bndiff2).sum(0, keepdim=True) # note: Bessel's correction (dividing by n-1, not n)\n",
        "# bnvar_inv = (bnvar + 1e-5)**-0.5\n",
        "# bnraw = bndiff * bnvar_inv\n",
        "# hpreact = bngain * bnraw + bnbias\n",
        "\n",
        "# now:\n",
        "hpreact_fast = bngain * (hprebn - hprebn.mean(0, keepdim=True)) / torch.sqrt(hprebn.var(0, keepdim=True, unbiased=True) + 1e-5) + bnbias\n",
        "print('max diff:', (hpreact_fast - hpreact).abs().max())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {
        "id": "POdeZSKT8PPy"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "hprebn          | exact: False | approximate: True  | maxdiff: 1.3969838619232178e-09\n"
          ]
        }
      ],
      "source": [
        "# backward pass\n",
        "\n",
        "# before we had:\n",
        "# dbnraw = bngain * dhpreact\n",
        "# dbndiff = bnvar_inv * dbnraw\n",
        "# dbnvar_inv = (bndiff * dbnraw).sum(0, keepdim=True)\n",
        "# dbnvar = (-0.5*(bnvar + 1e-5)**-1.5) * dbnvar_inv\n",
        "# dbndiff2 = (1.0/(n-1))*torch.ones_like(bndiff2) * dbnvar\n",
        "# dbndiff += (2*bndiff) * dbndiff2\n",
        "# dhprebn = dbndiff.clone()\n",
        "# dbnmeani = (-dbndiff).sum(0)\n",
        "# dhprebn += 1.0/n * (torch.ones_like(hprebn) * dbnmeani)\n",
        "\n",
        "# calculate dhprebn given dhpreact (i.e. backprop through the batchnorm)\n",
        "# (you'll also need to use some of the variables from the forward pass up above)\n",
        "\n",
        "# -----------------\n",
        "# YOUR CODE HERE :)\n",
        "dhprebn = bngain*bnvar_inv/n * (n*dhpreact - dhpreact.sum(0) - n/(n-1)*bnraw*(dhpreact*bnraw).sum(0))\n",
        "# -----------------\n",
        "\n",
        "cmp('hprebn', dhprebn, hprebn) # I can only get approximate to be true, my maxdiff is 9e-10"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {
        "id": "wPy8DhqB8PPz"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "12297\n",
            "      0/ 200000: 3.7655\n",
            "  10000/ 200000: 2.1754\n",
            "  20000/ 200000: 2.3733\n",
            "  30000/ 200000: 2.4834\n",
            "  40000/ 200000: 2.0243\n",
            "  50000/ 200000: 2.4005\n",
            "  60000/ 200000: 2.3633\n",
            "  70000/ 200000: 2.0211\n",
            "  80000/ 200000: 2.3625\n",
            "  90000/ 200000: 2.2026\n",
            " 100000/ 200000: 2.0031\n",
            " 110000/ 200000: 2.3039\n",
            " 120000/ 200000: 1.9287\n",
            " 130000/ 200000: 2.4301\n",
            " 140000/ 200000: 2.2502\n",
            " 150000/ 200000: 2.1879\n",
            " 160000/ 200000: 2.0090\n",
            " 170000/ 200000: 1.8695\n",
            " 180000/ 200000: 2.0596\n",
            " 190000/ 200000: 1.9262\n"
          ]
        }
      ],
      "source": [
        "# Exercise 4: putting it all together!\n",
        "# Train the MLP neural net with your own backward pass\n",
        "\n",
        "# init\n",
        "n_embd = 10 # the dimensionality of the character embedding vectors\n",
        "n_hidden = 200 # the number of neurons in the hidden layer of the MLP\n",
        "\n",
        "g = torch.Generator().manual_seed(2147483647) # for reproducibility\n",
        "C  = torch.randn((vocab_size, n_embd),            generator=g)\n",
        "# Layer 1\n",
        "W1 = torch.randn((n_embd * block_size, n_hidden), generator=g) * (5/3)/((n_embd * block_size)**0.5)\n",
        "b1 = torch.randn(n_hidden,                        generator=g) * 0.1\n",
        "# Layer 2\n",
        "W2 = torch.randn((n_hidden, vocab_size),          generator=g) * 0.1\n",
        "b2 = torch.randn(vocab_size,                      generator=g) * 0.1\n",
        "# BatchNorm parameters\n",
        "bngain = torch.randn((1, n_hidden))*0.1 + 1.0\n",
        "bnbias = torch.randn((1, n_hidden))*0.1\n",
        "\n",
        "parameters = [C, W1, b1, W2, b2, bngain, bnbias]\n",
        "print(sum(p.nelement() for p in parameters)) # number of parameters in total\n",
        "for p in parameters:\n",
        "  p.requires_grad = True\n",
        "\n",
        "# same optimization as last time\n",
        "max_steps = 200000\n",
        "batch_size = 32\n",
        "n = batch_size # convenience\n",
        "lossi = []\n",
        "\n",
        "# use this context manager for efficiency once your backward pass is written (TODO)\n",
        "with torch.no_grad():\n",
        "\n",
        "  # kick off optimization\n",
        "  for i in range(max_steps):\n",
        "\n",
        "    # minibatch construct\n",
        "    ix = torch.randint(0, Xtr.shape[0], (batch_size,), generator=g)\n",
        "    Xb, Yb = Xtr[ix], Ytr[ix] # batch X,Y\n",
        "\n",
        "    # forward pass\n",
        "    emb = C[Xb] # embed the characters into vectors\n",
        "    embcat = emb.view(emb.shape[0], -1) # concatenate the vectors\n",
        "    # Linear layer\n",
        "    hprebn = embcat @ W1 + b1 # hidden layer pre-activation\n",
        "    # BatchNorm layer\n",
        "    # -------------------------------------------------------------\n",
        "    bnmean = hprebn.mean(0, keepdim=True)\n",
        "    bnvar = hprebn.var(0, keepdim=True, unbiased=True)\n",
        "    bnvar_inv = (bnvar + 1e-5)**-0.5\n",
        "    bnraw = (hprebn - bnmean) * bnvar_inv\n",
        "    hpreact = bngain * bnraw + bnbias\n",
        "    # -------------------------------------------------------------\n",
        "    # Non-linearity\n",
        "    h = torch.tanh(hpreact) # hidden layer\n",
        "    logits = h @ W2 + b2 # output layer\n",
        "    loss = F.cross_entropy(logits, Yb) # loss function\n",
        "\n",
        "    # backward pass\n",
        "    for p in parameters:\n",
        "      p.grad = None\n",
        "    # loss.backward() # use this for correctness comparisons, delete it later!\n",
        "\n",
        "    # manual backprop! #swole_doge_meme\n",
        "    # -----------------\n",
        "    dlogits = F.softmax(logits, 1)\n",
        "    dlogits[range(n), Yb] -= 1\n",
        "    dlogits /= n\n",
        "    # 2nd layer backprop\n",
        "    dh = dlogits @ W2.T\n",
        "    dW2 = h.T @ dlogits\n",
        "    db2 = dlogits.sum(0)\n",
        "    # tanh\n",
        "    dhpreact = (1 - h**2) * dh\n",
        "    # batch norm backprop\n",
        "    dbngain = (bnraw * dhpreact).sum(0, keepdims=True)\n",
        "    dbnraw = bngain * dhpreact\n",
        "    dbnbias = dhpreact.sum(0, keepdims=True)\n",
        "    dhprebn = bngain*bnvar_inv/n * (n*dhpreact - dhpreact.sum(0) - n/(n-1)*bnraw*(dhpreact*bnraw).sum(0))\n",
        "    # 1st layer\n",
        "    dembcat = dhprebn @ W1.T\n",
        "    dW1 = embcat.T @ dhprebn\n",
        "    db1 = dhprebn.sum(0)\n",
        "    # embedding\n",
        "    demb = dembcat.view(emb.shape)\n",
        "    dC = torch.zeros_like(C)\n",
        "    for k in range(Xb.shape[0]):\n",
        "        for j in range(Xb.shape[1]):\n",
        "            dC[Xb[k,j]] += demb[k,j]\n",
        "\n",
        "    grads = [dC, dW1, db1, dW2, db2, dbngain, dbnbias]\n",
        "    # -----------------\n",
        "\n",
        "    # update\n",
        "    lr = 0.1 if i < 100000 else 0.01 # step learning rate decay\n",
        "    for p, grad in zip(parameters, grads):\n",
        "      # p.data += -lr * p.grad # old way of cheems doge (using PyTorch grad from .backward())\n",
        "      p.data += -lr * grad # new way of swole doge TODO: enable\n",
        "\n",
        "    # track stats\n",
        "    if i % 10000 == 0: # print every once in a while\n",
        "      print(f'{i:7d}/{max_steps:7d}: {loss.item():.4f}')\n",
        "    lossi.append(loss.log10().item())\n",
        "\n",
        "    # if i >= 100: # TODO: delete early breaking when you're ready to train the full net\n",
        "    #   break"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "id": "ZEpI0hMW8PPz"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "(27, 10)        | exact: False | approximate: True  | maxdiff: 1.1175870895385742e-08\n",
            "(30, 200)       | exact: False | approximate: True  | maxdiff: 1.1175870895385742e-08\n",
            "(200,)          | exact: False | approximate: True  | maxdiff: 4.6566128730773926e-09\n",
            "(200, 27)       | exact: False | approximate: True  | maxdiff: 2.2351741790771484e-08\n",
            "(27,)           | exact: False | approximate: True  | maxdiff: 7.450580596923828e-09\n",
            "(1, 200)        | exact: False | approximate: True  | maxdiff: 2.3283064365386963e-09\n",
            "(1, 200)        | exact: False | approximate: True  | maxdiff: 3.725290298461914e-09\n"
          ]
        }
      ],
      "source": [
        "# useful for checking your gradients\n",
        "# for p,g in zip(parameters, grads):\n",
        "#   cmp(str(tuple(p.shape)), g, p)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 23,
      "metadata": {
        "id": "KImLWNoh8PP0"
      },
      "outputs": [],
      "source": [
        "# calibrate the batch norm at the end of training\n",
        "\n",
        "with torch.no_grad():\n",
        "  # pass the training set through\n",
        "  emb = C[Xtr]\n",
        "  embcat = emb.view(emb.shape[0], -1)\n",
        "  hpreact = embcat @ W1 + b1\n",
        "  # measure the mean/std over the entire training set\n",
        "  bnmean = hpreact.mean(0, keepdim=True)\n",
        "  bnvar = hpreact.var(0, keepdim=True, unbiased=True)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 24,
      "metadata": {
        "id": "6aFnP_Zc8PP0"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "train 2.0688934326171875\n",
            "val 2.110504388809204\n"
          ]
        }
      ],
      "source": [
        "# evaluate train and val loss\n",
        "\n",
        "@torch.no_grad() # this decorator disables gradient tracking\n",
        "def split_loss(split):\n",
        "  x,y = {\n",
        "    'train': (Xtr, Ytr),\n",
        "    'val': (Xdev, Ydev),\n",
        "    'test': (Xte, Yte),\n",
        "  }[split]\n",
        "  emb = C[x] # (N, block_size, n_embd)\n",
        "  embcat = emb.view(emb.shape[0], -1) # concat into (N, block_size * n_embd)\n",
        "  hpreact = embcat @ W1 + b1\n",
        "  hpreact = bngain * (hpreact - bnmean) * (bnvar + 1e-5)**-0.5 + bnbias\n",
        "  h = torch.tanh(hpreact) # (N, n_hidden)\n",
        "  logits = h @ W2 + b2 # (N, vocab_size)\n",
        "  loss = F.cross_entropy(logits, y)\n",
        "  print(split, loss.item())\n",
        "\n",
        "split_loss('train')\n",
        "split_loss('val')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "esWqmhyj8PP1"
      },
      "outputs": [],
      "source": [
        "# I achieved:\n",
        "# train 2.0718822479248047\n",
        "# val 2.1162495613098145"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 25,
      "metadata": {
        "id": "xHeQNv3s8PP1"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "carlah.\n",
            "amelle.\n",
            "khi.\n",
            "mri.\n",
            "reigh.\n",
            "skanden.\n",
            "jazhnen.\n",
            "delynn.\n",
            "jareen.\n",
            "nellara.\n",
            "chaily.\n",
            "kaleigh.\n",
            "ham.\n",
            "joce.\n",
            "quint.\n",
            "shoison.\n",
            "jadbi.\n",
            "waythoniearyn.\n",
            "kai.\n",
            "eveigh.\n"
          ]
        }
      ],
      "source": [
        "# sample from the model\n",
        "g = torch.Generator().manual_seed(2147483647 + 10)\n",
        "\n",
        "for _ in range(20):\n",
        "\n",
        "    out = []\n",
        "    context = [0] * block_size # initialize with all ...\n",
        "    while True:\n",
        "      # forward pass\n",
        "      emb = C[torch.tensor([context])] # (1,block_size,d)\n",
        "      embcat = emb.view(emb.shape[0], -1) # concat into (N, block_size * n_embd)\n",
        "      hpreact = embcat @ W1 + b1\n",
        "      hpreact = bngain * (hpreact - bnmean) * (bnvar + 1e-5)**-0.5 + bnbias\n",
        "      h = torch.tanh(hpreact) # (N, n_hidden)\n",
        "      logits = h @ W2 + b2 # (N, vocab_size)\n",
        "      # sample\n",
        "      probs = F.softmax(logits, dim=1)\n",
        "      ix = torch.multinomial(probs, num_samples=1, generator=g).item()\n",
        "      context = context[1:] + [ix]\n",
        "      out.append(ix)\n",
        "      if ix == 0:\n",
        "        break\n",
        "\n",
        "    print(''.join(itos[i] for i in out))"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.11.4"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}