Chapter6 - Intro to Backpropagation - Building Your First DEEP Neural Network.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Creating a Matrix or Two in Python"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error:0.03999999999999998 Prediction:-0.19999999999999996\n",
      "Error:0.025599999999999973 Prediction:-0.15999999999999992\n",
      "Error:0.01638399999999997 Prediction:-0.1279999999999999\n",
      "Error:0.010485759999999964 Prediction:-0.10239999999999982\n",
      "Error:0.006710886399999962 Prediction:-0.08191999999999977\n",
      "Error:0.004294967295999976 Prediction:-0.06553599999999982\n",
      "Error:0.002748779069439994 Prediction:-0.05242879999999994\n",
      "Error:0.0017592186044416036 Prediction:-0.04194304000000004\n",
      "Error:0.0011258999068426293 Prediction:-0.03355443200000008\n",
      "Error:0.0007205759403792803 Prediction:-0.02684354560000002\n",
      "Error:0.0004611686018427356 Prediction:-0.021474836479999926\n",
      "Error:0.0002951479051793508 Prediction:-0.01717986918399994\n",
      "Error:0.00018889465931478573 Prediction:-0.013743895347199997\n",
      "Error:0.00012089258196146188 Prediction:-0.010995116277759953\n",
      "Error:7.737125245533561e-05 Prediction:-0.008796093022207963\n",
      "Error:4.951760157141604e-05 Prediction:-0.007036874417766459\n",
      "Error:3.169126500570676e-05 Prediction:-0.0056294995342132115\n",
      "Error:2.028240960365233e-05 Prediction:-0.004503599627370569\n",
      "Error:1.298074214633813e-05 Prediction:-0.003602879701896544\n",
      "Error:8.307674973656916e-06 Prediction:-0.002882303761517324\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "weights = np.array([0.5,0.48,-0.7])\n",
    "alpha = 0.1\n",
    "\n",
    "streetlights = np.array( [ [ 1, 0, 1 ],\n",
    "                           [ 0, 1, 1 ],\n",
    "                           [ 0, 0, 1 ],\n",
    "                           [ 1, 1, 1 ],\n",
    "                           [ 0, 1, 1 ],\n",
    "                           [ 1, 0, 1 ] ] )\n",
    "\n",
    "walk_vs_stop = np.array( [ 0, 1, 0, 1, 1, 0 ] )\n",
    "\n",
    "input = streetlights[0] # [1,0,1]\n",
    "goal_prediction = walk_vs_stop[0] # equals 0... i.e. \"stop\"\n",
    "\n",
    "for iteration in range(20):\n",
    "    prediction = input.dot(weights)\n",
    "    error = (goal_prediction - prediction) ** 2\n",
    "    delta = prediction - goal_prediction\n",
    "    weights = weights - (alpha * (input * delta))\t\n",
    "\n",
    "    print(\"Error:\" + str(error) + \" Prediction:\" + str(prediction))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Building Our Neural Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0 2 4 3]\n",
      "[2 3 4 4]\n",
      "[0.  0.5 1.  0.5]\n",
      "[0.5 1.5 2.5 1.5]\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "a = np.array([0,1,2,1])\n",
    "b = np.array([2,2,2,3])\n",
    "\n",
    "print(a*b) #elementwise multiplication\n",
    "print(a+b) #elementwise addition\n",
    "print(a * 0.5) # vector-scalar multiplication\n",
    "print(a + 0.5) # vector-scalar addition"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Learning the whole dataset!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prediction:-0.19999999999999996\n",
      "Prediction:-0.19999999999999996\n",
      "Prediction:-0.5599999999999999\n",
      "Prediction:0.6160000000000001\n",
      "Prediction:0.17279999999999995\n",
      "Prediction:0.17552\n",
      "Error:2.6561231104\n",
      "\n",
      "Prediction:0.14041599999999999\n",
      "Prediction:0.3066464\n",
      "Prediction:-0.34513824\n",
      "Prediction:1.006637344\n",
      "Prediction:0.4785034751999999\n",
      "Prediction:0.26700416768\n",
      "Error:0.9628701776715985\n",
      "\n",
      "Prediction:0.213603334144\n",
      "Prediction:0.5347420299776\n",
      "Prediction:-0.26067345110016\n",
      "Prediction:1.1319428845096962\n",
      "Prediction:0.6274723921901568\n",
      "Prediction:0.25433999330650114\n",
      "Error:0.5509165866836797\n",
      "\n",
      "Prediction:0.20347199464520088\n",
      "Prediction:0.6561967149569552\n",
      "Prediction:-0.221948503950995\n",
      "Prediction:1.166258650532124\n",
      "Prediction:0.7139004922542389\n",
      "Prediction:0.21471099528371604\n",
      "Error:0.36445836852222424\n",
      "\n",
      "Prediction:0.17176879622697283\n",
      "Prediction:0.7324724146523222\n",
      "Prediction:-0.19966478845083285\n",
      "Prediction:1.1697769945341199\n",
      "Prediction:0.7719890116601171\n",
      "Prediction:0.17297997428859369\n",
      "Error:0.2516768662079895\n",
      "\n",
      "Prediction:0.13838397943087496\n",
      "Prediction:0.7864548139561468\n",
      "Prediction:-0.1836567869927348\n",
      "Prediction:1.163248019006011\n",
      "Prediction:0.8148799260629888\n",
      "Prediction:0.1362897844408577\n",
      "Error:0.17797575048089034\n",
      "\n",
      "Prediction:0.10903182755268614\n",
      "Prediction:0.8273717796510367\n",
      "Prediction:-0.17037324196481937\n",
      "Prediction:1.1537962739591756\n",
      "Prediction:0.8481754931254761\n",
      "Prediction:0.1059488041691444\n",
      "Error:0.12864460733422164\n",
      "\n",
      "Prediction:0.0847590433353155\n",
      "Prediction:0.859469609749935\n",
      "Prediction:-0.1585508402022421\n",
      "Prediction:1.1438418857156731\n",
      "Prediction:0.8746623946770374\n",
      "Prediction:0.08148074110264475\n",
      "Error:0.09511036950476208\n",
      "\n",
      "Prediction:0.06518459288211581\n",
      "Prediction:0.8850633823431538\n",
      "Prediction:-0.14771905585408038\n",
      "Prediction:1.1341830033853888\n",
      "Prediction:0.8959860107828534\n",
      "Prediction:0.0619780399014222\n",
      "Error:0.07194564247043436\n",
      "\n",
      "Prediction:0.04958243192113776\n",
      "Prediction:0.9056327614440267\n",
      "Prediction:-0.13768337501215525\n",
      "Prediction:1.1250605910610996\n",
      "Prediction:0.9132624284442169\n",
      "Prediction:0.04653264583708144\n",
      "Error:0.05564914990717743\n",
      "\n",
      "Prediction:0.03722611666966513\n",
      "Prediction:0.922234066504699\n",
      "Prediction:-0.12834662236261596\n",
      "Prediction:1.116526024487899\n",
      "Prediction:0.9273167105424409\n",
      "Prediction:0.03435527296969987\n",
      "Error:0.04394763937673939\n",
      "\n",
      "Prediction:0.027484218375759886\n",
      "Prediction:0.9356694192994068\n",
      "Prediction:-0.11964712469387503\n",
      "Prediction:1.1085678053734553\n",
      "Prediction:0.9387866868342218\n",
      "Prediction:0.024792915481941458\n",
      "Error:0.035357967050948465\n",
      "\n",
      "Prediction:0.019834332385553155\n",
      "Prediction:0.946566624680628\n",
      "Prediction:-0.11153724870006754\n",
      "Prediction:1.1011550767549563\n",
      "Prediction:0.948176009263518\n",
      "Prediction:0.017315912033043404\n",
      "Error:0.02890700056547436\n",
      "\n",
      "Prediction:0.013852729626434732\n",
      "Prediction:0.9554239432448665\n",
      "Prediction:-0.10397589092234266\n",
      "Prediction:1.0942524239871314\n",
      "Prediction:0.9558862588907013\n",
      "Prediction:0.011498267782398985\n",
      "Error:0.023951660591138853\n",
      "\n",
      "Prediction:0.009198614225919194\n",
      "Prediction:0.9626393189117293\n",
      "Prediction:-0.09692579020989642\n",
      "Prediction:1.087824783849832\n",
      "Prediction:0.9622390773804066\n",
      "Prediction:0.006998674002545002\n",
      "Error:0.020063105176016144\n",
      "\n",
      "Prediction:0.005598939202035996\n",
      "Prediction:0.9685315005838672\n",
      "Prediction:-0.09035250869077546\n",
      "Prediction:1.0818389613301889\n",
      "Prediction:0.9674926590701334\n",
      "Prediction:0.003544193999268516\n",
      "Error:0.016952094519447087\n",
      "\n",
      "Prediction:0.0028353551994148157\n",
      "Prediction:0.9733561723362383\n",
      "Prediction:-0.0842239920152223\n",
      "Prediction:1.0762639960116431\n",
      "Prediction:0.9718545378681842\n",
      "Prediction:0.0009168131382832068\n",
      "Error:0.014420818295271236\n",
      "\n",
      "Prediction:0.0007334505106265654\n",
      "Prediction:0.9773186039296565\n",
      "Prediction:-0.07851033295953944\n",
      "Prediction:1.0710711494147542\n",
      "Prediction:0.9754916865567282\n",
      "Prediction:-0.0010574652271341245\n",
      "Error:0.012331739998443648\n",
      "\n",
      "Prediction:-0.0008459721817072885\n",
      "Prediction:0.9805836929862668\n",
      "Prediction:-0.07318360881847627\n",
      "Prediction:1.066233777045345\n",
      "Prediction:0.9785385598617921\n",
      "Prediction:-0.0025173975573930946\n",
      "Error:0.010587393171639842\n",
      "\n",
      "Prediction:-0.002013918045914484\n",
      "Prediction:0.9832839794497644\n",
      "Prediction:-0.06821774801198803\n",
      "Prediction:1.0617271739912904\n",
      "Prediction:0.9811035235627523\n",
      "Prediction:-0.0035735447350425317\n",
      "Error:0.009117233405426495\n",
      "\n",
      "Prediction:-0.002858835788034024\n",
      "Prediction:0.9855260569025094\n",
      "Prediction:-0.06358841060413677\n",
      "Prediction:1.05752842286588\n",
      "Prediction:0.9832740020092452\n",
      "Prediction:-0.004313918034364962\n",
      "Error:0.00786904226904208\n",
      "\n",
      "Prediction:-0.003451134427491974\n",
      "Prediction:0.9873957068535818\n",
      "Prediction:-0.059272877470408075\n",
      "Prediction:1.0536162524729626\n",
      "Prediction:0.9851206027353137\n",
      "Prediction:-0.004808501248434842\n",
      "Error:0.006803273214640502\n",
      "\n",
      "Prediction:-0.0038468009987478735\n",
      "Prediction:0.9889620124129692\n",
      "Prediction:-0.05524994626077355\n",
      "Prediction:1.049970908776931\n",
      "Prediction:0.9867004228010665\n",
      "Prediction:-0.005112871449710697\n",
      "Error:0.005889303541837786\n",
      "\n",
      "Prediction:-0.004090297159768559\n",
      "Prediction:0.9902806551018011\n",
      "Prediction:-0.051499833441728114\n",
      "Prediction:1.0465740376293469\n",
      "Prediction:0.9880596998997442\n",
      "Prediction:-0.0052710974096659285\n",
      "Error:0.0051029252561172675\n",
      "\n",
      "Prediction:-0.004216877927732746\n",
      "Prediction:0.9913965574535352\n",
      "Prediction:-0.048004082062078055\n",
      "Prediction:1.043408578143574\n",
      "Prediction:0.9892359385403211\n",
      "Prediction:-0.005318059364078823\n",
      "Error:0.004424644608684828\n",
      "\n",
      "Prediction:-0.0042544474912630525\n",
      "Prediction:0.992346001517791\n",
      "Prediction:-0.044745474990504665\n",
      "Prediction:1.0404586655589985\n",
      "Prediction:0.9902596156014837\n",
      "Prediction:-0.005281305317687134\n",
      "Error:0.0038385124412518303\n",
      "\n",
      "Prediction:-0.0042250442541497055\n",
      "Prediction:0.9931583274383705\n",
      "Prediction:-0.041707953394155776\n",
      "Prediction:1.0377095425371112\n",
      "Prediction:0.9911555487826897\n",
      "Prediction:-0.005182536193432452\n",
      "Error:0.0033313054558089675\n",
      "\n",
      "Prediction:-0.004146028954745959\n",
      "Prediction:0.9938572955409696\n",
      "Prediction:-0.03887654022599941\n",
      "Prediction:1.0351474779634813\n",
      "Prediction:0.9919439948626794\n",
      "Prediction:-0.00503879377425797\n",
      "Error:0.0028919416227737734\n",
      "\n",
      "Prediction:-0.004031035019406375\n",
      "Prediction:0.9944621787695098\n",
      "Prediction:-0.03623726848360008\n",
      "Prediction:1.032759692455092\n",
      "Prediction:0.9926415313729495\n",
      "Prediction:-0.004863410672429416\n",
      "Error:0.002511053608117256\n",
      "\n",
      "Prediction:-0.003890728537943533\n",
      "Prediction:0.9949886390193969\n",
      "Prediction:-0.03377711399894662\n",
      "Prediction:1.0305342898820642\n",
      "Prediction:0.9932617646389992\n",
      "Prediction:-0.004666769772712614\n",
      "Error:0.0021806703520253884\n",
      "\n",
      "Prediction:-0.003733415818170091\n",
      "Prediction:0.9954494302702878\n",
      "Prediction:-0.03148393251909879\n",
      "Prediction:1.0284601943056741\n",
      "Prediction:0.9938158986070053\n",
      "Prediction:-0.004456911151490314\n",
      "Error:0.0018939739123713475\n",
      "\n",
      "Prediction:-0.003565528921192253\n",
      "Prediction:0.9958549628928723\n",
      "Prediction:-0.029346400840475826\n",
      "Prediction:1.0265270918125804\n",
      "Prediction:0.9943131920358295\n",
      "Prediction:-0.004240016908292479\n",
      "Error:0.0016451096996342332\n",
      "\n",
      "Prediction:-0.0033920135266339822\n",
      "Prediction:0.9962137566721563\n",
      "Prediction:-0.02735396176499221\n",
      "Prediction:1.0247253767906936\n",
      "Prediction:0.9947613261560856\n",
      "Prediction:-0.004020798285770878\n",
      "Error:0.0014290353984827077\n",
      "\n",
      "Prediction:-0.003216638628616701\n",
      "Prediction:0.9965328046163073\n",
      "Prediction:-0.025496772653362886\n",
      "Prediction:1.0230461022472208\n",
      "Prediction:0.9951667005089379\n",
      "Prediction:-0.0038028045995257484\n",
      "Error:0.0012413985592149145\n",
      "\n",
      "Prediction:-0.003042243679620596\n",
      "Prediction:0.996817865235065\n",
      "Prediction:-0.023765657359234325\n",
      "Prediction:1.0214809338160067\n",
      "Prediction:0.995534671160774\n",
      "Prediction:-0.0035886696105582767\n",
      "Error:0.0010784359268087556\n",
      "\n",
      "Prediction:-0.0028709356884466207\n",
      "Prediction:0.9970736974585198\n",
      "Prediction:-0.022152061336940452\n",
      "Prediction:1.0200221071408409\n",
      "Prediction:0.9958697426723416\n",
      "Prediction:-0.0033803078583175654\n",
      "Error:0.0009368896209360312\n",
      "\n",
      "Prediction:-0.0027042462866540516\n",
      "Prediction:0.9973042495523706\n",
      "Prediction:-0.02064800972530455\n",
      "Prediction:1.018662388355171\n",
      "Prediction:0.9961757229433927\n",
      "Prediction:-0.0031790709774033414\n",
      "Error:0.0008139366504753339\n",
      "\n",
      "Prediction:-0.002543256781922673\n",
      "Prediction:0.9975128111306469\n",
      "Prediction:-0.019246068219762574\n",
      "Prediction:1.0173950374076535\n",
      "Prediction:0.9964558482449631\n",
      "Prediction:-0.0029858720226535913\n",
      "Error:0.0007071291752624441\n",
      "\n",
      "Prediction:-0.002388697618122871\n",
      "Prediction:0.9977021355600483\n",
      "Prediction:-0.01793930655497516\n",
      "Prediction:1.0162137740080082\n",
      "Prediction:0.9967128843019345\n",
      "Prediction:-0.0028012842268006904\n",
      "Error:0.0006143435674831474\n",
      "\n",
      "Prediction:-0.0022410273814405524\n",
      "Prediction:0.9978745386023716\n",
      "Prediction:-0.016721264429884947\n",
      "Prediction:1.0151127459893812\n",
      "Prediction:0.9969492081270097\n",
      "Prediction:-0.0026256193329783125\n",
      "Error:0.00053373677328488\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "weights = np.array([0.5,0.48,-0.7])\n",
    "alpha = 0.1\n",
    "\n",
    "streetlights = np.array( [[ 1, 0, 1 ],\n",
    "                          [ 0, 1, 1 ],\n",
    "                          [ 0, 0, 1 ],\n",
    "                          [ 1, 1, 1 ],\n",
    "                          [ 0, 1, 1 ],\n",
    "                          [ 1, 0, 1 ] ] )\n",
    "\n",
    "walk_vs_stop = np.array( [ 0, 1, 0, 1, 1, 0 ] )\n",
    "\n",
    "input = streetlights[0] # [1,0,1]\n",
    "goal_prediction = walk_vs_stop[0] # equals 0... i.e. \"stop\"\n",
    "\n",
    "for iteration in range(40):\n",
    "    error_for_all_lights = 0\n",
    "    for row_index in range(len(walk_vs_stop)):\n",
    "        input = streetlights[row_index]\n",
    "        goal_prediction = walk_vs_stop[row_index]\n",
    "        \n",
    "        prediction = input.dot(weights)\n",
    "        \n",
    "        error = (goal_prediction - prediction) ** 2\n",
    "        error_for_all_lights += error\n",
    "        \n",
    "        delta = prediction - goal_prediction\n",
    "        weights = weights - (alpha * (input * delta))\t\n",
    "        print(\"Prediction:\" + str(prediction))\n",
    "    print(\"Error:\" + str(error_for_all_lights) + \"\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Our First \"Deep\" Neural Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "np.random.seed(1)\n",
    "\n",
    "def relu(x):\n",
    "    return (x > 0) * x \n",
    "\n",
    "alpha = 0.2\n",
    "hidden_size = 4\n",
    "\n",
    "streetlights = np.array( [[ 1, 0, 1 ],\n",
    "                          [ 0, 1, 1 ],\n",
    "                          [ 0, 0, 1 ],\n",
    "                          [ 1, 1, 1 ] ] )\n",
    "\n",
    "walk_vs_stop = np.array([[ 1, 1, 0, 0]]).T\n",
    "\n",
    "weights_0_1 = 2*np.random.random((3,hidden_size)) - 1\n",
    "weights_1_2 = 2*np.random.random((hidden_size,1)) - 1\n",
    "\n",
    "layer_0 = streetlights[0]\n",
    "layer_1 = relu(np.dot(layer_0,weights_0_1))\n",
    "layer_2 = np.dot(layer_1,weights_1_2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Backpropagation in Code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error:0.6342311598444467\n",
      "Error:0.35838407676317513\n",
      "Error:0.0830183113303298\n",
      "Error:0.006467054957103705\n",
      "Error:0.0003292669000750734\n",
      "Error:1.5055622665134859e-05\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "np.random.seed(1)\n",
    "\n",
    "def relu(x):\n",
    "    return (x > 0) * x # returns x if x > 0\n",
    "                       # return 0 otherwise\n",
    "\n",
    "def relu2deriv(output):\n",
    "    return output>0 # returns 1 for input > 0\n",
    "                    # return 0 otherwise\n",
    "alpha = 0.2\n",
    "hidden_size = 4\n",
    "\n",
    "weights_0_1 = 2*np.random.random((3,hidden_size)) - 1\n",
    "weights_1_2 = 2*np.random.random((hidden_size,1)) - 1\n",
    "\n",
    "for iteration in range(60):\n",
    "   layer_2_error = 0\n",
    "   for i in range(len(streetlights)):\n",
    "      layer_0 = streetlights[i:i+1]\n",
    "      layer_1 = relu(np.dot(layer_0,weights_0_1))\n",
    "      layer_2 = np.dot(layer_1,weights_1_2)\n",
    "\n",
    "      layer_2_error += np.sum((layer_2 - walk_vs_stop[i:i+1]) ** 2)\n",
    "\n",
    "      layer_2_delta = (walk_vs_stop[i:i+1] - layer_2)\n",
    "      layer_1_delta=layer_2_delta.dot(weights_1_2.T)*relu2deriv(layer_1)\n",
    "\n",
    "      weights_1_2 += alpha * layer_1.T.dot(layer_2_delta)\n",
    "      weights_0_1 += alpha * layer_0.T.dot(layer_1_delta)\n",
    "\n",
    "   if(iteration % 10 == 9):\n",
    "      print(\"Error:\" + str(layer_2_error))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# One Iteration of Backpropagation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "np.random.seed(1)\n",
    "\n",
    "def relu(x):\n",
    "    return (x > 0) * x \n",
    "\n",
    "def relu2deriv(output):\n",
    "    return output>0 \n",
    "\n",
    "lights = np.array( [[ 1, 0, 1 ],\n",
    "                    [ 0, 1, 1 ],\n",
    "                    [ 0, 0, 1 ],\n",
    "                    [ 1, 1, 1 ] ] )\n",
    "\n",
    "walk_stop = np.array([[ 1, 1, 0, 0]]).T\n",
    "\n",
    "alpha = 0.2\n",
    "hidden_size = 3\n",
    "\n",
    "weights_0_1 = 2*np.random.random((3,hidden_size)) - 1\n",
    "weights_1_2 = 2*np.random.random((hidden_size,1)) - 1\n",
    "\n",
    "layer_0 = lights[0:1]\n",
    "layer_1 = np.dot(layer_0,weights_0_1)\n",
    "layer_1 = relu(layer_1)\n",
    "layer_2 = np.dot(layer_1,weights_1_2)\n",
    "\n",
    "error = (layer_2-walk_stop[0:1])**2\n",
    "\n",
    "layer_2_delta=(layer_2-walk_stop[0:1])\n",
    "\n",
    "layer_1_delta=layer_2_delta.dot(weights_1_2.T)\n",
    "layer_1_delta *= relu2deriv(layer_1)\n",
    "\n",
    "weight_delta_1_2 = layer_1.T.dot(layer_2_delta)\n",
    "weight_delta_0_1 = layer_0.T.dot(layer_1_delta)\n",
    "\n",
    "weights_1_2 -= alpha * weight_delta_1_2\n",
    "weights_0_1 -= alpha * weight_delta_0_1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Putting it all Together"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error:0.6342311598444467\n",
      "Error:0.35838407676317513\n",
      "Error:0.0830183113303298\n",
      "Error:0.006467054957103705\n",
      "Error:0.0003292669000750734\n",
      "Error:1.5055622665134859e-05\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "np.random.seed(1)\n",
    "\n",
    "def relu(x):\n",
    "    return (x > 0) * x # returns x if x > 0\n",
    "                       # return 0 otherwise\n",
    "\n",
    "def relu2deriv(output):\n",
    "    return output>0 # returns 1 for input > 0\n",
    "                    # return 0 otherwise\n",
    "\n",
    "streetlights = np.array( [[ 1, 0, 1 ],\n",
    "                          [ 0, 1, 1 ],\n",
    "                          [ 0, 0, 1 ],\n",
    "                          [ 1, 1, 1 ] ] )\n",
    "\n",
    "walk_vs_stop = np.array([[ 1, 1, 0, 0]]).T\n",
    "    \n",
    "alpha = 0.2\n",
    "hidden_size = 4\n",
    "\n",
    "weights_0_1 = 2*np.random.random((3,hidden_size)) - 1\n",
    "weights_1_2 = 2*np.random.random((hidden_size,1)) - 1\n",
    "\n",
    "for iteration in range(60):\n",
    "   layer_2_error = 0\n",
    "   for i in range(len(streetlights)):\n",
    "      layer_0 = streetlights[i:i+1]\n",
    "      layer_1 = relu(np.dot(layer_0,weights_0_1))\n",
    "      layer_2 = np.dot(layer_1,weights_1_2)\n",
    "\n",
    "      layer_2_error += np.sum((layer_2 - walk_vs_stop[i:i+1]) ** 2)\n",
    "\n",
    "      layer_2_delta = (layer_2 - walk_vs_stop[i:i+1])\n",
    "      layer_1_delta=layer_2_delta.dot(weights_1_2.T)*relu2deriv(layer_1)\n",
    "\n",
    "      weights_1_2 -= alpha * layer_1.T.dot(layer_2_delta)\n",
    "      weights_0_1 -= alpha * layer_0.T.dot(layer_1_delta)\n",
    "\n",
    "   if(iteration % 10 == 9):\n",
    "      print(\"Error:\" + str(layer_2_error))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}