Code updated with drop out logic

Code updated with drop put logic

Chapter 8/3. Drop out.ipynb

@@ -0,0 +1,131 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "import numpy as np\n",
+    "np.random.seed(1)\n",
+    "import sys"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def relu(x):\n",
+    "    return (x>0)*x\n",
+    "\n",
+    "def relu2deriv(output):\n",
+    "    return output>0\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "(784, 100)\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Input data\n",
+    "alpha,iterations,hidden_size=(0.005,300,100)\n",
+    "pixels_per_image,num_labels=(784,10)\n",
+    "\n",
+    "weights_0_1=0.2*np.random.random((pixels_per_image,hidden_size))-0.1\n",
+    "weights_1_2=0.2*np.random.random((hidden_size,num_labels))-0.1\n",
+    "print weights_0_1.shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "# Train the model\n",
+    "\n",
+    "for j in xrange(iterations):\n",
+    "    error=0.0\n",
+    "    correct_cnt=0\n",
+    "    for i in xrange(len(images)):\n",
+    "        layer_0=images[i:i+1]\n",
+    "        layer_1=relu(layer_0.dot(weights_0_1))\n",
+    "        dropout_mask=np.random.randint(2,size=layer_1.shape)\n",
+    "        layer_1*=dropout_mask*2\n",
+    "        layer_2=np.dot(layer_1,weights_1_2)\n",
+    "        error+=np.sum((layer_2-labels[i:i+1])**2)\n",
+    "        correct_cnt+=int(np.argmax(layer_2)==np.argmax(labels[i:i+1]))\n",
+    "        \n",
+    "        delta_layer_2=layer_2-labels[i:i+1]\n",
+    "        delta_layer_1=delta_layer_2.dot(weights_1_2.T)*relu2deriv(layer_1)\n",
+    "        delta_layer_1*=dropout_mask\n",
+    "        \n",
+    "        weights_1_2-=alpha*layer_1.T.dot(delta_layer_2)\n",
+    "        weights_0_1-=alpha*layer_0.T.dot(delta_layer_1)\n",
+    "        \n",
+    "        \n",
+    "    for (j%10==0):\n",
+    "        test_error=0\n",
+    "        test_correct_cnt=0\n",
+    "        for i in xrange(len(test_images)):\n",
+    "            layer_0=test_images[i:i+1]\n",
+    "            layer_1=relu(layer_0.dot(weights_0_1))\n",
+    "            layer_2=layer_1.dot(weights_1_2)\n",
+    "            \n",
+    "            test_error+=np.sum((layer_2-test_images[i:i+1])**2)\n",
+    "            test_correct_cnt=int(np.argmax(layer_2)==np.argmax(test_images[i:i+1]))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    " sys.stdout.write('\\n'+\\\n",
+    "                 'Iteration:'+str(j)+\\\n",
+    "                 'Test_Error:'+str(test_error/float(len(test_images)))[0:5]+\\\n",
+    "                 'Train_Error:'+str(error/float9(len(images)))[0:5]+\\\n",
+    "                 'Train_Acc:'+str(correct_cnt/float(len(images))))"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python [conda root]",
+   "language": "python",
+   "name": "conda-root-py"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.13"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}