Batch Gradient Descent

Chapter 8/4. Batch Gradient Descent.ipynb

@@ -0,0 +1,114 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Batch Gradient Descent\n",
+    "\n",
+    "Now we are going to train 100 training examples at a time averaging the weight updates between all 100 examples."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "import numpy as np\n",
+    "np.random.seed(1)\n",
+    "import sys"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "def relu(x):\n",
+    "    return (x>0)*x\n",
+    "\n",
+    "def relu2deriv(output):\n",
+    "    return (output>0)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "# Input data\n",
+    "\n",
+    "alpha,iterations=(0.1,100)\n",
+    "pixels_per_image,num_labels,hidden_size=(784,10,100)\n",
+    "\n",
+    "weights_0_1=0.2*np.random.random((pixels_per_image,hidden_size))-0.1\n",
+    "weights_1_2=0.2*np.random.random((hidden_size,num_labels))-0.1"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "for j in xrange(iteratins):\n",
+    "    error=0\n",
+    "    correct_cnt=0\n",
+    "    for i in xrange(len(images)/batch_size):\n",
+    "        batch_start,batch_end=((i*batch_size),((i+1)*batch_size))\n",
+    "        layer_0=images[batch_start:batch_end]\n",
+    "        layer_1=relu(np.dot(layer_0,weights_0_1))\n",
+    "        dropout_mask=np.random.randint(2,size=layer_1.shape)\n",
+    "        layer_1*=dropout_mask\n",
+    "        layer_2=np.dot(layer_1,weights_1_2)\n",
+    "        \n",
+    "        error+=np.sum((labels[batch_start:batch_end]-layer_2)**2)\n",
+    "        \n",
+    "        for k in xrange(batch_size):\n",
+    "            correct_cnt+=int(np.argmax(layer_2[k:k+1])==np.argmax(labels[batch_start+k:batch_start+k+1]))\n",
+    "        \n",
+    "        delta_layer_2=(layer_2-labels[batch_start:batch_end])/batch_size\n",
+    "        delta_layer_1=delta_layer_2.dot(weights_1_2.T)*relu2deriv(layer_1)\n",
+    "        \n",
+    "        delta_layer_1*=dropout_mask\n",
+    "        \n",
+    "        weights_1_2-=alpha*layer_1.T*dot(delta_layer_2)\n",
+    "        weights_0_1-=alpha*layer_0.T*dot(delta_layer_1)\n",
+    "        \n",
+    "\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python [conda root]",
+   "language": "python",
+   "name": "conda-root-py"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.13"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}