From c5ab7193ac2df867747df001e075e25d8200dcc7 Mon Sep 17 00:00:00 2001 From: Kerem Turgutlu Date: Fri, 23 Apr 2021 23:46:15 -0700 Subject: [PATCH 1/3] speed up act calc, add notes and slice for mutual information calc --- ...litNN Information Reduction Exercise.ipynb | 182 ++++++++++++++---- 1 file changed, 147 insertions(+), 35 deletions(-) diff --git a/split-nn/concepts-definitions-code/SplitNN Information Reduction Exercise.ipynb b/split-nn/concepts-definitions-code/SplitNN Information Reduction Exercise.ipynb index a8e01a2..814639b 100644 --- a/split-nn/concepts-definitions-code/SplitNN Information Reduction Exercise.ipynb +++ b/split-nn/concepts-definitions-code/SplitNN Information Reduction Exercise.ipynb @@ -18,7 +18,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -41,20 +41,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "BATCH_SIZE = 256\n", "BATCH_TEST_SIZE = 1024\n", - "data_train = MNIST('./data/mnist',\n", - " download=True,\n", + "data_train = MNIST('/Users/turgutlu/.syft/data/',\n", + " download=False,\n", " transform=transforms.Compose([\n", " transforms.Resize((32, 32)),\n", " transforms.ToTensor()]))\n", - "data_test = MNIST('./data/mnist',\n", + "data_test = MNIST('/Users/turgutlu/.syft/data/',\n", " train=False,\n", - " download=True,\n", + " download=False,\n", " transform=transforms.Compose([\n", " transforms.Resize((32, 32)),\n", " transforms.ToTensor()]))\n", @@ -77,7 +77,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -95,7 +95,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -124,13 +124,58 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Test Avg. Loss: 0.000023, Accuracy: 0.992900\n" + ] + } + ], "source": [ "validate (model_loaded, criterion)" ] }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "LeNet5_5(\n", + " (convnet): Sequential(\n", + " (c1): Conv2d(1, 6, kernel_size=(5, 5), stride=(1, 1))\n", + " (relu1): ReLU()\n", + " (s2): MaxPool2d(kernel_size=(2, 2), stride=2, padding=0, dilation=1, ceil_mode=False)\n", + " (c3): Conv2d(6, 16, kernel_size=(5, 5), stride=(1, 1))\n", + " (relu3): ReLU()\n", + " (s4): MaxPool2d(kernel_size=(2, 2), stride=2, padding=0, dilation=1, ceil_mode=False)\n", + " (c5): Conv2d(16, 120, kernel_size=(5, 5), stride=(1, 1))\n", + " (relu5): ReLU()\n", + " )\n", + " (fc): Sequential(\n", + " (f6): Linear(in_features=120, out_features=84, bias=True)\n", + " (relu6): ReLU()\n", + " (f7): Linear(in_features=84, out_features=2, bias=True)\n", + " (sig7): LogSoftmax(dim=-1)\n", + " )\n", + ")" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model_loaded" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -149,26 +194,55 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": { - "tags": [] - }, + "execution_count": 7, + "metadata": {}, "outputs": [], + "source": [ + "from tqdm.notebook import tqdm" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "45b025ea274146e7963ba0e8fad2ccc4", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=10000.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], "source": [ "imgs = []\n", "intermediate_activations = []\n", "total_correct = 0\n", "\n", "model_loaded.eval()\n", - "for i, (images, labels) in enumerate(data_test_loader2):\n", - " imgs.append(((np.reshape(np.squeeze(images.detach().numpy()), (1,-1)) )))\n", - " x = images\n", - " x = model_loaded.convnet(x) \n", - " \n", - " intermediate_activations.append(((np.reshape(np.squeeze(x.detach().numpy()), (1,-1)) )))\n", - " \n", - " np.save(\"images\", np.array(imgs).squeeze(1))\n", - " np.save(\"intermediate_act\", np.array(intermediate_activations).squeeze(1))" + "with torch.no_grad():\n", + " for i, (images, labels) in tqdm(enumerate(data_test_loader2), total=len(data_test_loader2)):\n", + " imgs.append(images.squeeze(0).view(1,-1))\n", + "\n", + " x = model_loaded.convnet(images)\n", + " intermediate_activations.append(x.view(1,120))\n", + "\n", + " np.save(\"images\", torch.cat(imgs).numpy())\n", + " np.save(\"intermediate_act\", torch.cat(intermediate_activations).numpy())" ] }, { @@ -182,7 +256,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ @@ -200,9 +274,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(10000, 1024)\n", + "(10000, 120)\n" + ] + } + ], "source": [ "images_raw=np.load(\"images.npy\")\n", "print(images_raw.shape)\n", @@ -217,9 +300,20 @@ "#### Mutual Information function" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We will calculate self-information and mutual information of the first 1,000 test images. You can remove the slicing to calculate on full 10,000 test images but it will take longer to run.\n", + "\n", + "$$\n", + "I(X;Y) = \\sum\\limits_{(x,y) \\in \\mathcal{X}\\times\\mathcal{Y}} P_{XY}(x,y) \\log \\frac{P_{XY}(x,y)}{P_X(x)P_Y(y)}\n", + "$$" + ] + }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "metadata": {}, "outputs": [], "source": [ @@ -235,12 +329,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(1000, 2048)\n", + "455.97089317698345\n" + ] + } + ], "source": [ "ds = np.array([1024, 1024])\n", - "y = np.concatenate((images_raw, images_raw),axis=1)\n", + "y = np.concatenate((images_raw[:1000], images_raw[:1000]),axis=1)\n", "print(y.shape)\n", "i = co.estimation(y, ds) \n", "print(i)" @@ -255,12 +358,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(1000, 1144)\n", + "163.148586532957\n" + ] + } + ], "source": [ "ds = np.array([1024, 120])\n", - "y = np.concatenate((images_raw, intermediate_activation),axis=1)\n", + "y = np.concatenate((images_raw[:1000], intermediate_activation[:1000]),axis=1)\n", "print(y.shape)\n", "i = co.estimation(y, ds) \n", "print(i)" @@ -270,7 +382,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "We can see that the raw image contained 730 bits of self-information, whereas the intermediate activations only contain 303 bits of information that was originally in the raw image (the 730 bits). This shows that the first layers of the neural network, alone, have degraded more than half of the original information in the raw input. " + "We can see that the raw image contained 455 bits of self-information, whereas the intermediate activations only contain 163 bits of information that was originally in the raw image (the 455 bits). This shows that the first layers of the neural network, alone, have degraded more than half of the original information in the raw input. " ] } ], @@ -290,7 +402,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.12" + "version": "3.7.10" } }, "nbformat": 4, From 40ad96c22664dbaea3082caefea2022ce80544bc Mon Sep 17 00:00:00 2001 From: Kerem Turgutlu Date: Sat, 24 Apr 2021 00:00:33 -0700 Subject: [PATCH 2/3] revert dataset setup and clean nb --- ...litNN Information Reduction Exercise.ipynb | 143 +++--------------- 1 file changed, 23 insertions(+), 120 deletions(-) diff --git a/split-nn/concepts-definitions-code/SplitNN Information Reduction Exercise.ipynb b/split-nn/concepts-definitions-code/SplitNN Information Reduction Exercise.ipynb index 814639b..053c49b 100644 --- a/split-nn/concepts-definitions-code/SplitNN Information Reduction Exercise.ipynb +++ b/split-nn/concepts-definitions-code/SplitNN Information Reduction Exercise.ipynb @@ -18,7 +18,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -41,20 +41,20 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ "BATCH_SIZE = 256\n", "BATCH_TEST_SIZE = 1024\n", - "data_train = MNIST('/Users/turgutlu/.syft/data/',\n", - " download=False,\n", + "data_train = MNIST('.data/mnist',\n", + " download=True,\n", " transform=transforms.Compose([\n", " transforms.Resize((32, 32)),\n", " transforms.ToTensor()]))\n", - "data_test = MNIST('/Users/turgutlu/.syft/data/',\n", + "data_test = MNIST('.data/mnist',\n", " train=False,\n", - " download=False,\n", + " download=True,\n", " transform=transforms.Compose([\n", " transforms.Resize((32, 32)),\n", " transforms.ToTensor()]))\n", @@ -77,7 +77,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -95,7 +95,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -124,54 +124,18 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Test Avg. Loss: 0.000023, Accuracy: 0.992900\n" - ] - } - ], + "outputs": [], "source": [ "validate (model_loaded, criterion)" ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "LeNet5_5(\n", - " (convnet): Sequential(\n", - " (c1): Conv2d(1, 6, kernel_size=(5, 5), stride=(1, 1))\n", - " (relu1): ReLU()\n", - " (s2): MaxPool2d(kernel_size=(2, 2), stride=2, padding=0, dilation=1, ceil_mode=False)\n", - " (c3): Conv2d(6, 16, kernel_size=(5, 5), stride=(1, 1))\n", - " (relu3): ReLU()\n", - " (s4): MaxPool2d(kernel_size=(2, 2), stride=2, padding=0, dilation=1, ceil_mode=False)\n", - " (c5): Conv2d(16, 120, kernel_size=(5, 5), stride=(1, 1))\n", - " (relu5): ReLU()\n", - " )\n", - " (fc): Sequential(\n", - " (f6): Linear(in_features=120, out_features=84, bias=True)\n", - " (relu6): ReLU()\n", - " (f7): Linear(in_features=84, out_features=2, bias=True)\n", - " (sig7): LogSoftmax(dim=-1)\n", - " )\n", - ")" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "model_loaded" ] @@ -194,7 +158,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -203,31 +167,9 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "45b025ea274146e7963ba0e8fad2ccc4", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=10000.0), HTML(value='')))" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - } - ], + "outputs": [], "source": [ "imgs = []\n", "intermediate_activations = []\n", @@ -256,7 +198,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -274,18 +216,9 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(10000, 1024)\n", - "(10000, 120)\n" - ] - } - ], + "outputs": [], "source": [ "images_raw=np.load(\"images.npy\")\n", "print(images_raw.shape)\n", @@ -313,7 +246,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -329,18 +262,9 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(1000, 2048)\n", - "455.97089317698345\n" - ] - } - ], + "outputs": [], "source": [ "ds = np.array([1024, 1024])\n", "y = np.concatenate((images_raw[:1000], images_raw[:1000]),axis=1)\n", @@ -358,18 +282,9 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(1000, 1144)\n", - "163.148586532957\n" - ] - } - ], + "outputs": [], "source": [ "ds = np.array([1024, 120])\n", "y = np.concatenate((images_raw[:1000], intermediate_activation[:1000]),axis=1)\n", @@ -391,18 +306,6 @@ "display_name": "Python 3", "language": "python", "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.10" } }, "nbformat": 4, From 31a35b3cc64d3b73df7769eb9157647f3009dbfc Mon Sep 17 00:00:00 2001 From: Kerem Turgutlu Date: Sat, 24 Apr 2021 00:02:21 -0700 Subject: [PATCH 3/3] revert data path --- .../SplitNN Information Reduction Exercise.ipynb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/split-nn/concepts-definitions-code/SplitNN Information Reduction Exercise.ipynb b/split-nn/concepts-definitions-code/SplitNN Information Reduction Exercise.ipynb index 053c49b..2fe06be 100644 --- a/split-nn/concepts-definitions-code/SplitNN Information Reduction Exercise.ipynb +++ b/split-nn/concepts-definitions-code/SplitNN Information Reduction Exercise.ipynb @@ -47,12 +47,12 @@ "source": [ "BATCH_SIZE = 256\n", "BATCH_TEST_SIZE = 1024\n", - "data_train = MNIST('.data/mnist',\n", + "data_train = MNIST('./data/mnist',\n", " download=True,\n", " transform=transforms.Compose([\n", " transforms.Resize((32, 32)),\n", " transforms.ToTensor()]))\n", - "data_test = MNIST('.data/mnist',\n", + "data_test = MNIST('./data/mnist',\n", " train=False,\n", " download=True,\n", " transform=transforms.Compose([\n", @@ -310,4 +310,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file