Adding TCN Layers + Tutorial to TFA Layers

docs/tutorials/temporal_convolutional_network.ipynb

@@ -0,0 +1,310 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "##### Copyright 2019 The TensorFlow Authors."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+    "# you may not use this file except in compliance with the License.\n",
+    "# You may obtain a copy of the License at\n",
+    "#\n",
+    "# https://www.apache.org/licenses/LICENSE-2.0\n",
+    "#\n",
+    "# Unless required by applicable law or agreed to in writing, software\n",
+    "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+    "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+    "# See the License for the specific language governing permissions and\n",
+    "# limitations under the License."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# TensorFlow Addons Layers: Temporal Convolutional Network"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "<table class=\"tfo-notebook-buttons\" align=\"left\">\n",
+    "  <td>\n",
+    "    <a target=\"_blank\" href=\"https://www.tensorflow.org/addons/tutorials/temporal_convolutional_network\"><img src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" />View on TensorFlow.org</a>\n",
+    "  </td>\n",
+    "  <td>\n",
+    "    <a target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/addons/blob/master/docs/tutorials/temporal_convolutional_network.ipynb\"><img src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" />Run in Google Colab</a>\n",
+    "  </td>\n",
+    "  <td>\n",
+    "    <a target=\"_blank\" href=\"https://github.com/tensorflow/addons/blob/master/docs/tutorials/temporal_convolutional_network.ipynb\"><img src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" />View source on GitHub</a>\n",
+    "  </td>\n",
+    "      <td>\n",
+    "    <a href=\"https://storage.googleapis.com/tensorflow_docs/addons/docs/tutorials/temporal_convolutional_network.ipynb\"><img src=\"https://www.tensorflow.org/images/download_logo_32px.png\" />Download notebook</a>\n",
+    "  </td>\n",
+    "</table>"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Overview\n",
+    "This notebook will demonstrate how to use Temporal Convolutional Network in TensorFlow Addons."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Setup"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from __future__ import absolute_import, division, print_function"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "try:\n",
+    "    # %tensorflow_version only exists in Colab.\n",
+    "    %tensorflow_version 2.x\n",
+    "except Exception:\n",
+    "    pass"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import numpy as np\n",
+    "import tensorflow as tf\n",
+    "import tensorflow_addons as tfa\n",
+    "\n",
+    "import tensorflow.keras as keras\n",
+    "from tensorflow.keras.datasets import imdb\n",
+    "from tensorflow.keras.models import Model\n",
+    "from tensorflow.keras import Input\n",
+    "from tensorflow.keras.layers import Dense, Dropout, Embedding\n",
+    "from tensorflow.keras.preprocessing import sequence"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Global Configurations"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "max_features = 20000\n",
+    "# cut texts after this number of words\n",
+    "# (among top max_features most common words)\n",
+    "maxlen = 100\n",
+    "batch_size = 32"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Import IMDB data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# the data, split between train and test sets\n",
+    "(x_train, y_train), (x_test, y_test) = imdb.load_data(num_words=max_features)\n",
+    "\n",
+    "# pad the training data\n",
+    "x_train = sequence.pad_sequences(x_train, maxlen=maxlen)\n",
+    "x_test = sequence.pad_sequences(x_test, maxlen=maxlen)\n",
+    "y_train = np.array(y_train)\n",
+    "y_test = np.array(y_test)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Build Simple TCN Model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# build the model using functional API\n",
+    "i = Input(shape=(maxlen,))\n",
+    "x = Embedding(max_features, 128)(i)\n",
+    "x = tfa.layers.TCN()(x)\n",
+    "x = Dropout(0.2)(x)\n",
+    "x = Dense(1, activation='sigmoid')(x)\n",
+    "\n",
+    "model = Model(inputs=[i], outputs=[x])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Summary of the model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Model: \"model\"\n",
+      "_________________________________________________________________\n",
+      "Layer (type)                 Output Shape              Param #   \n",
+      "=================================================================\n",
+      "input_1 (InputLayer)         [(None, 100)]             0         \n",
+      "_________________________________________________________________\n",
+      "embedding (Embedding)        (None, 100, 128)          2560000   \n",
+      "_________________________________________________________________\n",
+      "tcn (TCN)                    (None, 64)                148800    \n",
+      "_________________________________________________________________\n",
+      "dropout (Dropout)            (None, 64)                0         \n",
+      "_________________________________________________________________\n",
+      "dense (Dense)                (None, 1)                 65        \n",
+      "=================================================================\n",
+      "Total params: 2,708,865\n",
+      "Trainable params: 2,708,865\n",
+      "Non-trainable params: 0\n",
+      "_________________________________________________________________\n"
+     ]
+    }
+   ],
+   "source": [
+    "model.summary()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Compile the model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "model.compile(optimizer='adam',\n",
+    "              loss = 'binary_crossentropy',\n",
+    "              metrics=['accuracy'])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Fit the model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Train on 25000 samples, validate on 25000 samples\n",
+      "Epoch 1/3\n",
+      "25000/25000 [==============================] - 100s 4ms/sample - loss: 0.5734 - accuracy: 0.6722 - val_loss: 0.4147 - val_accuracy: 0.8041\n",
+      "Epoch 2/3\n",
+      "25000/25000 [==============================] - 95s 4ms/sample - loss: 0.2852 - accuracy: 0.8811 - val_loss: 0.4136 - val_accuracy: 0.8128\n",
+      "Epoch 3/3\n",
+      "25000/25000 [==============================] - 97s 4ms/sample - loss: 0.1372 - accuracy: 0.9485 - val_loss: 0.4949 - val_accuracy: 0.8190\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "<tensorflow.python.keras.callbacks.History at 0x109a5b4e0>"
+      ]
+     },
+     "execution_count": 11,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "model.fit(x_train, y_train,\n",
+    "          batch_size=batch_size,\n",
+    "          epochs=3,\n",
+    "          validation_data=(x_test, y_test))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

tensorflow_addons/layers/BUILD

@@ -12,6 +12,7 @@ py_library(
         "optical_flow.py",
         "poincare.py",
         "sparsemax.py",
+        "tcn.py",
         "wrappers.py",
     ],
     data = [
@@ -101,3 +102,16 @@ py_test(
         ":layers",
     ],
 )
+
+py_test(
+    name = "tcn_test",
+    size = "small",
+    srcs = [
+        "tcn_test.py",
+    ],
+    main = "tcn_test.py",
+    srcs_version = "PY2AND3",
+    deps = [
+        ":layers",
+    ],
+)

tensorflow_addons/layers/README.md

@@ -9,6 +9,7 @@
 | opticalflow | @fsx950223 | [email protected] |
 | poincare | @rahulunair | [email protected]  |
 | sparsemax | @AndreasMadsen | [email protected] |
+| tcn | @shun-lin | [email protected] |
 | wrappers | @seanpmorgan | [email protected] |
 
 ## Components
@@ -20,7 +21,8 @@
 | normalizations | InstanceNormalization | https://arxiv.org/abs/1607.08022 |
 | opticalflow | CorrelationCost | https://arxiv.org/abs/1504.06852   |
 | poincare | PoincareNormalize | https://arxiv.org/abs/1705.08039    |
-| sparsemax| Sparsemax | https://arxiv.org/abs/1602.02068 |
+| sparsemax | Sparsemax | https://arxiv.org/abs/1602.02068 |
+| tcn | TCN (Temporal Convolutional Network) | https://arxiv.org/pdf/1803.01271 |
 | wrappers | WeightNormalization | https://arxiv.org/abs/1602.07868 |
 
 ## Contribution Guidelines

tensorflow_addons/layers/__init__.py

@@ -14,15 +14,14 @@
 # ==============================================================================
 """Additional layers that conform to Keras API."""
 
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
 
 from tensorflow_addons.layers.gelu import GeLU
 from tensorflow_addons.layers.maxout import Maxout
-from tensorflow_addons.layers.normalizations import GroupNormalization
-from tensorflow_addons.layers.normalizations import InstanceNormalization
+from tensorflow_addons.layers.normalizations import (GroupNormalization,
+                                                     InstanceNormalization)
 from tensorflow_addons.layers.optical_flow import CorrelationCost
 from tensorflow_addons.layers.poincare import PoincareNormalize
 from tensorflow_addons.layers.sparsemax import Sparsemax
-from tensorflow_addons.layers.wrappers import WeightNormalization
+from tensorflow_addons.layers.tcn import TCN
+from tensorflow_addons.layers.wrappers import WeightNormalization