From 4bb44dfcfbb63e9b6ca583b3a61d221e61c5f8b9 Mon Sep 17 00:00:00 2001
From: Shun Lin <shunlin@google.com>
Date: Thu, 7 Nov 2019 16:55:27 -0800
Subject: [PATCH 01/20] Adding files, modify README, BUILD, __init__

---
 tensorflow_addons/layers/BUILD       |  14 ++
 tensorflow_addons/layers/README.md   |   2 +
 tensorflow_addons/layers/__init__.py |   3 +-
 tensorflow_addons/layers/tcn.py      | 311 +++++++++++++++++++++++++++
 tensorflow_addons/layers/tcn_test.py |  31 +++
 5 files changed, 360 insertions(+), 1 deletion(-)
 create mode 100644 tensorflow_addons/layers/tcn.py
 create mode 100644 tensorflow_addons/layers/tcn_test.py

diff --git a/tensorflow_addons/layers/BUILD b/tensorflow_addons/layers/BUILD
index 01f475a2fb..ac61cc1ef5 100644
--- a/tensorflow_addons/layers/BUILD
+++ b/tensorflow_addons/layers/BUILD
@@ -13,6 +13,7 @@ py_library(
         "poincare.py",
         "sparsemax.py",
         "wrappers.py",
+        "tcn.py",
     ],
     data = [
         "//tensorflow_addons/custom_ops/layers:_correlation_cost_ops.so",
@@ -101,3 +102,16 @@ py_test(
         ":layers",
     ],
 )
+
+py_test(
+    name = "tcn_test",
+    size = "small",
+    srcs = [
+        "tcn_test.py",
+    ],
+    main = "tcn_test.py",
+    srcs_version = "PY2AND3",
+    deps = [
+        ":layers",
+    ],
+)
diff --git a/tensorflow_addons/layers/README.md b/tensorflow_addons/layers/README.md
index 087fcfebbd..c582fcea9b 100644
--- a/tensorflow_addons/layers/README.md
+++ b/tensorflow_addons/layers/README.md
@@ -10,6 +10,7 @@
 | poincare | @rahulunair | rahulunair@gmail.com  |
 | sparsemax | @AndreasMadsen | amwwebdk+github@gmail.com |
 | wrappers | @seanpmorgan | seanmorgan@outlook.com |
+| tcn | @shun-lin | shunlin@google.com |
 
 ## Components
 | Submodule  | Layer |  Reference  |
@@ -22,6 +23,7 @@
 | poincare | PoincareNormalize | https://arxiv.org/abs/1705.08039    |
 | sparsemax| Sparsemax | https://arxiv.org/abs/1602.02068 |
 | wrappers | WeightNormalization | https://arxiv.org/abs/1602.07868 |
+| tcn | TCN (Temporal Convolutional Network) | https://arxiv.org/pdf/1803.01271 |
 
 ## Contribution Guidelines
 #### Standard API
diff --git a/tensorflow_addons/layers/__init__.py b/tensorflow_addons/layers/__init__.py
index d527e16362..b1fdc8cdb4 100644
--- a/tensorflow_addons/layers/__init__.py
+++ b/tensorflow_addons/layers/__init__.py
@@ -25,4 +25,5 @@
 from tensorflow_addons.layers.optical_flow import CorrelationCost
 from tensorflow_addons.layers.poincare import PoincareNormalize
 from tensorflow_addons.layers.sparsemax import Sparsemax
-from tensorflow_addons.layers.wrappers import WeightNormalization
\ No newline at end of file
+from tensorflow_addons.layers.wrappers import WeightNormalization
+from tensorflow_addons.layers.tcn import TCN
diff --git a/tensorflow_addons/layers/tcn.py b/tensorflow_addons/layers/tcn.py
new file mode 100644
index 0000000000..aa460fcd82
--- /dev/null
+++ b/tensorflow_addons/layers/tcn.py
@@ -0,0 +1,311 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Implements TCN layer."""
+
+from __future__ import absolute_import, division, print_function
+
+import tensorflow as tf
+import tensorflow.keras.backend as K
+from tensorflow.keras.layers import (Activation, BatchNormalization, Conv1D,
+                                     Dense, Lambda, SpatialDropout1D, add)
+from tensorflow_addons.utils import keras_utils
+
+
+@tf.keras.utils.register_keras_serializable(package='Addons')
+class ResidualBlock(tf.keras.Layer):
+    """Defines the residual block for the WaveNet TCN
+        Arguments:
+            x: The previous layer in the model
+            training: boolean indicating whether the layer should behave in training mode or in inference mode
+            dilation_rate: The dilation power of 2 we are using for this residual block
+            nb_filters: The number of convolutional filters to use in this block
+            kernel_size: The size of the convolutional kernel
+            padding: The padding used in the convolutional layers, 'same' or 'causal'.
+            activation: The final activation used in o = Activation(x + F(x))
+            dropout_rate: Float between 0 and 1. Fraction of the input units to drop.
+            kernel_initializer: Initializer for the kernel weights matrix (Conv1D).
+            use_batch_norm: Whether to use batch normalization in the residual layers or not.
+            kwargs: Any initializers for Layer class.
+        """
+
+    def __init__(self,
+                 dilation_rate,
+                 nb_filters,
+                 kernel_size,
+                 padding,
+                 activation = 'relu',
+                 dropout_rate = 0,
+                 kernel_initializer = 'he_normal',
+                 use_batch_norm = False,
+                 last_block = True,
+                 **kwargs):
+
+        self.dilation_rate=dilation_rate
+        self.nb_filters=nb_filters
+        self.kernel_size=kernel_size
+        self.padding=padding
+        self.activation=activation
+        self.dropout_rate=dropout_rate
+        self.use_batch_norm=use_batch_norm
+        self.kernel_initializer=kernel_initializer
+        self.last_block=last_block
+
+        super(ResidualBlock, self).__init__(**kwargs)
+
+    def _add_and_activate_layer(self, layer):
+        """Helper function for building layer
+        Args:
+            layer: Appends layer to internal layer list and builds it based on the current output
+                   shape of ResidualBlock. Updates current output shape.
+        """
+        self.residual_layers.append(layer)
+        self.residual_layers[-1].build(self.res_output_shape)
+        self.res_output_shape=self.residual_layers[-1].compute_output_shape(
+            self.res_output_shape)
+
+    def build(self, input_shape):
+
+        # name scope used to make sure weights get unique names
+        with K.name_scope(self.name):
+            self.residual_layers=list()
+            self.res_output_shape=input_shape
+
+            for k in range(2):
+                name='conv1D_{}'.format(k)
+                # name scope used to make sure weights get unique names
+                with K.name_scope(name):
+                    self._add_and_activate_layer(Conv1D(filters=self.nb_filters,
+                                                        kernel_size=self.kernel_size,
+                                                        dilation_rate=self.dilation_rate,
+                                                        padding=self.padding,
+                                                        name=name,
+                                                        kernel_initializer=self.kernel_initializer))
+
+                if self.use_batch_norm:
+                    # TODO should be WeightNorm here, but using batchNorm
+                    # instead
+                    self._add_and_activate_layer(BatchNormalization())
+
+                self._add_and_activate_layer(Activation('relu'))
+                self._add_and_activate_layer(
+                    SpatialDropout1D(rate=self.dropout_rate))
+
+            if not self.last_block:
+                # 1x1 conv to match the shapes (channel dimension).
+                name= 'conv1D_{}'.format(k + 1)
+                with K.name_scope(name):
+                    # make and build this layer separately because it directly
+                    # uses input_shape
+                    self.shape_match_conv= Conv1D(filters = self.nb_filters,
+                                                   kernel_size = 1,
+                                                   padding = 'same',
+                                                   name = name,
+                                                   kernel_initializer = self.kernel_initializer)
+
+            else:
+                 self.shape_match_conv=Lambda(lambda x: x, name='identity')
+
+            self.shape_match_conv.build(input_shape)
+            self.res_output_shape= self.shape_match_conv.compute_output_shape(input_shape)
+
+            self.final_activation= Activation(self.activation)
+            self.final_activation.build(
+    self.res_output_shape)  # probably isn't necessary
+
+            # this is done to force keras to add the layers in the list to
+            # self._layers
+            for layer in self.residual_layers:
+                self.__setattr__(layer.name, layer)
+
+            # done to make sure self.built is set True
+            super(ResidualBlock, self).build(input_shape)
+
+    def call(self, inputs, training=None):
+        """
+        Returns: A tuple where the first element is the residual model tensor, and the second
+                 is the skip connection tensor.
+        """
+        x= inputs
+        for layer in self.residual_layers:
+            if isinstance(layer, SpatialDropout1D):
+                x= layer(x, training=training)
+            else:
+                x= layer(x)
+
+        x2= self.shape_match_conv(inputs)
+        res_x= add([x2, x])
+        return [self.final_activation(res_x), x]
+
+    def compute_output_shape(self, input_shape):
+        return [self.res_output_shape, self.res_output_shape]
+
+    # TODO(shunlin): fix this get_config after finish fixing the APIs 
+    def get_config(self):
+        config = {}
+        base_config = super(ResidualBlock, self).get_config()
+        return dict(list(base_config.items()) + list(config.items()))
+
+
+def process_dilations(dilations):
+    def is_power_of_two(num):
+        return num != 0 and ((num & (num - 1)) == 0)
+
+    if all([is_power_of_two(i) for i in dilations]):
+        return dilations
+
+    else:
+        new_dilations= [2 ** i for i in dilations]
+        return new_dilations
+
+
+@tf.keras.utils.register_keras_serializable(package='Addons')
+class TCN(tf.keras.Layer):
+    """Creates a TCN layer.
+        Input shape:
+            A tensor of shape (batch_size, timesteps, input_dim).
+        Args:
+            nb_filters: The number of filters to use in the convolutional layers.
+            kernel_size: The size of the kernel to use in each convolutional layer.
+            dilations: The list of the dilations. Example is: [1, 2, 4, 8, 16, 32, 64].
+            nb_stacks : The number of stacks of residual blocks to use.
+            padding: The padding to use in the convolutional layers, 'causal' or 'same'.
+            use_skip_connections: Boolean. If we want to add skip connections from input to each residual block.
+            return_sequences: Boolean. Whether to return the last output in the output sequence, or the full sequence.
+            activation: The activation used in the residual blocks o = Activation(x + F(x)).
+            dropout_rate: Float between 0 and 1. Fraction of the input units to drop.
+            kernel_initializer: Initializer for the kernel weights matrix (Conv1D).
+            use_batch_norm: Whether to use batch normalization in the residual layers or not.
+            kwargs: Any other arguments for configuring parent class Layer. For example "name=str", Name of the model.
+                    Use unique names when using multiple TCN.
+        Returns:
+            A TCN layer.
+        """
+
+    def __init__(self,
+                 nb_filters = 64,
+                 kernel_size = 2,
+                 nb_stacks = 1,
+                 dilations = (1, 2, 4, 8, 16, 32),
+                 padding = 'causal',
+                 use_skip_connections = True,
+                 dropout_rate = 0.0,
+                 return_sequences = False,
+                 activation = 'linear',
+                 kernel_initializer = 'he_normal',
+                 use_batch_norm = False,
+                 **kwargs):
+
+        self.return_sequences= return_sequences
+        self.dropout_rate= dropout_rate
+        self.use_skip_connections= use_skip_connections
+        self.dilations= dilations
+        self.nb_stacks= nb_stacks
+        self.kernel_size= kernel_size
+        self.nb_filters= nb_filters
+        self.activation= activation
+        self.padding= padding
+        self.kernel_initializer= kernel_initializer
+        self.use_batch_norm= use_batch_norm
+
+        if padding != 'causal' and padding != 'same':
+            raise ValueError(
+                "Only 'causal' or 'same' padding are compatible for this layer.")
+
+        # initialize parent class
+        super(TCN, self).__init__(**kwargs)
+
+    def build(self, input_shape):
+        self.main_conv1D = Conv1D(filters =self.nb_filters,
+                                  kernel_size = 1,
+                                  padding = self.padding,
+                                  kernel_initializer =self.kernel_initializer)
+        self.main_conv1D.build(input_shape)
+
+        # member to hold current output shape of the layer for building
+        # purposes
+        self.build_output_shape= self.main_conv1D.compute_output_shape(input_shape)
+
+        # list to hold all the member ResidualBlocks
+        self.residual_blocks= list()
+        total_num_blocks= self.nb_stacks * len(self.dilations)
+        if not self.use_skip_connections:
+            total_num_blocks += 1  # cheap way to do a false case for below
+
+        for _ in range(self.nb_stacks):
+            for d in self.dilations:
+                self.residual_blocks.append(ResidualBlock(dilation_rate=d,
+                                                          nb_filters=self.nb_filters,
+                                                          kernel_size=self.kernel_size,
+                                                          padding=self.padding,
+                                                          activation=self.activation,
+                                                          dropout_rate=self.dropout_rate,
+                                                          use_batch_norm=self.use_batch_norm,
+                                                          kernel_initializer=self.kernel_initializer,
+                                                          last_block = len(self.residual_blocks) +1 == total_num_blocks,
+                                                          name='residual_block_{}'.format(len(self.residual_blocks))))
+                # build newest residual block
+                self.residual_blocks[-1].build(self.build_output_shape)
+                self.build_output_shape= self.residual_blocks[-1].res_output_shape
+
+        # this is done to force keras to add the layers in the list to
+        # self._layers
+        for layer in self.residual_blocks:
+            self.__setattr__(layer.name, layer)
+
+        self.lambda_layer= Lambda(lambda tt: tt[:, -1, :])
+        self.lambda_ouput_shape= self.lambda_layer.compute_output_shape(self.build_output_shape)
+
+    def compute_output_shape(self, input_shape):
+        """
+        Overridden in case keras uses it somewhere... no idea. Just trying to avoid future errors.
+        """
+        if not self.built:
+            self.build(input_shape)
+        if not self.return_sequences:
+            return self.lambda_ouput_shape
+        else:
+            return self.build_output_shape
+
+    def call(self, inputs, training=None):
+        x= inputs
+        x= self.main_conv1D(x)
+        skip_connections= list()
+        for layer in self.residual_blocks:
+            x, skip_out= layer(x, training=training)
+            skip_connections.append(skip_out)
+
+        if self.use_skip_connections:
+            x= add(skip_connections)
+        if not self.return_sequences:
+            x= self.lambda_layer(x)
+        return x
+
+    def get_config(self):
+        config = dict()
+        config['nb_filters']= self.nb_filters
+        config['kernel_size']= self.kernel_size
+        config['nb_stacks']= self.nb_stacks
+        config['dilations']= self.dilations
+        config['padding']= self.padding
+        config['use_skip_connections']= self.use_skip_connections
+        config['dropout_rate']= self.dropout_rate
+        config['return_sequences']= self.return_sequences
+        config['activation']= self.activation
+        config['use_batch_norm']= self.use_batch_norm
+        config['kernel_initializer']= self.kernel_initializer
+
+        base_config = super(TCN, self).get_config()
+
+        return dict(list(base_config.items()) + list(config.items()))
diff --git a/tensorflow_addons/layers/tcn_test.py b/tensorflow_addons/layers/tcn_test.py
new file mode 100644
index 0000000000..7a8764411e
--- /dev/null
+++ b/tensorflow_addons/layers/tcn_test.py
@@ -0,0 +1,31 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for TCN layer."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+import tensorflow as tf
+from tensorflow_addons.utils import test_utils
+from tensorflow_addons.layers import TCN
+
+
+@test_utils.run_all_in_graph_and_eager_modes
+class TCNTest(tf.test.TestCase):
+    # TODO(shunlin): implment unit test
+    def test_simple(self):
+        pass

From a8c0e84e0f4476902efa3d6c09ed9fe011f7184d Mon Sep 17 00:00:00 2001
From: Shun Lin <shunlin@google.com>
Date: Thu, 7 Nov 2019 17:11:49 -0800
Subject: [PATCH 02/20] remove nb_ in variable naming as do not confine to
 tensorflow naming convention

---
 tensorflow_addons/layers/tcn.py | 35 ++++++++++++++++-----------------
 1 file changed, 17 insertions(+), 18 deletions(-)

diff --git a/tensorflow_addons/layers/tcn.py b/tensorflow_addons/layers/tcn.py
index aa460fcd82..b20ff5cb50 100644
--- a/tensorflow_addons/layers/tcn.py
+++ b/tensorflow_addons/layers/tcn.py
@@ -28,9 +28,8 @@ class ResidualBlock(tf.keras.Layer):
     """Defines the residual block for the WaveNet TCN
         Arguments:
             x: The previous layer in the model
-            training: boolean indicating whether the layer should behave in training mode or in inference mode
             dilation_rate: The dilation power of 2 we are using for this residual block
-            nb_filters: The number of convolutional filters to use in this block
+            filters: The number of convolutional filters to use in this block
             kernel_size: The size of the convolutional kernel
             padding: The padding used in the convolutional layers, 'same' or 'causal'.
             activation: The final activation used in o = Activation(x + F(x))
@@ -42,7 +41,7 @@ class ResidualBlock(tf.keras.Layer):
 
     def __init__(self,
                  dilation_rate,
-                 nb_filters,
+                 filters,
                  kernel_size,
                  padding,
                  activation = 'relu',
@@ -53,7 +52,7 @@ def __init__(self,
                  **kwargs):
 
         self.dilation_rate=dilation_rate
-        self.nb_filters=nb_filters
+        self.filters=filters
         self.kernel_size=kernel_size
         self.padding=padding
         self.activation=activation
@@ -86,7 +85,7 @@ def build(self, input_shape):
                 name='conv1D_{}'.format(k)
                 # name scope used to make sure weights get unique names
                 with K.name_scope(name):
-                    self._add_and_activate_layer(Conv1D(filters=self.nb_filters,
+                    self._add_and_activate_layer(Conv1D(filters=self.filters,
                                                         kernel_size=self.kernel_size,
                                                         dilation_rate=self.dilation_rate,
                                                         padding=self.padding,
@@ -108,7 +107,7 @@ def build(self, input_shape):
                 with K.name_scope(name):
                     # make and build this layer separately because it directly
                     # uses input_shape
-                    self.shape_match_conv= Conv1D(filters = self.nb_filters,
+                    self.shape_match_conv= Conv1D(filters = self.filters,
                                                    kernel_size = 1,
                                                    padding = 'same',
                                                    name = name,
@@ -176,10 +175,10 @@ class TCN(tf.keras.Layer):
         Input shape:
             A tensor of shape (batch_size, timesteps, input_dim).
         Args:
-            nb_filters: The number of filters to use in the convolutional layers.
+            filters: The number of filters to use in the convolutional layers.
             kernel_size: The size of the kernel to use in each convolutional layer.
             dilations: The list of the dilations. Example is: [1, 2, 4, 8, 16, 32, 64].
-            nb_stacks : The number of stacks of residual blocks to use.
+            stacks : The number of stacks of residual blocks to use.
             padding: The padding to use in the convolutional layers, 'causal' or 'same'.
             use_skip_connections: Boolean. If we want to add skip connections from input to each residual block.
             return_sequences: Boolean. Whether to return the last output in the output sequence, or the full sequence.
@@ -194,9 +193,9 @@ class TCN(tf.keras.Layer):
         """
 
     def __init__(self,
-                 nb_filters = 64,
+                 filters = 64,
                  kernel_size = 2,
-                 nb_stacks = 1,
+                 stacks = 1,
                  dilations = (1, 2, 4, 8, 16, 32),
                  padding = 'causal',
                  use_skip_connections = True,
@@ -211,9 +210,9 @@ def __init__(self,
         self.dropout_rate= dropout_rate
         self.use_skip_connections= use_skip_connections
         self.dilations= dilations
-        self.nb_stacks= nb_stacks
+        self.stacks= stacks
         self.kernel_size= kernel_size
-        self.nb_filters= nb_filters
+        self.filters= filters
         self.activation= activation
         self.padding= padding
         self.kernel_initializer= kernel_initializer
@@ -227,7 +226,7 @@ def __init__(self,
         super(TCN, self).__init__(**kwargs)
 
     def build(self, input_shape):
-        self.main_conv1D = Conv1D(filters =self.nb_filters,
+        self.main_conv1D = Conv1D(filters =self.filters,
                                   kernel_size = 1,
                                   padding = self.padding,
                                   kernel_initializer =self.kernel_initializer)
@@ -239,14 +238,14 @@ def build(self, input_shape):
 
         # list to hold all the member ResidualBlocks
         self.residual_blocks= list()
-        total_num_blocks= self.nb_stacks * len(self.dilations)
+        total_num_blocks= self.stacks * len(self.dilations)
         if not self.use_skip_connections:
             total_num_blocks += 1  # cheap way to do a false case for below
 
-        for _ in range(self.nb_stacks):
+        for _ in range(self.stacks):
             for d in self.dilations:
                 self.residual_blocks.append(ResidualBlock(dilation_rate=d,
-                                                          nb_filters=self.nb_filters,
+                                                          filters=self.filters,
                                                           kernel_size=self.kernel_size,
                                                           padding=self.padding,
                                                           activation=self.activation,
@@ -294,9 +293,9 @@ def call(self, inputs, training=None):
 
     def get_config(self):
         config = dict()
-        config['nb_filters']= self.nb_filters
+        config['filters']= self.filters
         config['kernel_size']= self.kernel_size
-        config['nb_stacks']= self.nb_stacks
+        config['stacks']= self.stacks
         config['dilations']= self.dilations
         config['padding']= self.padding
         config['use_skip_connections']= self.use_skip_connections

From 96eb25e00361b565e65228330c3154b3e36c85dd Mon Sep 17 00:00:00 2001
From: Shun Lin <shunlin@google.com>
Date: Sun, 10 Nov 2019 09:34:18 -0800
Subject: [PATCH 03/20] clean equal sign format

---
 tensorflow_addons/layers/tcn.py | 293 ++++++++++++++++++--------------
 1 file changed, 165 insertions(+), 128 deletions(-)

diff --git a/tensorflow_addons/layers/tcn.py b/tensorflow_addons/layers/tcn.py
index b20ff5cb50..d7f64d13e8 100644
--- a/tensorflow_addons/layers/tcn.py
+++ b/tensorflow_addons/layers/tcn.py
@@ -26,71 +26,85 @@
 @tf.keras.utils.register_keras_serializable(package='Addons')
 class ResidualBlock(tf.keras.Layer):
     """Defines the residual block for the WaveNet TCN
+
         Arguments:
-            x: The previous layer in the model
-            dilation_rate: The dilation power of 2 we are using for this residual block
-            filters: The number of convolutional filters to use in this block
-            kernel_size: The size of the convolutional kernel
-            padding: The padding used in the convolutional layers, 'same' or 'causal'.
-            activation: The final activation used in o = Activation(x + F(x))
-            dropout_rate: Float between 0 and 1. Fraction of the input units to drop.
-            kernel_initializer: Initializer for the kernel weights matrix (Conv1D).
-            use_batch_norm: Whether to use batch normalization in the residual layers or not.
+            dilation_rate (int): The dilation power of 2 we are using
+                for this residual block.
+            filters (int): The number of convolutional
+                filters to use in this block.
+            kernel_size (int): The size of the convolutional kernel.
+            padding (String): The padding used in the convolutional layers,
+                'same' or 'causal'.
+            activation (String): The final activation used
+                in o = Activation(x + F(x)).
+            dropout_rate (Float): Float between 0 and 1. Fraction
+                of the input units to drop.
+            kernel_initializer (String): Initializer for the kernel weights
+                matrix (Conv1D).
+            use_batch_norm (bool): Whether to use batch normalization in the
+                residual layers or not.
+            last_block (bool):
             kwargs: Any initializers for Layer class.
+
+        Returns:
+            A Residual Blcok.
         """
 
     def __init__(self,
-                 dilation_rate,
-                 filters,
-                 kernel_size,
-                 padding,
-                 activation = 'relu',
-                 dropout_rate = 0,
-                 kernel_initializer = 'he_normal',
-                 use_batch_norm = False,
-                 last_block = True,
+                 dilation_rate=1,
+                 filters=64,
+                 kernel_size=2,
+                 padding=0,
+                 activation='relu',
+                 dropout_rate=0,
+                 kernel_initializer='he_normal',
+                 use_batch_norm=False,
+                 last_block=True,
                  **kwargs):
 
-        self.dilation_rate=dilation_rate
-        self.filters=filters
-        self.kernel_size=kernel_size
-        self.padding=padding
-        self.activation=activation
-        self.dropout_rate=dropout_rate
-        self.use_batch_norm=use_batch_norm
-        self.kernel_initializer=kernel_initializer
-        self.last_block=last_block
+        self.dilation_rate = dilation_rate
+        self.filters = filters
+        self.kernel_size = kernel_size
+        self.padding = padding
+        self.activation = activation
+        self.dropout_rate = dropout_rate
+        self.use_batch_norm = use_batch_norm
+        self.kernel_initializer = kernel_initializer
+        self.last_block = last_block
 
         super(ResidualBlock, self).__init__(**kwargs)
 
     def _add_and_activate_layer(self, layer):
         """Helper function for building layer
         Args:
-            layer: Appends layer to internal layer list and builds it based on the current output
-                   shape of ResidualBlock. Updates current output shape.
+            layer: Appends layer to internal layer list and builds it based on
+                    the current output shape of ResidualBlock.
+                    Updates current output shape.
         """
         self.residual_layers.append(layer)
         self.residual_layers[-1].build(self.res_output_shape)
-        self.res_output_shape=self.residual_layers[-1].compute_output_shape(
+        self.res_output_shape = self.residual_layers[-1].compute_output_shape(
             self.res_output_shape)
 
     def build(self, input_shape):
 
         # name scope used to make sure weights get unique names
         with K.name_scope(self.name):
-            self.residual_layers=list()
-            self.res_output_shape=input_shape
+            self.residual_layers = list()
+            self.res_output_shape = input_shape
 
             for k in range(2):
-                name='conv1D_{}'.format(k)
+                name = 'conv1D_{}'.format(k)
                 # name scope used to make sure weights get unique names
                 with K.name_scope(name):
-                    self._add_and_activate_layer(Conv1D(filters=self.filters,
-                                                        kernel_size=self.kernel_size,
-                                                        dilation_rate=self.dilation_rate,
-                                                        padding=self.padding,
-                                                        name=name,
-                                                        kernel_initializer=self.kernel_initializer))
+                    self._add_and_activate_layer(
+                        Conv1D(
+                            filters=self.filters,
+                            kernel_size=self.kernel_size,
+                            dilation_rate=self.dilation_rate,
+                            padding=self.padding,
+                            name=name,
+                            kernel_initializer=self.kernel_initializer))
 
                 if self.use_batch_norm:
                     # TODO should be WeightNorm here, but using batchNorm
@@ -103,25 +117,27 @@ def build(self, input_shape):
 
             if not self.last_block:
                 # 1x1 conv to match the shapes (channel dimension).
-                name= 'conv1D_{}'.format(k + 1)
+                name = 'conv1D_{}'.format(k + 1)
                 with K.name_scope(name):
                     # make and build this layer separately because it directly
                     # uses input_shape
-                    self.shape_match_conv= Conv1D(filters = self.filters,
-                                                   kernel_size = 1,
-                                                   padding = 'same',
-                                                   name = name,
-                                                   kernel_initializer = self.kernel_initializer)
+                    self.shape_match_conv = Conv1D(
+                        filters=self.filters,
+                        kernel_size=1,
+                        padding='same',
+                        name=name,
+                        kernel_initializer=self.kernel_initializer)
 
             else:
-                 self.shape_match_conv=Lambda(lambda x: x, name='identity')
+                self.shape_match_conv = Lambda(lambda x: x, name='identity')
 
             self.shape_match_conv.build(input_shape)
-            self.res_output_shape= self.shape_match_conv.compute_output_shape(input_shape)
+            self.res_output_shape = self.shape_match_conv.compute_output_shape(
+                input_shape)
 
-            self.final_activation= Activation(self.activation)
+            self.final_activation = Activation(self.activation)
             self.final_activation.build(
-    self.res_output_shape)  # probably isn't necessary
+                self.res_output_shape)  # probably isn't necessary
 
             # this is done to force keras to add the layers in the list to
             # self._layers
@@ -136,23 +152,24 @@ def call(self, inputs, training=None):
         Returns: A tuple where the first element is the residual model tensor, and the second
                  is the skip connection tensor.
         """
-        x= inputs
+        x = inputs
         for layer in self.residual_layers:
             if isinstance(layer, SpatialDropout1D):
-                x= layer(x, training=training)
+                x = layer(x, training=training)
             else:
-                x= layer(x)
+                x = layer(x)
 
-        x2= self.shape_match_conv(inputs)
-        res_x= add([x2, x])
+        x2 = self.shape_match_conv(inputs)
+        res_x = add([x2, x])
         return [self.final_activation(res_x), x]
 
     def compute_output_shape(self, input_shape):
         return [self.res_output_shape, self.res_output_shape]
 
-    # TODO(shunlin): fix this get_config after finish fixing the APIs 
+    # TODO(shunlin): fix this get_config after finish fixing the APIs
     def get_config(self):
-        config = {}
+        config = dict()
+
         base_config = super(ResidualBlock, self).get_config()
         return dict(list(base_config.items()) + list(config.items()))
 
@@ -165,58 +182,70 @@ def is_power_of_two(num):
         return dilations
 
     else:
-        new_dilations= [2 ** i for i in dilations]
+        new_dilations = [2 ** i for i in dilations]
         return new_dilations
 
 
 @tf.keras.utils.register_keras_serializable(package='Addons')
 class TCN(tf.keras.Layer):
     """Creates a TCN layer.
+
         Input shape:
             A tensor of shape (batch_size, timesteps, input_dim).
-        Args:
+
+        Arguments:
             filters: The number of filters to use in the convolutional layers.
-            kernel_size: The size of the kernel to use in each convolutional layer.
-            dilations: The list of the dilations. Example is: [1, 2, 4, 8, 16, 32, 64].
+            kernel_size: The size of the kernel to use in each
+                convolutional layer.
+            dilations: The list of the dilations.
+                Example is: [1, 2, 4, 8, 16, 32, 64].
             stacks : The number of stacks of residual blocks to use.
-            padding: The padding to use in the convolutional layers, 'causal' or 'same'.
-            use_skip_connections: Boolean. If we want to add skip connections from input to each residual block.
-            return_sequences: Boolean. Whether to return the last output in the output sequence, or the full sequence.
-            activation: The activation used in the residual blocks o = Activation(x + F(x)).
-            dropout_rate: Float between 0 and 1. Fraction of the input units to drop.
-            kernel_initializer: Initializer for the kernel weights matrix (Conv1D).
-            use_batch_norm: Whether to use batch normalization in the residual layers or not.
-            kwargs: Any other arguments for configuring parent class Layer. For example "name=str", Name of the model.
-                    Use unique names when using multiple TCN.
+            padding: The padding to use in the convolutional layers,
+                'causal' or 'same'.
+            use_skip_connections: Boolean. If we want to add skip
+                connections from input to each residual block.
+            return_sequences: Boolean. Whether to return the last
+                output in the output sequence, or the full sequence.
+            activation: The activation used in the residual
+                blocks o = Activation(x + F(x)).
+            dropout_rate: Float between 0 and 1. Fraction of the input
+                units to drop.
+            kernel_initializer: Initializer for the kernel weights
+                matrix (Conv1D).
+            use_batch_norm: Whether to use batch normalization in the
+                residual layers or not.
+            kwargs: Any other arguments for configuring parent class Layer.
+                For example "name=str", Name of the model.
+                Use unique names when using multiple TCN.
         Returns:
             A TCN layer.
         """
 
     def __init__(self,
-                 filters = 64,
-                 kernel_size = 2,
-                 stacks = 1,
-                 dilations = (1, 2, 4, 8, 16, 32),
-                 padding = 'causal',
-                 use_skip_connections = True,
-                 dropout_rate = 0.0,
-                 return_sequences = False,
-                 activation = 'linear',
-                 kernel_initializer = 'he_normal',
-                 use_batch_norm = False,
+                 filters=64,
+                 kernel_size=2,
+                 stacks=1,
+                 dilations=(1, 2, 4, 8, 16, 32),
+                 padding='causal',
+                 use_skip_connections=True,
+                 dropout_rate=0.0,
+                 return_sequences=False,
+                 activation='linear',
+                 kernel_initializer='he_normal',
+                 use_batch_norm=False,
                  **kwargs):
 
-        self.return_sequences= return_sequences
-        self.dropout_rate= dropout_rate
-        self.use_skip_connections= use_skip_connections
-        self.dilations= dilations
-        self.stacks= stacks
-        self.kernel_size= kernel_size
-        self.filters= filters
-        self.activation= activation
-        self.padding= padding
-        self.kernel_initializer= kernel_initializer
-        self.use_batch_norm= use_batch_norm
+        self.return_sequences = return_sequences
+        self.dropout_rate = dropout_rate
+        self.use_skip_connections = use_skip_connections
+        self.dilations = dilations
+        self.stacks = stacks
+        self.kernel_size = kernel_size
+        self.filters = filters
+        self.activation = activation
+        self.padding = padding
+        self.kernel_initializer = kernel_initializer
+        self.use_batch_norm = use_batch_norm
 
         if padding != 'causal' and padding != 'same':
             raise ValueError(
@@ -226,45 +255,53 @@ def __init__(self,
         super(TCN, self).__init__(**kwargs)
 
     def build(self, input_shape):
-        self.main_conv1D = Conv1D(filters =self.filters,
-                                  kernel_size = 1,
-                                  padding = self.padding,
-                                  kernel_initializer =self.kernel_initializer)
+        self.main_conv1D = Conv1D(filters=self.filters,
+                                  kernel_size=1,
+                                  padding=self.padding,
+                                  kernel_initializer=self.kernel_initializer)
         self.main_conv1D.build(input_shape)
 
         # member to hold current output shape of the layer for building
         # purposes
-        self.build_output_shape= self.main_conv1D.compute_output_shape(input_shape)
+        self.build_output_shape = self.main_conv1D.compute_output_shape(
+            input_shape)
 
         # list to hold all the member ResidualBlocks
-        self.residual_blocks= list()
-        total_num_blocks= self.stacks * len(self.dilations)
+        self.residual_blocks = list()
+        total_num_blocks = self.stacks * len(self.dilations)
         if not self.use_skip_connections:
             total_num_blocks += 1  # cheap way to do a false case for below
 
         for _ in range(self.stacks):
             for d in self.dilations:
-                self.residual_blocks.append(ResidualBlock(dilation_rate=d,
-                                                          filters=self.filters,
-                                                          kernel_size=self.kernel_size,
-                                                          padding=self.padding,
-                                                          activation=self.activation,
-                                                          dropout_rate=self.dropout_rate,
-                                                          use_batch_norm=self.use_batch_norm,
-                                                          kernel_initializer=self.kernel_initializer,
-                                                          last_block = len(self.residual_blocks) +1 == total_num_blocks,
-                                                          name='residual_block_{}'.format(len(self.residual_blocks))))
+                self.residual_blocks.append(
+                    ResidualBlock(
+                        dilation_rate=d,
+                        filters=self.filters,
+                        kernel_size=self.kernel_size,
+                        padding=self.padding,
+                        activation=self.activation,
+                        dropout_rate=self.dropout_rate,
+                        use_batch_norm=self.use_batch_norm,
+                        kernel_initializer=self.kernel_initializer,
+                        last_block=len(
+                            self.residual_blocks) +
+                        1 == total_num_blocks,
+                        name='residual_block_{}'.format(
+                            len(
+                                self.residual_blocks))))
                 # build newest residual block
                 self.residual_blocks[-1].build(self.build_output_shape)
-                self.build_output_shape= self.residual_blocks[-1].res_output_shape
+                self.build_output_shape = self.residual_blocks[-1].res_output_shape
 
         # this is done to force keras to add the layers in the list to
         # self._layers
         for layer in self.residual_blocks:
             self.__setattr__(layer.name, layer)
 
-        self.lambda_layer= Lambda(lambda tt: tt[:, -1, :])
-        self.lambda_ouput_shape= self.lambda_layer.compute_output_shape(self.build_output_shape)
+        self.lambda_layer = Lambda(lambda tt: tt[:, -1, :])
+        self.lambda_ouput_shape = self.lambda_layer.compute_output_shape(
+            self.build_output_shape)
 
     def compute_output_shape(self, input_shape):
         """
@@ -278,32 +315,32 @@ def compute_output_shape(self, input_shape):
             return self.build_output_shape
 
     def call(self, inputs, training=None):
-        x= inputs
-        x= self.main_conv1D(x)
-        skip_connections= list()
+        x = inputs
+        x = self.main_conv1D(x)
+        skip_connections = list()
         for layer in self.residual_blocks:
-            x, skip_out= layer(x, training=training)
+            x, skip_out = layer(x, training=training)
             skip_connections.append(skip_out)
 
         if self.use_skip_connections:
-            x= add(skip_connections)
+            x = add(skip_connections)
         if not self.return_sequences:
-            x= self.lambda_layer(x)
+            x = self.lambda_layer(x)
         return x
 
     def get_config(self):
         config = dict()
-        config['filters']= self.filters
-        config['kernel_size']= self.kernel_size
-        config['stacks']= self.stacks
-        config['dilations']= self.dilations
-        config['padding']= self.padding
-        config['use_skip_connections']= self.use_skip_connections
-        config['dropout_rate']= self.dropout_rate
-        config['return_sequences']= self.return_sequences
-        config['activation']= self.activation
-        config['use_batch_norm']= self.use_batch_norm
-        config['kernel_initializer']= self.kernel_initializer
+        config['filters'] = self.filters
+        config['kernel_size'] = self.kernel_size
+        config['stacks'] = self.stacks
+        config['dilations'] = self.dilations
+        config['padding'] = self.padding
+        config['use_skip_connections'] = self.use_skip_connections
+        config['dropout_rate'] = self.dropout_rate
+        config['return_sequences'] = self.return_sequences
+        config['activation'] = self.activation
+        config['use_batch_norm'] = self.use_batch_norm
+        config['kernel_initializer'] = self.kernel_initializer
 
         base_config = super(TCN, self).get_config()
 

From bdbf308f57b8391a1f9d146d3081ec7efdf37acb Mon Sep 17 00:00:00 2001
From: Shun Lin <shunlin@google.com>
Date: Sun, 10 Nov 2019 15:27:25 -0800
Subject: [PATCH 04/20] improve documentation

---
 tensorflow_addons/layers/tcn.py | 48 ++++++++++++++++++---------------
 1 file changed, 27 insertions(+), 21 deletions(-)

diff --git a/tensorflow_addons/layers/tcn.py b/tensorflow_addons/layers/tcn.py
index d7f64d13e8..2185241dad 100644
--- a/tensorflow_addons/layers/tcn.py
+++ b/tensorflow_addons/layers/tcn.py
@@ -29,21 +29,23 @@ class ResidualBlock(tf.keras.Layer):
 
         Arguments:
             dilation_rate (int): The dilation power of 2 we are using
-                for this residual block.
+                for this residual block. Defaults to 1.
             filters (int): The number of convolutional
-                filters to use in this block.
-            kernel_size (int): The size of the convolutional kernel.
+                filters to use in this block. Defaults to 64.
+            kernel_size (int): The size of the convolutional kernel. Defaults
+                to 2.
             padding (String): The padding used in the convolutional layers,
-                'same' or 'causal'.
+                'same' or 'causal'. Defaults to 'same'
             activation (String): The final activation used
-                in o = Activation(x + F(x)).
+                in o = Activation(x + F(x)). Defaults to 'relu'
             dropout_rate (Float): Float between 0 and 1. Fraction
-                of the input units to drop.
+                of the input units to drop. Defaults to 0.0.
             kernel_initializer (String): Initializer for the kernel weights
-                matrix (Conv1D).
+                matrix (Conv1D). Defaults to 'he_normal'
             use_batch_norm (bool): Whether to use batch normalization in the
-                residual layers or not.
-            last_block (bool):
+                residual layers or not. Defaults to False.
+            last_block (bool): Whether or not this block is the last residual
+                block of the network. Defaults to False.
             kwargs: Any initializers for Layer class.
 
         Returns:
@@ -54,12 +56,12 @@ def __init__(self,
                  dilation_rate=1,
                  filters=64,
                  kernel_size=2,
-                 padding=0,
+                 padding='same',
                  activation='relu',
-                 dropout_rate=0,
+                 dropout_rate=0.0,
                  kernel_initializer='he_normal',
                  use_batch_norm=False,
-                 last_block=True,
+                 last_block=False,
                  **kwargs):
 
         self.dilation_rate = dilation_rate
@@ -195,25 +197,29 @@ class TCN(tf.keras.Layer):
 
         Arguments:
             filters: The number of filters to use in the convolutional layers.
+                Defaults to 64.
             kernel_size: The size of the kernel to use in each
-                convolutional layer.
-            dilations: The list of the dilations.
-                Example is: [1, 2, 4, 8, 16, 32, 64].
-            stacks : The number of stacks of residual blocks to use.
+                convolutional layer. Defaults to 2.
+            dilations: The list-like input of the dilations.
+                Defults to (1, 2, 4, 8, 16, 32, 64).
+            stacks : The number of stacks of residual blocks to use. Defaults
+                to 1.
             padding: The padding to use in the convolutional layers,
-                'causal' or 'same'.
+                'causal' or 'same'. Defaults to 'causal'.
             use_skip_connections: Boolean. If we want to add skip
                 connections from input to each residual block.
+                Defaults to True.
             return_sequences: Boolean. Whether to return the last
                 output in the output sequence, or the full sequence.
+                Defaults to False.
             activation: The activation used in the residual
-                blocks o = Activation(x + F(x)).
+                blocks o = Activation(x + F(x)). Defaults to 'linear'
             dropout_rate: Float between 0 and 1. Fraction of the input
-                units to drop.
+                units to drop. Defaults to 0.0.
             kernel_initializer: Initializer for the kernel weights
-                matrix (Conv1D).
+                matrix (Conv1D). Defaulst to 'he_normal'
             use_batch_norm: Whether to use batch normalization in the
-                residual layers or not.
+                residual layers or not. Defaulst to False.
             kwargs: Any other arguments for configuring parent class Layer.
                 For example "name=str", Name of the model.
                 Use unique names when using multiple TCN.

From 9290146a03888d7d6913f1e304083403ef500f1c Mon Sep 17 00:00:00 2001
From: Shun Lin <shunlin@google.com>
Date: Sun, 10 Nov 2019 15:43:36 -0800
Subject: [PATCH 05/20] simplify padding check

---
 tensorflow_addons/layers/tcn.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tensorflow_addons/layers/tcn.py b/tensorflow_addons/layers/tcn.py
index 2185241dad..afe50d80c4 100644
--- a/tensorflow_addons/layers/tcn.py
+++ b/tensorflow_addons/layers/tcn.py
@@ -253,9 +253,11 @@ def __init__(self,
         self.kernel_initializer = kernel_initializer
         self.use_batch_norm = use_batch_norm
 
-        if padding != 'causal' and padding != 'same':
+        validate_paddings = ['causal', 'same']
+
+        if padding not in validate_paddings:
             raise ValueError(
-                "Only 'causal' or 'same' padding are compatible for this layer.")
+                "Only 'causal' or 'same' padding are compatible for this layer")
 
         # initialize parent class
         super(TCN, self).__init__(**kwargs)

From 4db6f1e9432dc26ec809a35cb85c9db1929add76 Mon Sep 17 00:00:00 2001
From: Shun Lin <shunlin@google.com>
Date: Sun, 10 Nov 2019 15:57:33 -0800
Subject: [PATCH 06/20] delete non-used code and changed default behavior of
 TCN

---
 tensorflow_addons/layers/tcn.py | 29 ++++++++++++-----------------
 1 file changed, 12 insertions(+), 17 deletions(-)

diff --git a/tensorflow_addons/layers/tcn.py b/tensorflow_addons/layers/tcn.py
index afe50d80c4..0391b59596 100644
--- a/tensorflow_addons/layers/tcn.py
+++ b/tensorflow_addons/layers/tcn.py
@@ -176,18 +176,6 @@ def get_config(self):
         return dict(list(base_config.items()) + list(config.items()))
 
 
-def process_dilations(dilations):
-    def is_power_of_two(num):
-        return num != 0 and ((num & (num - 1)) == 0)
-
-    if all([is_power_of_two(i) for i in dilations]):
-        return dilations
-
-    else:
-        new_dilations = [2 ** i for i in dilations]
-        return new_dilations
-
-
 @tf.keras.utils.register_keras_serializable(package='Addons')
 class TCN(tf.keras.Layer):
     """Creates a TCN layer.
@@ -200,8 +188,10 @@ class TCN(tf.keras.Layer):
                 Defaults to 64.
             kernel_size: The size of the kernel to use in each
                 convolutional layer. Defaults to 2.
-            dilations: The list-like input of the dilations.
-                Defults to (1, 2, 4, 8, 16, 32, 64).
+            dilations: The list-like input of the dilations. The size of this
+                input should be equal to the number of stacks. If None,
+                will defaults to power of 2 dilutions. For example, if
+                stacks = 3, will Defaults to [1,2,4].
             stacks : The number of stacks of residual blocks to use. Defaults
                 to 1.
             padding: The padding to use in the convolutional layers,
@@ -231,7 +221,7 @@ def __init__(self,
                  filters=64,
                  kernel_size=2,
                  stacks=1,
-                 dilations=(1, 2, 4, 8, 16, 32),
+                 dilations=None,
                  padding='causal',
                  use_skip_connections=True,
                  dropout_rate=0.0,
@@ -244,7 +234,6 @@ def __init__(self,
         self.return_sequences = return_sequences
         self.dropout_rate = dropout_rate
         self.use_skip_connections = use_skip_connections
-        self.dilations = dilations
         self.stacks = stacks
         self.kernel_size = kernel_size
         self.filters = filters
@@ -253,8 +242,14 @@ def __init__(self,
         self.kernel_initializer = kernel_initializer
         self.use_batch_norm = use_batch_norm
 
-        validate_paddings = ['causal', 'same']
+        # generate default dilations if custom dilations is not provided.
+        if not dilations:
+            self.dilations = [2 ** i for i in range(stacks)]
+        else:
+            self.dilations = dilations
 
+        # validate paddings
+        validate_paddings = ['causal', 'same']
         if padding not in validate_paddings:
             raise ValueError(
                 "Only 'causal' or 'same' padding are compatible for this layer")

From dfe3828e57f0e5239338ad7f1249816438c5cbe7 Mon Sep 17 00:00:00 2001
From: Shun Lin <shunlin@google.com>
Date: Sun, 10 Nov 2019 16:29:20 -0800
Subject: [PATCH 07/20] changed back the dilations

---
 tensorflow_addons/layers/tcn.py | 17 +++--------------
 1 file changed, 3 insertions(+), 14 deletions(-)

diff --git a/tensorflow_addons/layers/tcn.py b/tensorflow_addons/layers/tcn.py
index 0391b59596..a4e2525485 100644
--- a/tensorflow_addons/layers/tcn.py
+++ b/tensorflow_addons/layers/tcn.py
@@ -188,10 +188,8 @@ class TCN(tf.keras.Layer):
                 Defaults to 64.
             kernel_size: The size of the kernel to use in each
                 convolutional layer. Defaults to 2.
-            dilations: The list-like input of the dilations. The size of this
-                input should be equal to the number of stacks. If None,
-                will defaults to power of 2 dilutions. For example, if
-                stacks = 3, will Defaults to [1,2,4].
+            dilations: The array-like input of the dilations.
+                Defaults to [1,2,4,8,16,32,64]
             stacks : The number of stacks of residual blocks to use. Defaults
                 to 1.
             padding: The padding to use in the convolutional layers,
@@ -221,7 +219,7 @@ def __init__(self,
                  filters=64,
                  kernel_size=2,
                  stacks=1,
-                 dilations=None,
+                 dilations=[1, 2, 4, 8, 16, 32, 64],
                  padding='causal',
                  use_skip_connections=True,
                  dropout_rate=0.0,
@@ -242,12 +240,6 @@ def __init__(self,
         self.kernel_initializer = kernel_initializer
         self.use_batch_norm = use_batch_norm
 
-        # generate default dilations if custom dilations is not provided.
-        if not dilations:
-            self.dilations = [2 ** i for i in range(stacks)]
-        else:
-            self.dilations = dilations
-
         # validate paddings
         validate_paddings = ['causal', 'same']
         if padding not in validate_paddings:
@@ -307,9 +299,6 @@ def build(self, input_shape):
             self.build_output_shape)
 
     def compute_output_shape(self, input_shape):
-        """
-        Overridden in case keras uses it somewhere... no idea. Just trying to avoid future errors.
-        """
         if not self.built:
             self.build(input_shape)
         if not self.return_sequences:

From 8c6044562297f2d05a1d6aa9089fbebf408408e1 Mon Sep 17 00:00:00 2001
From: Shun Lin <shunlin@google.com>
Date: Sun, 10 Nov 2019 18:01:34 -0800
Subject: [PATCH 08/20] added basic ipynb tutorial and update get_config

---
 .../temporal_convolutional_network.ipynb      | 317 ++++++++++++++++++
 tensorflow_addons/layers/tcn.py               |  20 +-
 2 files changed, 332 insertions(+), 5 deletions(-)
 create mode 100644 docs/tutorials/temporal_convolutional_network.ipynb

diff --git a/docs/tutorials/temporal_convolutional_network.ipynb b/docs/tutorials/temporal_convolutional_network.ipynb
new file mode 100644
index 0000000000..659851f3dc
--- /dev/null
+++ b/docs/tutorials/temporal_convolutional_network.ipynb
@@ -0,0 +1,317 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "##### Copyright 2019 The TensorFlow Authors."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+    "# you may not use this file except in compliance with the License.\n",
+    "# You may obtain a copy of the License at\n",
+    "#\n",
+    "# https://www.apache.org/licenses/LICENSE-2.0\n",
+    "#\n",
+    "# Unless required by applicable law or agreed to in writing, software\n",
+    "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+    "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+    "# See the License for the specific language governing permissions and\n",
+    "# limitations under the License."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# TensorFlow Addons Layers: Temporal Convolutional Network"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "<table class=\"tfo-notebook-buttons\" align=\"left\">\n",
+    "  <td>\n",
+    "    <a target=\"_blank\" href=\"https://www.tensorflow.org/addons/tutorials/temporal_convolutional_network\"><img src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" />View on TensorFlow.org</a>\n",
+    "  </td>\n",
+    "  <td>\n",
+    "    <a target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/addons/blob/master/docs/tutorials/temporal_convolutional_network.ipynb\"><img src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" />Run in Google Colab</a>\n",
+    "  </td>\n",
+    "  <td>\n",
+    "    <a target=\"_blank\" href=\"https://github.com/tensorflow/addons/blob/master/docs/tutorials/temporal_convolutional_network.ipynb\"><img src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" />View source on GitHub</a>\n",
+    "  </td>\n",
+    "      <td>\n",
+    "    <a href=\"https://storage.googleapis.com/tensorflow_docs/addons/docs/tutorials/temporal_convolutional_network.ipynb\"><img src=\"https://www.tensorflow.org/images/download_logo_32px.png\" />Download notebook</a>\n",
+    "  </td>\n",
+    "</table>"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Overview\n",
+    "This notebook will demonstrate how to use Temporal Convolutional Network in TensorFlow Addons."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Setup"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "try:\n",
+    "    # %tensorflow_version only exists in Colab.\n",
+    "    %tensorflow_version 2.x\n",
+    "except Exception:\n",
+    "    pass"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import numpy as np\n",
+    "import tensorflow as tf\n",
+    "import tensorflow_addons as tfa\n",
+    "\n",
+    "import tensorflow.keras as keras\n",
+    "from tensorflow.keras.datasets import imdb\n",
+    "from tensorflow.keras.models import Model\n",
+    "from tensorflow.keras import Input\n",
+    "from tensorflow.keras.layers import Dense, Dropout, Embedding\n",
+    "from tensorflow.keras.preprocessing import sequence"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Global Configurations"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "max_features = 20000\n",
+    "# cut texts after this number of words\n",
+    "# (among top max_features most common words)\n",
+    "maxlen = 100\n",
+    "batch_size = 32"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Import IMDB data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# the data, split between train and test sets\n",
+    "(x_train, y_train), (x_test, y_test) = imdb.load_data(num_words=max_features)\n",
+    "\n",
+    "# pad the training data\n",
+    "x_train = sequence.pad_sequences(x_train, maxlen=maxlen)\n",
+    "x_test = sequence.pad_sequences(x_test, maxlen=maxlen)\n",
+    "y_train = np.array(y_train)\n",
+    "y_test = np.array(y_test)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Build Simple TCN Model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# build the model using the Sequential API\n",
+    "i = Input(shape=(maxlen,))\n",
+    "x = Embedding(max_features, 128)(i)\n",
+    "x = tfa.layers.TCN()(x)\n",
+    "x = Dropout(0.2)(x)\n",
+    "x = Dense(1, activation='sigmoid')(x)\n",
+    "\n",
+    "model = Model(inputs=[i], outputs=[x])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Summary of the model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Model: \"model\"\n",
+      "_________________________________________________________________\n",
+      "Layer (type)                 Output Shape              Param #   \n",
+      "=================================================================\n",
+      "input_1 (InputLayer)         [(None, 100)]             0         \n",
+      "_________________________________________________________________\n",
+      "embedding (Embedding)        (None, 100, 128)          2560000   \n",
+      "_________________________________________________________________\n",
+      "tcn_1 (TCN)                  (None, 64)                148800    \n",
+      "_________________________________________________________________\n",
+      "dropout (Dropout)            (None, 64)                0         \n",
+      "_________________________________________________________________\n",
+      "dense (Dense)                (None, 1)                 65        \n",
+      "=================================================================\n",
+      "Total params: 2,708,865\n",
+      "Trainable params: 2,708,865\n",
+      "Non-trainable params: 0\n",
+      "_________________________________________________________________\n"
+     ]
+    }
+   ],
+   "source": [
+    "model.summary()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Compute the model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "model.compile(optimizer='adam',\n",
+    "              loss = 'binary_crossentropy',\n",
+    "              metrics=['accuracy'])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Fit the model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Train on 25000 samples, validate on 25000 samples\n",
+      "Epoch 1/10\n",
+      "25000/25000 [==============================] - 83s 3ms/sample - loss: 0.5829 - accuracy: 0.6682 - val_loss: 0.3685 - val_accuracy: 0.8346\n",
+      "Epoch 2/10\n",
+      "25000/25000 [==============================] - 82s 3ms/sample - loss: 0.2817 - accuracy: 0.8841 - val_loss: 0.5567 - val_accuracy: 0.7630\n",
+      "Epoch 3/10\n",
+      "25000/25000 [==============================] - 84s 3ms/sample - loss: 0.1397 - accuracy: 0.9483 - val_loss: 0.4691 - val_accuracy: 0.8311\n",
+      "Epoch 4/10\n",
+      "25000/25000 [==============================] - 84s 3ms/sample - loss: 0.0666 - accuracy: 0.9761 - val_loss: 0.6161 - val_accuracy: 0.8248\n",
+      "Epoch 5/10\n",
+      " 2720/25000 [==>...........................] - ETA: 59s - loss: 0.0256 - accuracy: 0.9922"
+     ]
+    },
+    {
+     "ename": "KeyboardInterrupt",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
+      "\u001b[0;32m<ipython-input-10-3f69760dd88c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      2\u001b[0m           \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m           \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m           validation_data=(x_test, y_test))\n\u001b[0m",
+      "\u001b[0;32m~/addons/env/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_freq, max_queue_size, workers, use_multiprocessing, **kwargs)\u001b[0m\n\u001b[1;32m    791\u001b[0m         \u001b[0mmax_queue_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmax_queue_size\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    792\u001b[0m         \u001b[0mworkers\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mworkers\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 793\u001b[0;31m         use_multiprocessing=use_multiprocessing)\n\u001b[0m\u001b[1;32m    794\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    795\u001b[0m   def evaluate(self,\n",
+      "\u001b[0;32m~/addons/env/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/training_v2.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, model, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_freq, max_queue_size, workers, use_multiprocessing, **kwargs)\u001b[0m\n\u001b[1;32m    340\u001b[0m                 \u001b[0mmode\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mModeKeys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTRAIN\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    341\u001b[0m                 \u001b[0mtraining_context\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtraining_context\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 342\u001b[0;31m                 total_epochs=epochs)\n\u001b[0m\u001b[1;32m    343\u001b[0m             \u001b[0mcbks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmake_logs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepoch_logs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtraining_result\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mModeKeys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTRAIN\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    344\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/addons/env/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/training_v2.py\u001b[0m in \u001b[0;36mrun_one_epoch\u001b[0;34m(model, iterator, execution_function, dataset_size, batch_size, strategy, steps_per_epoch, num_samples, mode, training_context, total_epochs)\u001b[0m\n\u001b[1;32m    126\u001b[0m         step=step, mode=mode, size=current_batch_size) as batch_logs:\n\u001b[1;32m    127\u001b[0m       \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 128\u001b[0;31m         \u001b[0mbatch_outs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mexecution_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    129\u001b[0m       \u001b[0;32mexcept\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mStopIteration\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOutOfRangeError\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    130\u001b[0m         \u001b[0;31m# TODO(kaftan): File bug about tf function and errors.OutOfRangeError?\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/addons/env/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/training_v2_utils.py\u001b[0m in \u001b[0;36mexecution_function\u001b[0;34m(input_fn)\u001b[0m\n\u001b[1;32m     96\u001b[0m     \u001b[0;31m# `numpy` translates Tensors to values in Eager mode.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     97\u001b[0m     return nest.map_structure(_non_none_constant_value,\n\u001b[0;32m---> 98\u001b[0;31m                               distributed_function(input_fn))\n\u001b[0m\u001b[1;32m     99\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    100\u001b[0m   \u001b[0;32mreturn\u001b[0m \u001b[0mexecution_function\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/addons/env/lib/python3.6/site-packages/tensorflow_core/python/eager/def_function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    566\u001b[0m         \u001b[0mxla_context\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mExit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    567\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 568\u001b[0;31m       \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    569\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    570\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mtracing_count\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_tracing_count\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/addons/env/lib/python3.6/site-packages/tensorflow_core/python/eager/def_function.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    597\u001b[0m       \u001b[0;31m# In this case we have created variables on the first call, so we run the\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    598\u001b[0m       \u001b[0;31m# defunned version which is guaranteed to never create variables.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 599\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateless_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=not-callable\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    600\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateful_fn\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    601\u001b[0m       \u001b[0;31m# Release the lock early so that multiple threads can perform the call\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/addons/env/lib/python3.6/site-packages/tensorflow_core/python/eager/function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   2339\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_lock\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2340\u001b[0m       \u001b[0mgraph_function\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_maybe_define_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2341\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mgraph_function\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_filtered_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2342\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2343\u001b[0m   \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/addons/env/lib/python3.6/site-packages/tensorflow_core/python/eager/function.py\u001b[0m in \u001b[0;36m_filtered_call\u001b[0;34m(self, args, kwargs)\u001b[0m\n\u001b[1;32m   1587\u001b[0m          if isinstance(t, (ops.Tensor,\n\u001b[1;32m   1588\u001b[0m                            resource_variable_ops.BaseResourceVariable))),\n\u001b[0;32m-> 1589\u001b[0;31m         self.captured_inputs)\n\u001b[0m\u001b[1;32m   1590\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1591\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_call_flat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcaptured_inputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcancellation_manager\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/addons/env/lib/python3.6/site-packages/tensorflow_core/python/eager/function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[0;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[1;32m   1668\u001b[0m       \u001b[0;31m# No tape is watching; skip to running the function.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1669\u001b[0m       return self._build_call_outputs(self._inference_function.call(\n\u001b[0;32m-> 1670\u001b[0;31m           ctx, args, cancellation_manager=cancellation_manager))\n\u001b[0m\u001b[1;32m   1671\u001b[0m     forward_backward = self._select_forward_and_backward_functions(\n\u001b[1;32m   1672\u001b[0m         \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/addons/env/lib/python3.6/site-packages/tensorflow_core/python/eager/function.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[1;32m    521\u001b[0m               \u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    522\u001b[0m               \u001b[0mattrs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"executor_type\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexecutor_type\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"config_proto\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconfig\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 523\u001b[0;31m               ctx=ctx)\n\u001b[0m\u001b[1;32m    524\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    525\u001b[0m           outputs = execute.execute_with_cancellation(\n",
+      "\u001b[0;32m~/addons/env/lib/python3.6/site-packages/tensorflow_core/python/eager/execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m     59\u001b[0m     tensors = pywrap_tensorflow.TFE_Py_Execute(ctx._handle, device_name,\n\u001b[1;32m     60\u001b[0m                                                \u001b[0mop_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattrs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 61\u001b[0;31m                                                num_outputs)\n\u001b[0m\u001b[1;32m     62\u001b[0m   \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     63\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
+     ]
+    }
+   ],
+   "source": [
+    "model.fit(x_train, y_train,\n",
+    "          batch_size=batch_size,\n",
+    "          epochs=10,\n",
+    "          validation_data=(x_test, y_test))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/tensorflow_addons/layers/tcn.py b/tensorflow_addons/layers/tcn.py
index a4e2525485..bc9b96dec8 100644
--- a/tensorflow_addons/layers/tcn.py
+++ b/tensorflow_addons/layers/tcn.py
@@ -24,7 +24,7 @@
 
 
 @tf.keras.utils.register_keras_serializable(package='Addons')
-class ResidualBlock(tf.keras.Layer):
+class ResidualBlock(tf.keras.layers.Layer):
     """Defines the residual block for the WaveNet TCN
 
         Arguments:
@@ -151,8 +151,8 @@ def build(self, input_shape):
 
     def call(self, inputs, training=None):
         """
-        Returns: A tuple where the first element is the residual model tensor, and the second
-                 is the skip connection tensor.
+        Returns: A tuple where the first element is the residual model tensor,
+                 and the second is the skip connection tensor.
         """
         x = inputs
         for layer in self.residual_layers:
@@ -168,16 +168,25 @@ def call(self, inputs, training=None):
     def compute_output_shape(self, input_shape):
         return [self.res_output_shape, self.res_output_shape]
 
-    # TODO(shunlin): fix this get_config after finish fixing the APIs
     def get_config(self):
         config = dict()
 
+        config['dilation_rate'] = self.dilation_rate
+        config['filters'] = self.filters
+        config['kernel_size'] = self.kernel_size
+        config['padding'] = self.padding
+        config['activation'] = self.activation
+        config['dropout_rate'] = self.dropout_rate
+        config['use_batch_norm'] = self.use_batch_norm
+        config['kernel_initializer'] = self.kernel_initializer
+        config['last_block'] = self.last_block
+
         base_config = super(ResidualBlock, self).get_config()
         return dict(list(base_config.items()) + list(config.items()))
 
 
 @tf.keras.utils.register_keras_serializable(package='Addons')
-class TCN(tf.keras.Layer):
+class TCN(tf.keras.layers.Layer):
     """Creates a TCN layer.
 
         Input shape:
@@ -235,6 +244,7 @@ def __init__(self,
         self.stacks = stacks
         self.kernel_size = kernel_size
         self.filters = filters
+        self.dilations = dilations
         self.activation = activation
         self.padding = padding
         self.kernel_initializer = kernel_initializer

From 011f2d5c7d990cb5b2f39c0fa0972dbc0dc3a085 Mon Sep 17 00:00:00 2001
From: Shun Lin <shunlin@google.com>
Date: Sun, 10 Nov 2019 21:53:36 -0800
Subject: [PATCH 09/20] update tutorial

---
 .../temporal_convolutional_network.ipynb      | 81 +++++++++----------
 1 file changed, 37 insertions(+), 44 deletions(-)

diff --git a/docs/tutorials/temporal_convolutional_network.ipynb b/docs/tutorials/temporal_convolutional_network.ipynb
index 659851f3dc..d6a2cc4c32 100644
--- a/docs/tutorials/temporal_convolutional_network.ipynb
+++ b/docs/tutorials/temporal_convolutional_network.ipynb
@@ -68,6 +68,15 @@
     "## Setup"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": 36,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from __future__ import absolute_import, division, print_function"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": 2,
@@ -108,7 +117,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 29,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -128,7 +137,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 30,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -151,11 +160,11 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 31,
    "metadata": {},
    "outputs": [],
    "source": [
-    "# build the model using the Sequential API\n",
+    "# build the model using functional API\n",
     "i = Input(shape=(maxlen,))\n",
     "x = Embedding(max_features, 128)(i)\n",
     "x = tfa.layers.TCN()(x)\n",
@@ -174,26 +183,26 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 32,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Model: \"model\"\n",
+      "Model: \"model_4\"\n",
       "_________________________________________________________________\n",
       "Layer (type)                 Output Shape              Param #   \n",
       "=================================================================\n",
-      "input_1 (InputLayer)         [(None, 100)]             0         \n",
+      "input_5 (InputLayer)         [(None, 100)]             0         \n",
       "_________________________________________________________________\n",
-      "embedding (Embedding)        (None, 100, 128)          2560000   \n",
+      "embedding_4 (Embedding)      (None, 100, 128)          2560000   \n",
       "_________________________________________________________________\n",
-      "tcn_1 (TCN)                  (None, 64)                148800    \n",
+      "tcn_5 (TCN)                  (None, 64)                148800    \n",
       "_________________________________________________________________\n",
-      "dropout (Dropout)            (None, 64)                0         \n",
+      "dropout_4 (Dropout)          (None, 64)                0         \n",
       "_________________________________________________________________\n",
-      "dense (Dense)                (None, 1)                 65        \n",
+      "dense_4 (Dense)              (None, 1)                 65        \n",
       "=================================================================\n",
       "Total params: 2,708,865\n",
       "Trainable params: 2,708,865\n",
@@ -215,7 +224,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 33,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -233,7 +242,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 35,
    "metadata": {
     "scrolled": true
    },
@@ -243,45 +252,29 @@
      "output_type": "stream",
      "text": [
       "Train on 25000 samples, validate on 25000 samples\n",
-      "Epoch 1/10\n",
-      "25000/25000 [==============================] - 83s 3ms/sample - loss: 0.5829 - accuracy: 0.6682 - val_loss: 0.3685 - val_accuracy: 0.8346\n",
-      "Epoch 2/10\n",
-      "25000/25000 [==============================] - 82s 3ms/sample - loss: 0.2817 - accuracy: 0.8841 - val_loss: 0.5567 - val_accuracy: 0.7630\n",
-      "Epoch 3/10\n",
-      "25000/25000 [==============================] - 84s 3ms/sample - loss: 0.1397 - accuracy: 0.9483 - val_loss: 0.4691 - val_accuracy: 0.8311\n",
-      "Epoch 4/10\n",
-      "25000/25000 [==============================] - 84s 3ms/sample - loss: 0.0666 - accuracy: 0.9761 - val_loss: 0.6161 - val_accuracy: 0.8248\n",
-      "Epoch 5/10\n",
-      " 2720/25000 [==>...........................] - ETA: 59s - loss: 0.0256 - accuracy: 0.9922"
+      "Epoch 1/3\n",
+      "25000/25000 [==============================] - 90s 4ms/sample - loss: 0.2968 - accuracy: 0.8742 - val_loss: 0.4443 - val_accuracy: 0.7968\n",
+      "Epoch 2/3\n",
+      "25000/25000 [==============================] - 90s 4ms/sample - loss: 0.1521 - accuracy: 0.9418 - val_loss: 0.5506 - val_accuracy: 0.8182\n",
+      "Epoch 3/3\n",
+      "25000/25000 [==============================] - 90s 4ms/sample - loss: 0.0779 - accuracy: 0.9721 - val_loss: 0.5611 - val_accuracy: 0.8075\n"
      ]
     },
     {
-     "ename": "KeyboardInterrupt",
-     "evalue": "",
-     "output_type": "error",
-     "traceback": [
-      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
-      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
-      "\u001b[0;32m<ipython-input-10-3f69760dd88c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      2\u001b[0m           \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m           \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m           validation_data=(x_test, y_test))\n\u001b[0m",
-      "\u001b[0;32m~/addons/env/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_freq, max_queue_size, workers, use_multiprocessing, **kwargs)\u001b[0m\n\u001b[1;32m    791\u001b[0m         \u001b[0mmax_queue_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmax_queue_size\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    792\u001b[0m         \u001b[0mworkers\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mworkers\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 793\u001b[0;31m         use_multiprocessing=use_multiprocessing)\n\u001b[0m\u001b[1;32m    794\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    795\u001b[0m   def evaluate(self,\n",
-      "\u001b[0;32m~/addons/env/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/training_v2.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, model, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_freq, max_queue_size, workers, use_multiprocessing, **kwargs)\u001b[0m\n\u001b[1;32m    340\u001b[0m                 \u001b[0mmode\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mModeKeys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTRAIN\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    341\u001b[0m                 \u001b[0mtraining_context\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtraining_context\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 342\u001b[0;31m                 total_epochs=epochs)\n\u001b[0m\u001b[1;32m    343\u001b[0m             \u001b[0mcbks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmake_logs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepoch_logs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtraining_result\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mModeKeys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTRAIN\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    344\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
-      "\u001b[0;32m~/addons/env/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/training_v2.py\u001b[0m in \u001b[0;36mrun_one_epoch\u001b[0;34m(model, iterator, execution_function, dataset_size, batch_size, strategy, steps_per_epoch, num_samples, mode, training_context, total_epochs)\u001b[0m\n\u001b[1;32m    126\u001b[0m         step=step, mode=mode, size=current_batch_size) as batch_logs:\n\u001b[1;32m    127\u001b[0m       \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 128\u001b[0;31m         \u001b[0mbatch_outs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mexecution_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    129\u001b[0m       \u001b[0;32mexcept\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mStopIteration\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOutOfRangeError\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    130\u001b[0m         \u001b[0;31m# TODO(kaftan): File bug about tf function and errors.OutOfRangeError?\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
-      "\u001b[0;32m~/addons/env/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/training_v2_utils.py\u001b[0m in \u001b[0;36mexecution_function\u001b[0;34m(input_fn)\u001b[0m\n\u001b[1;32m     96\u001b[0m     \u001b[0;31m# `numpy` translates Tensors to values in Eager mode.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     97\u001b[0m     return nest.map_structure(_non_none_constant_value,\n\u001b[0;32m---> 98\u001b[0;31m                               distributed_function(input_fn))\n\u001b[0m\u001b[1;32m     99\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    100\u001b[0m   \u001b[0;32mreturn\u001b[0m \u001b[0mexecution_function\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
-      "\u001b[0;32m~/addons/env/lib/python3.6/site-packages/tensorflow_core/python/eager/def_function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    566\u001b[0m         \u001b[0mxla_context\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mExit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    567\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 568\u001b[0;31m       \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    569\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    570\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mtracing_count\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_tracing_count\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
-      "\u001b[0;32m~/addons/env/lib/python3.6/site-packages/tensorflow_core/python/eager/def_function.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    597\u001b[0m       \u001b[0;31m# In this case we have created variables on the first call, so we run the\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    598\u001b[0m       \u001b[0;31m# defunned version which is guaranteed to never create variables.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 599\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateless_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=not-callable\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    600\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateful_fn\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    601\u001b[0m       \u001b[0;31m# Release the lock early so that multiple threads can perform the call\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
-      "\u001b[0;32m~/addons/env/lib/python3.6/site-packages/tensorflow_core/python/eager/function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   2339\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_lock\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2340\u001b[0m       \u001b[0mgraph_function\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_maybe_define_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2341\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mgraph_function\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_filtered_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2342\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2343\u001b[0m   \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
-      "\u001b[0;32m~/addons/env/lib/python3.6/site-packages/tensorflow_core/python/eager/function.py\u001b[0m in \u001b[0;36m_filtered_call\u001b[0;34m(self, args, kwargs)\u001b[0m\n\u001b[1;32m   1587\u001b[0m          if isinstance(t, (ops.Tensor,\n\u001b[1;32m   1588\u001b[0m                            resource_variable_ops.BaseResourceVariable))),\n\u001b[0;32m-> 1589\u001b[0;31m         self.captured_inputs)\n\u001b[0m\u001b[1;32m   1590\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1591\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_call_flat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcaptured_inputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcancellation_manager\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
-      "\u001b[0;32m~/addons/env/lib/python3.6/site-packages/tensorflow_core/python/eager/function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[0;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[1;32m   1668\u001b[0m       \u001b[0;31m# No tape is watching; skip to running the function.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1669\u001b[0m       return self._build_call_outputs(self._inference_function.call(\n\u001b[0;32m-> 1670\u001b[0;31m           ctx, args, cancellation_manager=cancellation_manager))\n\u001b[0m\u001b[1;32m   1671\u001b[0m     forward_backward = self._select_forward_and_backward_functions(\n\u001b[1;32m   1672\u001b[0m         \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
-      "\u001b[0;32m~/addons/env/lib/python3.6/site-packages/tensorflow_core/python/eager/function.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[1;32m    521\u001b[0m               \u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    522\u001b[0m               \u001b[0mattrs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"executor_type\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexecutor_type\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"config_proto\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconfig\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 523\u001b[0;31m               ctx=ctx)\n\u001b[0m\u001b[1;32m    524\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    525\u001b[0m           outputs = execute.execute_with_cancellation(\n",
-      "\u001b[0;32m~/addons/env/lib/python3.6/site-packages/tensorflow_core/python/eager/execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m     59\u001b[0m     tensors = pywrap_tensorflow.TFE_Py_Execute(ctx._handle, device_name,\n\u001b[1;32m     60\u001b[0m                                                \u001b[0mop_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattrs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 61\u001b[0;31m                                                num_outputs)\n\u001b[0m\u001b[1;32m     62\u001b[0m   \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     63\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
-      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
-     ]
+     "data": {
+      "text/plain": [
+       "<tensorflow.python.keras.callbacks.History at 0x164290668>"
+      ]
+     },
+     "execution_count": 35,
+     "metadata": {},
+     "output_type": "execute_result"
     }
    ],
    "source": [
     "model.fit(x_train, y_train,\n",
     "          batch_size=batch_size,\n",
-    "          epochs=10,\n",
+    "          epochs=3,\n",
     "          validation_data=(x_test, y_test))"
    ]
   },

From 700d0112dbbf3b2a02d7731c19b6442444ce515b Mon Sep 17 00:00:00 2001
From: Shun Lin <shunlin@google.com>
Date: Mon, 11 Nov 2019 20:58:13 -0800
Subject: [PATCH 10/20] added a simple base test, update tcn docs, update
 example codelab

---
 .../temporal_convolutional_network.ipynb      |  42 ++---
 tensorflow_addons/layers/tcn.py               | 161 +++++++++---------
 tensorflow_addons/layers/tcn_test.py          |  11 +-
 3 files changed, 106 insertions(+), 108 deletions(-)

diff --git a/docs/tutorials/temporal_convolutional_network.ipynb b/docs/tutorials/temporal_convolutional_network.ipynb
index d6a2cc4c32..31c7df6cf1 100644
--- a/docs/tutorials/temporal_convolutional_network.ipynb
+++ b/docs/tutorials/temporal_convolutional_network.ipynb
@@ -9,7 +9,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 22,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -70,7 +70,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 36,
+   "execution_count": 23,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -79,7 +79,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 24,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -92,7 +92,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 25,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -117,7 +117,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 29,
+   "execution_count": 26,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -137,7 +137,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 30,
+   "execution_count": 27,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -160,7 +160,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 31,
+   "execution_count": 28,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -183,26 +183,26 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 32,
+   "execution_count": 29,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Model: \"model_4\"\n",
+      "Model: \"model_1\"\n",
       "_________________________________________________________________\n",
       "Layer (type)                 Output Shape              Param #   \n",
       "=================================================================\n",
-      "input_5 (InputLayer)         [(None, 100)]             0         \n",
+      "input_2 (InputLayer)         [(None, 100)]             0         \n",
       "_________________________________________________________________\n",
-      "embedding_4 (Embedding)      (None, 100, 128)          2560000   \n",
+      "embedding_1 (Embedding)      (None, 100, 128)          2560000   \n",
       "_________________________________________________________________\n",
-      "tcn_5 (TCN)                  (None, 64)                148800    \n",
+      "tcn_2 (TCN)                  (None, 64)                148800    \n",
       "_________________________________________________________________\n",
-      "dropout_4 (Dropout)          (None, 64)                0         \n",
+      "dropout_2 (Dropout)          (None, 64)                0         \n",
       "_________________________________________________________________\n",
-      "dense_4 (Dense)              (None, 1)                 65        \n",
+      "dense_1 (Dense)              (None, 1)                 65        \n",
       "=================================================================\n",
       "Total params: 2,708,865\n",
       "Trainable params: 2,708,865\n",
@@ -224,7 +224,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 33,
+   "execution_count": 30,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -242,7 +242,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 35,
+   "execution_count": 31,
    "metadata": {
     "scrolled": true
    },
@@ -253,20 +253,20 @@
      "text": [
       "Train on 25000 samples, validate on 25000 samples\n",
       "Epoch 1/3\n",
-      "25000/25000 [==============================] - 90s 4ms/sample - loss: 0.2968 - accuracy: 0.8742 - val_loss: 0.4443 - val_accuracy: 0.7968\n",
+      "25000/25000 [==============================] - 89s 4ms/sample - loss: 0.5478 - accuracy: 0.7011 - val_loss: 0.3653 - val_accuracy: 0.8366\n",
       "Epoch 2/3\n",
-      "25000/25000 [==============================] - 90s 4ms/sample - loss: 0.1521 - accuracy: 0.9418 - val_loss: 0.5506 - val_accuracy: 0.8182\n",
+      "25000/25000 [==============================] - 82s 3ms/sample - loss: 0.2755 - accuracy: 0.8866 - val_loss: 0.3609 - val_accuracy: 0.8436\n",
       "Epoch 3/3\n",
-      "25000/25000 [==============================] - 90s 4ms/sample - loss: 0.0779 - accuracy: 0.9721 - val_loss: 0.5611 - val_accuracy: 0.8075\n"
+      "25000/25000 [==============================] - 83s 3ms/sample - loss: 0.1299 - accuracy: 0.9513 - val_loss: 0.4395 - val_accuracy: 0.8242\n"
      ]
     },
     {
      "data": {
       "text/plain": [
-       "<tensorflow.python.keras.callbacks.History at 0x164290668>"
+       "<tensorflow.python.keras.callbacks.History at 0x10bc456a0>"
       ]
      },
-     "execution_count": 35,
+     "execution_count": 31,
      "metadata": {},
      "output_type": "execute_result"
     }
diff --git a/tensorflow_addons/layers/tcn.py b/tensorflow_addons/layers/tcn.py
index bc9b96dec8..1f4b10115e 100644
--- a/tensorflow_addons/layers/tcn.py
+++ b/tensorflow_addons/layers/tcn.py
@@ -17,40 +17,38 @@
 from __future__ import absolute_import, division, print_function
 
 import tensorflow as tf
-import tensorflow.keras.backend as K
 from tensorflow.keras.layers import (Activation, BatchNormalization, Conv1D,
-                                     Dense, Lambda, SpatialDropout1D, add)
-from tensorflow_addons.utils import keras_utils
+                                     Lambda, SpatialDropout1D, add)
 
 
 @tf.keras.utils.register_keras_serializable(package='Addons')
 class ResidualBlock(tf.keras.layers.Layer):
-    """Defines the residual block for the WaveNet TCN
-
-        Arguments:
-            dilation_rate (int): The dilation power of 2 we are using
-                for this residual block. Defaults to 1.
-            filters (int): The number of convolutional
-                filters to use in this block. Defaults to 64.
-            kernel_size (int): The size of the convolutional kernel. Defaults
-                to 2.
-            padding (String): The padding used in the convolutional layers,
-                'same' or 'causal'. Defaults to 'same'
-            activation (String): The final activation used
-                in o = Activation(x + F(x)). Defaults to 'relu'
-            dropout_rate (Float): Float between 0 and 1. Fraction
-                of the input units to drop. Defaults to 0.0.
-            kernel_initializer (String): Initializer for the kernel weights
-                matrix (Conv1D). Defaults to 'he_normal'
-            use_batch_norm (bool): Whether to use batch normalization in the
-                residual layers or not. Defaults to False.
-            last_block (bool): Whether or not this block is the last residual
-                block of the network. Defaults to False.
-            kwargs: Any initializers for Layer class.
-
-        Returns:
-            A Residual Blcok.
-        """
+    """Defines the residual block for the WaveNet TCN.
+
+    Arguments:
+        dilation_rate (int): The dilation power of 2 we are using
+            for this residual block. Defaults to 1.
+        filters (int): The number of convolutional
+            filters to use in this block. Defaults to 64.
+        kernel_size (int): The size of the convolutional kernel. Defaults
+            to 2.
+        padding (String): The padding used in the convolutional layers,
+            'same' or 'causal'. Defaults to 'same'
+        activation (String): The final activation used
+            in o = Activation(x + F(x)). Defaults to 'relu'
+        dropout_rate (Float): Float between 0 and 1. Fraction
+            of the input units to drop. Defaults to 0.0.
+        kernel_initializer (String): Initializer for the kernel weights
+            matrix (Conv1D). Defaults to 'he_normal'
+        use_batch_norm (bool): Whether to use batch normalization in the
+            residual layers or not. Defaults to False.
+        last_block (bool): Whether or not this block is the last residual
+            block of the network. Defaults to False.
+        kwargs: Any initializers for Layer class.
+
+    Returns:
+        A Residual Blcok.
+    """
 
     def __init__(self,
                  dilation_rate=1,
@@ -77,8 +75,9 @@ def __init__(self,
         super(ResidualBlock, self).__init__(**kwargs)
 
     def _add_and_activate_layer(self, layer):
-        """Helper function for building layer
-        Args:
+        """Helper function for building layer.
+
+        Arguments:
             layer: Appends layer to internal layer list and builds it based on
                     the current output shape of ResidualBlock.
                     Updates current output shape.
@@ -91,14 +90,14 @@ def _add_and_activate_layer(self, layer):
     def build(self, input_shape):
 
         # name scope used to make sure weights get unique names
-        with K.name_scope(self.name):
+        with tf.name_scope(self.name):
             self.residual_layers = list()
             self.res_output_shape = input_shape
 
             for k in range(2):
                 name = 'conv1D_{}'.format(k)
                 # name scope used to make sure weights get unique names
-                with K.name_scope(name):
+                with tf.name_scope(name):
                     self._add_and_activate_layer(
                         Conv1D(
                             filters=self.filters,
@@ -109,8 +108,6 @@ def build(self, input_shape):
                             kernel_initializer=self.kernel_initializer))
 
                 if self.use_batch_norm:
-                    # TODO should be WeightNorm here, but using batchNorm
-                    # instead
                     self._add_and_activate_layer(BatchNormalization())
 
                 self._add_and_activate_layer(Activation('relu'))
@@ -120,7 +117,7 @@ def build(self, input_shape):
             if not self.last_block:
                 # 1x1 conv to match the shapes (channel dimension).
                 name = 'conv1D_{}'.format(k + 1)
-                with K.name_scope(name):
+                with tf.name_scope(name):
                     # make and build this layer separately because it directly
                     # uses input_shape
                     self.shape_match_conv = Conv1D(
@@ -138,15 +135,12 @@ def build(self, input_shape):
                 input_shape)
 
             self.final_activation = Activation(self.activation)
-            self.final_activation.build(
-                self.res_output_shape)  # probably isn't necessary
 
             # this is done to force keras to add the layers in the list to
             # self._layers
             for layer in self.residual_layers:
                 self.__setattr__(layer.name, layer)
 
-            # done to make sure self.built is set True
             super(ResidualBlock, self).build(input_shape)
 
     def call(self, inputs, training=None):
@@ -189,40 +183,40 @@ def get_config(self):
 class TCN(tf.keras.layers.Layer):
     """Creates a TCN layer.
 
-        Input shape:
-            A tensor of shape (batch_size, timesteps, input_dim).
-
-        Arguments:
-            filters: The number of filters to use in the convolutional layers.
-                Defaults to 64.
-            kernel_size: The size of the kernel to use in each
-                convolutional layer. Defaults to 2.
-            dilations: The array-like input of the dilations.
-                Defaults to [1,2,4,8,16,32,64]
-            stacks : The number of stacks of residual blocks to use. Defaults
-                to 1.
-            padding: The padding to use in the convolutional layers,
-                'causal' or 'same'. Defaults to 'causal'.
-            use_skip_connections: Boolean. If we want to add skip
-                connections from input to each residual block.
-                Defaults to True.
-            return_sequences: Boolean. Whether to return the last
-                output in the output sequence, or the full sequence.
-                Defaults to False.
-            activation: The activation used in the residual
-                blocks o = Activation(x + F(x)). Defaults to 'linear'
-            dropout_rate: Float between 0 and 1. Fraction of the input
-                units to drop. Defaults to 0.0.
-            kernel_initializer: Initializer for the kernel weights
-                matrix (Conv1D). Defaulst to 'he_normal'
-            use_batch_norm: Whether to use batch normalization in the
-                residual layers or not. Defaulst to False.
-            kwargs: Any other arguments for configuring parent class Layer.
-                For example "name=str", Name of the model.
-                Use unique names when using multiple TCN.
-        Returns:
-            A TCN layer.
-        """
+    Input shape:
+        A tensor of shape (batch_size, timesteps, input_dim).
+
+    Arguments:
+        filters: The number of filters to use in the convolutional layers.
+            Defaults to 64.
+        kernel_size: The size of the kernel to use in each
+            convolutional layer. Defaults to 2.
+        dilations: The array-like input of the dilations.
+            Defaults to [1,2,4,8,16,32,64]
+        stacks : The number of stacks of residual blocks to use. Defaults
+            to 1.
+        padding: The padding to use in the convolutional layers,
+            'causal' or 'same'. Defaults to 'causal'.
+        use_skip_connections: Boolean. If we want to add skip
+            connections from input to each residual block.
+            Defaults to True.
+        return_sequences: Boolean. Whether to return the last
+            output in the output sequence, or the full sequence.
+            Defaults to False.
+        activation: The activation used in the residual
+            blocks o = Activation(x + F(x)). Defaults to 'linear'
+        dropout_rate: Float between 0 and 1. Fraction of the input
+            units to drop. Defaults to 0.0.
+        kernel_initializer: Initializer for the kernel weights
+            matrix (Conv1D). Defaulst to 'he_normal'
+        use_batch_norm: Whether to use batch normalization in the
+            residual layers or not. Defaulst to False.
+        kwargs: Any other arguments for configuring parent class Layer.
+            For example "name=str", Name of the model.
+            Use unique names when using multiple TCN.
+    Returns:
+        A TCN layer.
+    """
 
     def __init__(self,
                  filters=64,
@@ -254,16 +248,18 @@ def __init__(self,
         validate_paddings = ['causal', 'same']
         if padding not in validate_paddings:
             raise ValueError(
-                "Only 'causal' or 'same' padding are compatible for this layer")
+                "Only 'causal' or 'same' padding are compatible for this layer"
+            )
 
         # initialize parent class
         super(TCN, self).__init__(**kwargs)
 
     def build(self, input_shape):
-        self.main_conv1D = Conv1D(filters=self.filters,
-                                  kernel_size=1,
-                                  padding=self.padding,
-                                  kernel_initializer=self.kernel_initializer)
+        self.main_conv1D = Conv1D(
+            filters=self.filters,
+            kernel_size=1,
+            padding=self.padding,
+            kernel_initializer=self.kernel_initializer)
         self.main_conv1D.build(input_shape)
 
         # member to hold current output shape of the layer for building
@@ -289,15 +285,14 @@ def build(self, input_shape):
                         dropout_rate=self.dropout_rate,
                         use_batch_norm=self.use_batch_norm,
                         kernel_initializer=self.kernel_initializer,
-                        last_block=len(
-                            self.residual_blocks) +
+                        last_block=len(self.residual_blocks) +
                         1 == total_num_blocks,
                         name='residual_block_{}'.format(
-                            len(
-                                self.residual_blocks))))
+                            len(self.residual_blocks))))
                 # build newest residual block
                 self.residual_blocks[-1].build(self.build_output_shape)
-                self.build_output_shape = self.residual_blocks[-1].res_output_shape
+                self.build_output_shape = self.residual_blocks[
+                    -1].res_output_shape
 
         # this is done to force keras to add the layers in the list to
         # self._layers
diff --git a/tensorflow_addons/layers/tcn_test.py b/tensorflow_addons/layers/tcn_test.py
index 7a8764411e..8db6cbf973 100644
--- a/tensorflow_addons/layers/tcn_test.py
+++ b/tensorflow_addons/layers/tcn_test.py
@@ -18,14 +18,17 @@
 from __future__ import division
 from __future__ import print_function
 
-import numpy as np
 import tensorflow as tf
+
 from tensorflow_addons.utils import test_utils
 from tensorflow_addons.layers import TCN
 
 
 @test_utils.run_all_in_graph_and_eager_modes
 class TCNTest(tf.test.TestCase):
-    # TODO(shunlin): implment unit test
-    def test_simple(self):
-        pass
+    def test_tcn(self):
+        test_utils.layer_test(TCN, input_shape=(2, 4, 4, 3))
+
+
+if __name__ == "__main__":
+    tf.test.main()

From 88f9cafab9ab2942dac6633565fb578865efefdb Mon Sep 17 00:00:00 2001
From: Shun Lin <shunlin@google.com>
Date: Tue, 12 Nov 2019 00:23:35 -0800
Subject: [PATCH 11/20] fix tcn_test.py

---
 tensorflow_addons/layers/tcn_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow_addons/layers/tcn_test.py b/tensorflow_addons/layers/tcn_test.py
index 8db6cbf973..8597f30560 100644
--- a/tensorflow_addons/layers/tcn_test.py
+++ b/tensorflow_addons/layers/tcn_test.py
@@ -27,7 +27,7 @@
 @test_utils.run_all_in_graph_and_eager_modes
 class TCNTest(tf.test.TestCase):
     def test_tcn(self):
-        test_utils.layer_test(TCN, input_shape=(2, 4, 4, 3))
+        test_utils.layer_test(TCN, input_shape=(2, 4, 4))
 
 
 if __name__ == "__main__":

From 1998ef00cd324af8a2688cba272812053c0af315 Mon Sep 17 00:00:00 2001
From: Shun Lin <shunlin@google.com>
Date: Sat, 16 Nov 2019 09:55:00 -0800
Subject: [PATCH 12/20] delete tensorflow.keras.layers imports to use
 tf.keras.layers directly

---
 tensorflow_addons/layers/tcn.py | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/tensorflow_addons/layers/tcn.py b/tensorflow_addons/layers/tcn.py
index 1f4b10115e..0dd6cde886 100644
--- a/tensorflow_addons/layers/tcn.py
+++ b/tensorflow_addons/layers/tcn.py
@@ -17,8 +17,6 @@
 from __future__ import absolute_import, division, print_function
 
 import tensorflow as tf
-from tensorflow.keras.layers import (Activation, BatchNormalization, Conv1D,
-                                     Lambda, SpatialDropout1D, add)
 
 
 @tf.keras.utils.register_keras_serializable(package='Addons')
@@ -99,7 +97,7 @@ def build(self, input_shape):
                 # name scope used to make sure weights get unique names
                 with tf.name_scope(name):
                     self._add_and_activate_layer(
-                        Conv1D(
+                        tf.keras.layers.Conv1D(
                             filters=self.filters,
                             kernel_size=self.kernel_size,
                             dilation_rate=self.dilation_rate,
@@ -108,11 +106,13 @@ def build(self, input_shape):
                             kernel_initializer=self.kernel_initializer))
 
                 if self.use_batch_norm:
-                    self._add_and_activate_layer(BatchNormalization())
+                    self._add_and_activate_layer(
+                        tf.keras.layers.BatchNormalization())
 
-                self._add_and_activate_layer(Activation('relu'))
                 self._add_and_activate_layer(
-                    SpatialDropout1D(rate=self.dropout_rate))
+                    tf.keras.layers.Activation('relu'))
+                self._add_and_activate_layer(
+                    tf.keras.layers.SpatialDropout1D(rate=self.dropout_rate))
 
             if not self.last_block:
                 # 1x1 conv to match the shapes (channel dimension).
@@ -120,7 +120,7 @@ def build(self, input_shape):
                 with tf.name_scope(name):
                     # make and build this layer separately because it directly
                     # uses input_shape
-                    self.shape_match_conv = Conv1D(
+                    self.shape_match_conv = tf.keras.layers.Conv1D(
                         filters=self.filters,
                         kernel_size=1,
                         padding='same',
@@ -128,7 +128,8 @@ def build(self, input_shape):
                         kernel_initializer=self.kernel_initializer)
 
             else:
-                self.shape_match_conv = Lambda(lambda x: x, name='identity')
+                self.shape_match_conv = tf.keras.layers.Lambda(
+                    lambda x: x, name='identity')
 
             self.shape_match_conv.build(input_shape)
             self.res_output_shape = self.shape_match_conv.compute_output_shape(
@@ -150,13 +151,13 @@ def call(self, inputs, training=None):
         """
         x = inputs
         for layer in self.residual_layers:
-            if isinstance(layer, SpatialDropout1D):
+            if isinstance(layer, tf.keras.layers.SpatialDropout1D):
                 x = layer(x, training=training)
             else:
                 x = layer(x)
 
         x2 = self.shape_match_conv(inputs)
-        res_x = add([x2, x])
+        res_x = tf.keras.layers.add([x2, x])
         return [self.final_activation(res_x), x]
 
     def compute_output_shape(self, input_shape):
@@ -251,11 +252,10 @@ def __init__(self,
                 "Only 'causal' or 'same' padding are compatible for this layer"
             )
 
-        # initialize parent class
         super(TCN, self).__init__(**kwargs)
 
     def build(self, input_shape):
-        self.main_conv1D = Conv1D(
+        self.main_conv1D = tf.keras.layers.Conv1D(
             filters=self.filters,
             kernel_size=1,
             padding=self.padding,
@@ -299,7 +299,7 @@ def build(self, input_shape):
         for layer in self.residual_blocks:
             self.__setattr__(layer.name, layer)
 
-        self.lambda_layer = Lambda(lambda tt: tt[:, -1, :])
+        self.lambda_layer = tf.keras.layers.Lambda(lambda tt: tt[:, -1, :])
         self.lambda_ouput_shape = self.lambda_layer.compute_output_shape(
             self.build_output_shape)
 
@@ -320,7 +320,7 @@ def call(self, inputs, training=None):
             skip_connections.append(skip_out)
 
         if self.use_skip_connections:
-            x = add(skip_connections)
+            x = tf.keras.layers.add(skip_connections)
         if not self.return_sequences:
             x = self.lambda_layer(x)
         return x

From 7736b5ac30fa9e7ec71898b24eebab6b48f416f6 Mon Sep 17 00:00:00 2001
From: Shun Lin <shunlin@google.com>
Date: Sun, 17 Nov 2019 16:19:07 -0800
Subject: [PATCH 13/20] update residual block's __init__ and build so that we
 initiailze layers in __init__

---
 tensorflow_addons/layers/tcn.py | 85 ++++++++++++++++-----------------
 1 file changed, 40 insertions(+), 45 deletions(-)

diff --git a/tensorflow_addons/layers/tcn.py b/tensorflow_addons/layers/tcn.py
index 0dd6cde886..28743e1800 100644
--- a/tensorflow_addons/layers/tcn.py
+++ b/tensorflow_addons/layers/tcn.py
@@ -72,54 +72,35 @@ def __init__(self,
 
         super(ResidualBlock, self).__init__(**kwargs)
 
-    def _add_and_activate_layer(self, layer):
-        """Helper function for building layer.
+        self.residual_layers = list()
 
-        Arguments:
-            layer: Appends layer to internal layer list and builds it based on
-                    the current output shape of ResidualBlock.
-                    Updates current output shape.
-        """
-        self.residual_layers.append(layer)
-        self.residual_layers[-1].build(self.res_output_shape)
-        self.res_output_shape = self.residual_layers[-1].compute_output_shape(
-            self.res_output_shape)
-
-    def build(self, input_shape):
-
-        # name scope used to make sure weights get unique names
         with tf.name_scope(self.name):
-            self.residual_layers = list()
-            self.res_output_shape = input_shape
-
             for k in range(2):
-                name = 'conv1D_{}'.format(k)
-                # name scope used to make sure weights get unique names
+                name = 'con1D_{}'.format(k)
+
                 with tf.name_scope(name):
-                    self._add_and_activate_layer(
-                        tf.keras.layers.Conv1D(
-                            filters=self.filters,
-                            kernel_size=self.kernel_size,
-                            dilation_rate=self.dilation_rate,
-                            padding=self.padding,
-                            name=name,
-                            kernel_initializer=self.kernel_initializer))
+                    conv_layer = tf.keras.layers.Conv1D(
+                        filters=self.filters,
+                        kernel_size=self.kernel_size,
+                        dilation_rate=self.dilation_rate,
+                        padding=self.padding,
+                        name=name,
+                        kernel_initializer=self.kernel_initializer)
+                    self.residual_layers.append(conv_layer)
 
                 if self.use_batch_norm:
-                    self._add_and_activate_layer(
-                        tf.keras.layers.BatchNormalization())
+                    batch_norm_layer = tf.keras.layers.BatchNormalization()
+                    self.residual_layers.append(batch_norm_layer)
 
-                self._add_and_activate_layer(
-                    tf.keras.layers.Activation('relu'))
-                self._add_and_activate_layer(
-                    tf.keras.layers.SpatialDropout1D(rate=self.dropout_rate))
+                self.residual_layers.append(tf.keras.layers.Activation('relu'))
+                self.residual_layers.append(
+                    tf.keras.layers.SpatialDropout1D(
+                        rate=self.dropout_rate))
 
             if not self.last_block:
                 # 1x1 conv to match the shapes (channel dimension).
                 name = 'conv1D_{}'.format(k + 1)
                 with tf.name_scope(name):
-                    # make and build this layer separately because it directly
-                    # uses input_shape
                     self.shape_match_conv = tf.keras.layers.Conv1D(
                         filters=self.filters,
                         kernel_size=1,
@@ -131,18 +112,31 @@ def build(self, input_shape):
                 self.shape_match_conv = tf.keras.layers.Lambda(
                     lambda x: x, name='identity')
 
-            self.shape_match_conv.build(input_shape)
-            self.res_output_shape = self.shape_match_conv.compute_output_shape(
-                input_shape)
-
-            self.final_activation = Activation(self.activation)
+            self.final_activation = tf.keras.layers.Activation(self.activation)
 
             # this is done to force keras to add the layers in the list to
             # self._layers
             for layer in self.residual_layers:
                 self.__setattr__(layer.name, layer)
 
-            super(ResidualBlock, self).build(input_shape)
+    def build(self, input_shape):
+
+        # build residual layers
+        self.res_output_shape = input_shape
+        for layer in self.residual_layers:
+            layer.build(self.res_output_shape)
+            self.res_output_shape = layer.compute_output_shape(
+                self.res_output_shape)
+
+        # build shape matching convolutional layer
+        self.shape_match_conv.build(input_shape)
+        self.res_output_shape = self.shape_match_conv.compute_output_shape(
+            input_shape)
+
+        # build final activation layer
+        self.final_activation.build(self.res_output_shape)
+
+        super(ResidualBlock, self).build(input_shape)
 
     def call(self, inputs, training=None):
         """
@@ -252,14 +246,15 @@ def __init__(self,
                 "Only 'causal' or 'same' padding are compatible for this layer"
             )
 
-        super(TCN, self).__init__(**kwargs)
-
-    def build(self, input_shape):
         self.main_conv1D = tf.keras.layers.Conv1D(
             filters=self.filters,
             kernel_size=1,
             padding=self.padding,
             kernel_initializer=self.kernel_initializer)
+
+        super(TCN, self).__init__(**kwargs)
+
+    def build(self, input_shape):
         self.main_conv1D.build(input_shape)
 
         # member to hold current output shape of the layer for building

From 0e3568bb29cb5dbb9c743d7a43622aa71264c9b3 Mon Sep 17 00:00:00 2001
From: Shun Lin <shunlin@google.com>
Date: Sun, 17 Nov 2019 16:39:29 -0800
Subject: [PATCH 14/20] update tcn layer's __init__ and build method so that we
 initialize layers in __init__ instead of in build

---
 tensorflow_addons/layers/tcn.py | 31 ++++++++++++++-----------------
 1 file changed, 14 insertions(+), 17 deletions(-)

diff --git a/tensorflow_addons/layers/tcn.py b/tensorflow_addons/layers/tcn.py
index 28743e1800..b262530c5c 100644
--- a/tensorflow_addons/layers/tcn.py
+++ b/tensorflow_addons/layers/tcn.py
@@ -227,6 +227,8 @@ def __init__(self,
                  use_batch_norm=False,
                  **kwargs):
 
+        super(TCN, self).__init__(**kwargs)
+
         self.return_sequences = return_sequences
         self.dropout_rate = dropout_rate
         self.use_skip_connections = use_skip_connections
@@ -252,16 +254,6 @@ def __init__(self,
             padding=self.padding,
             kernel_initializer=self.kernel_initializer)
 
-        super(TCN, self).__init__(**kwargs)
-
-    def build(self, input_shape):
-        self.main_conv1D.build(input_shape)
-
-        # member to hold current output shape of the layer for building
-        # purposes
-        self.build_output_shape = self.main_conv1D.compute_output_shape(
-            input_shape)
-
         # list to hold all the member ResidualBlocks
         self.residual_blocks = list()
         total_num_blocks = self.stacks * len(self.dilations)
@@ -284,10 +276,6 @@ def build(self, input_shape):
                         1 == total_num_blocks,
                         name='residual_block_{}'.format(
                             len(self.residual_blocks))))
-                # build newest residual block
-                self.residual_blocks[-1].build(self.build_output_shape)
-                self.build_output_shape = self.residual_blocks[
-                    -1].res_output_shape
 
         # this is done to force keras to add the layers in the list to
         # self._layers
@@ -295,14 +283,23 @@ def build(self, input_shape):
             self.__setattr__(layer.name, layer)
 
         self.lambda_layer = tf.keras.layers.Lambda(lambda tt: tt[:, -1, :])
-        self.lambda_ouput_shape = self.lambda_layer.compute_output_shape(
-            self.build_output_shape)
+
+    def build(self, input_shape):
+        self.main_conv1D.build(input_shape)
+
+        self.build_output_shape = self.main_conv1D.compute_output_shape(
+            input_shape)
+
+        for residual_block in self.residual_blocks:
+            residual_block.build(self.build_output_shape)
+            self.build_output_shape = residual_block.res_output_shape
 
     def compute_output_shape(self, input_shape):
         if not self.built:
             self.build(input_shape)
         if not self.return_sequences:
-            return self.lambda_ouput_shape
+            return self.lambda_layer.compute_output_shape(
+                self.build_output_shape)
         else:
             return self.build_output_shape
 

From ea6050f06e04c3880aeeeac789f8731c5899d600 Mon Sep 17 00:00:00 2001
From: Shun Lin <shunlin@google.com>
Date: Mon, 18 Nov 2019 16:04:08 -0800
Subject: [PATCH 15/20] sort imports, BUILD, README.md, and some formatting on
 tcn.py

---
 tensorflow_addons/layers/BUILD       |  2 +-
 tensorflow_addons/layers/README.md   |  6 +++---
 tensorflow_addons/layers/__init__.py | 10 ++++------
 tensorflow_addons/layers/tcn.py      |  9 +++------
 4 files changed, 11 insertions(+), 16 deletions(-)

diff --git a/tensorflow_addons/layers/BUILD b/tensorflow_addons/layers/BUILD
index ac61cc1ef5..ba31922c5a 100644
--- a/tensorflow_addons/layers/BUILD
+++ b/tensorflow_addons/layers/BUILD
@@ -12,8 +12,8 @@ py_library(
         "optical_flow.py",
         "poincare.py",
         "sparsemax.py",
-        "wrappers.py",
         "tcn.py",
+        "wrappers.py",
     ],
     data = [
         "//tensorflow_addons/custom_ops/layers:_correlation_cost_ops.so",
diff --git a/tensorflow_addons/layers/README.md b/tensorflow_addons/layers/README.md
index c582fcea9b..04235d8248 100644
--- a/tensorflow_addons/layers/README.md
+++ b/tensorflow_addons/layers/README.md
@@ -9,8 +9,8 @@
 | opticalflow | @fsx950223 | fsx950223@gmail.com |
 | poincare | @rahulunair | rahulunair@gmail.com  |
 | sparsemax | @AndreasMadsen | amwwebdk+github@gmail.com |
-| wrappers | @seanpmorgan | seanmorgan@outlook.com |
 | tcn | @shun-lin | shunlin@google.com |
+| wrappers | @seanpmorgan | seanmorgan@outlook.com |
 
 ## Components
 | Submodule  | Layer |  Reference  |
@@ -21,9 +21,9 @@
 | normalizations | InstanceNormalization | https://arxiv.org/abs/1607.08022 |
 | opticalflow | CorrelationCost | https://arxiv.org/abs/1504.06852   |
 | poincare | PoincareNormalize | https://arxiv.org/abs/1705.08039    |
-| sparsemax| Sparsemax | https://arxiv.org/abs/1602.02068 |
-| wrappers | WeightNormalization | https://arxiv.org/abs/1602.07868 |
+| sparsemax | Sparsemax | https://arxiv.org/abs/1602.02068 |
 | tcn | TCN (Temporal Convolutional Network) | https://arxiv.org/pdf/1803.01271 |
+| wrappers | WeightNormalization | https://arxiv.org/abs/1602.07868 |
 
 ## Contribution Guidelines
 #### Standard API
diff --git a/tensorflow_addons/layers/__init__.py b/tensorflow_addons/layers/__init__.py
index b1fdc8cdb4..7b7b85ef76 100644
--- a/tensorflow_addons/layers/__init__.py
+++ b/tensorflow_addons/layers/__init__.py
@@ -14,16 +14,14 @@
 # ==============================================================================
 """Additional layers that conform to Keras API."""
 
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
 
 from tensorflow_addons.layers.gelu import GeLU
 from tensorflow_addons.layers.maxout import Maxout
-from tensorflow_addons.layers.normalizations import GroupNormalization
-from tensorflow_addons.layers.normalizations import InstanceNormalization
+from tensorflow_addons.layers.normalizations import (GroupNormalization,
+                                                     InstanceNormalization)
 from tensorflow_addons.layers.optical_flow import CorrelationCost
 from tensorflow_addons.layers.poincare import PoincareNormalize
 from tensorflow_addons.layers.sparsemax import Sparsemax
-from tensorflow_addons.layers.wrappers import WeightNormalization
 from tensorflow_addons.layers.tcn import TCN
+from tensorflow_addons.layers.wrappers import WeightNormalization
diff --git a/tensorflow_addons/layers/tcn.py b/tensorflow_addons/layers/tcn.py
index b262530c5c..181412e502 100644
--- a/tensorflow_addons/layers/tcn.py
+++ b/tensorflow_addons/layers/tcn.py
@@ -145,13 +145,10 @@ def call(self, inputs, training=None):
         """
         x = inputs
         for layer in self.residual_layers:
-            if isinstance(layer, tf.keras.layers.SpatialDropout1D):
-                x = layer(x, training=training)
-            else:
-                x = layer(x)
+            x = layer(x, training=training)
 
         x2 = self.shape_match_conv(inputs)
-        res_x = tf.keras.layers.add([x2, x])
+        res_x = x2 + x
         return [self.final_activation(res_x), x]
 
     def compute_output_shape(self, input_shape):
@@ -187,7 +184,7 @@ class TCN(tf.keras.layers.Layer):
         kernel_size: The size of the kernel to use in each
             convolutional layer. Defaults to 2.
         dilations: The array-like input of the dilations.
-            Defaults to [1,2,4,8,16,32,64]
+            Defaults to [1, 2, 4, 8, 16, 32, 64]
         stacks : The number of stacks of residual blocks to use. Defaults
             to 1.
         padding: The padding to use in the convolutional layers,

From ec0494d30f8f8ad97b995b161fe0efc255f6991e Mon Sep 17 00:00:00 2001
From: Shun Lin <shunlin@google.com>
Date: Mon, 18 Nov 2019 16:24:44 -0800
Subject: [PATCH 16/20] change one of the subtitle from compute the model to
 compile the model

---
 .../temporal_convolutional_network.ipynb      | 44 +++++++++----------
 1 file changed, 22 insertions(+), 22 deletions(-)

diff --git a/docs/tutorials/temporal_convolutional_network.ipynb b/docs/tutorials/temporal_convolutional_network.ipynb
index 31c7df6cf1..52a0c36c41 100644
--- a/docs/tutorials/temporal_convolutional_network.ipynb
+++ b/docs/tutorials/temporal_convolutional_network.ipynb
@@ -9,7 +9,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 22,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -70,7 +70,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 23,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -79,7 +79,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 24,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -92,7 +92,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 25,
+   "execution_count": 5,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -117,7 +117,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 26,
+   "execution_count": 6,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -137,7 +137,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 27,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -160,7 +160,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 28,
+   "execution_count": 8,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -183,26 +183,26 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 29,
+   "execution_count": 9,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Model: \"model_1\"\n",
+      "Model: \"model\"\n",
       "_________________________________________________________________\n",
       "Layer (type)                 Output Shape              Param #   \n",
       "=================================================================\n",
-      "input_2 (InputLayer)         [(None, 100)]             0         \n",
+      "input_1 (InputLayer)         [(None, 100)]             0         \n",
       "_________________________________________________________________\n",
-      "embedding_1 (Embedding)      (None, 100, 128)          2560000   \n",
+      "embedding (Embedding)        (None, 100, 128)          2560000   \n",
       "_________________________________________________________________\n",
-      "tcn_2 (TCN)                  (None, 64)                148800    \n",
+      "tcn (TCN)                    (None, 64)                148800    \n",
       "_________________________________________________________________\n",
-      "dropout_2 (Dropout)          (None, 64)                0         \n",
+      "dropout (Dropout)            (None, 64)                0         \n",
       "_________________________________________________________________\n",
-      "dense_1 (Dense)              (None, 1)                 65        \n",
+      "dense (Dense)                (None, 1)                 65        \n",
       "=================================================================\n",
       "Total params: 2,708,865\n",
       "Trainable params: 2,708,865\n",
@@ -219,12 +219,12 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "## Compute the model"
+    "## Compile the model"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 30,
+   "execution_count": 10,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -242,7 +242,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 31,
+   "execution_count": 11,
    "metadata": {
     "scrolled": true
    },
@@ -253,20 +253,20 @@
      "text": [
       "Train on 25000 samples, validate on 25000 samples\n",
       "Epoch 1/3\n",
-      "25000/25000 [==============================] - 89s 4ms/sample - loss: 0.5478 - accuracy: 0.7011 - val_loss: 0.3653 - val_accuracy: 0.8366\n",
+      "25000/25000 [==============================] - 100s 4ms/sample - loss: 0.5734 - accuracy: 0.6722 - val_loss: 0.4147 - val_accuracy: 0.8041\n",
       "Epoch 2/3\n",
-      "25000/25000 [==============================] - 82s 3ms/sample - loss: 0.2755 - accuracy: 0.8866 - val_loss: 0.3609 - val_accuracy: 0.8436\n",
+      "25000/25000 [==============================] - 95s 4ms/sample - loss: 0.2852 - accuracy: 0.8811 - val_loss: 0.4136 - val_accuracy: 0.8128\n",
       "Epoch 3/3\n",
-      "25000/25000 [==============================] - 83s 3ms/sample - loss: 0.1299 - accuracy: 0.9513 - val_loss: 0.4395 - val_accuracy: 0.8242\n"
+      "25000/25000 [==============================] - 97s 4ms/sample - loss: 0.1372 - accuracy: 0.9485 - val_loss: 0.4949 - val_accuracy: 0.8190\n"
      ]
     },
     {
      "data": {
       "text/plain": [
-       "<tensorflow.python.keras.callbacks.History at 0x10bc456a0>"
+       "<tensorflow.python.keras.callbacks.History at 0x109a5b4e0>"
       ]
      },
-     "execution_count": 31,
+     "execution_count": 11,
      "metadata": {},
      "output_type": "execute_result"
     }

From 4d977a0fa68b7f222073a780a0fdd4b7d9fcfa6b Mon Sep 17 00:00:00 2001
From: Shun Lin <shunlin@google.com>
Date: Mon, 18 Nov 2019 21:01:41 -0800
Subject: [PATCH 17/20] changed the variable name for lambda layer and
 initilize it only when condition is met

---
 tensorflow_addons/layers/tcn.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/tensorflow_addons/layers/tcn.py b/tensorflow_addons/layers/tcn.py
index 181412e502..58bd0563f1 100644
--- a/tensorflow_addons/layers/tcn.py
+++ b/tensorflow_addons/layers/tcn.py
@@ -192,8 +192,8 @@ class TCN(tf.keras.layers.Layer):
         use_skip_connections: Boolean. If we want to add skip
             connections from input to each residual block.
             Defaults to True.
-        return_sequences: Boolean. Whether to return the last
-            output in the output sequence, or the full sequence.
+        return_sequences: Boolean. Whether to return the full sequence
+            (when True) or the last output in the output sequence (when False).
             Defaults to False.
         activation: The activation used in the residual
             blocks o = Activation(x + F(x)). Defaults to 'linear'
@@ -279,7 +279,9 @@ def __init__(self,
         for layer in self.residual_blocks:
             self.__setattr__(layer.name, layer)
 
-        self.lambda_layer = tf.keras.layers.Lambda(lambda tt: tt[:, -1, :])
+        if not self.return_sequences:
+            self.last_output_layer = tf.keras.layers.Lambda(
+                lambda tt: tt[:, -1, :])
 
     def build(self, input_shape):
         self.main_conv1D.build(input_shape)
@@ -295,7 +297,7 @@ def compute_output_shape(self, input_shape):
         if not self.built:
             self.build(input_shape)
         if not self.return_sequences:
-            return self.lambda_layer.compute_output_shape(
+            return self.last_output_layer.compute_output_shape(
                 self.build_output_shape)
         else:
             return self.build_output_shape
@@ -311,7 +313,7 @@ def call(self, inputs, training=None):
         if self.use_skip_connections:
             x = tf.keras.layers.add(skip_connections)
         if not self.return_sequences:
-            x = self.lambda_layer(x)
+            x = self.last_output_layer(x)
         return x
 
     def get_config(self):

From ede2428c2843f8186b2b435c8c03341676ad26c0 Mon Sep 17 00:00:00 2001
From: Shun Lin <shunlin@google.com>
Date: Mon, 18 Nov 2019 22:28:23 -0800
Subject: [PATCH 18/20] added test for serialization and deserialization

---
 tensorflow_addons/layers/tcn.py      | 10 -----
 tensorflow_addons/layers/tcn_test.py | 58 ++++++++++++++++++++++++++++
 2 files changed, 58 insertions(+), 10 deletions(-)

diff --git a/tensorflow_addons/layers/tcn.py b/tensorflow_addons/layers/tcn.py
index 58bd0563f1..ee558efbbb 100644
--- a/tensorflow_addons/layers/tcn.py
+++ b/tensorflow_addons/layers/tcn.py
@@ -114,11 +114,6 @@ def __init__(self,
 
             self.final_activation = tf.keras.layers.Activation(self.activation)
 
-            # this is done to force keras to add the layers in the list to
-            # self._layers
-            for layer in self.residual_layers:
-                self.__setattr__(layer.name, layer)
-
     def build(self, input_shape):
 
         # build residual layers
@@ -274,11 +269,6 @@ def __init__(self,
                         name='residual_block_{}'.format(
                             len(self.residual_blocks))))
 
-        # this is done to force keras to add the layers in the list to
-        # self._layers
-        for layer in self.residual_blocks:
-            self.__setattr__(layer.name, layer)
-
         if not self.return_sequences:
             self.last_output_layer = tf.keras.layers.Lambda(
                 lambda tt: tt[:, -1, :])
diff --git a/tensorflow_addons/layers/tcn_test.py b/tensorflow_addons/layers/tcn_test.py
index 8597f30560..ee587724e5 100644
--- a/tensorflow_addons/layers/tcn_test.py
+++ b/tensorflow_addons/layers/tcn_test.py
@@ -22,6 +22,7 @@
 
 from tensorflow_addons.utils import test_utils
 from tensorflow_addons.layers import TCN
+from tensorflow_addons.layers.tcn import ResidualBlock
 
 
 @test_utils.run_all_in_graph_and_eager_modes
@@ -29,6 +30,63 @@ class TCNTest(tf.test.TestCase):
     def test_tcn(self):
         test_utils.layer_test(TCN, input_shape=(2, 4, 4))
 
+    def test_config_tcn(self):
+
+        # test default config
+        tcn = TCN()
+        self.assertEqual(tcn.filters, 64)
+        self.assertEqual(tcn.kernel_size, 2)
+        self.assertEqual(tcn.stacks, 1)
+        self.assertEqual(tcn.dilations, [1, 2, 4, 8, 16, 32, 64])
+        self.assertEqual(tcn.padding, 'causal')
+        self.assertEqual(tcn.use_skip_connections, True)
+        self.assertEqual(tcn.dropout_rate, 0.0)
+        self.assertEqual(tcn.return_sequences, False)
+        self.assertEqual(tcn.activation, 'linear')
+        self.assertEqual(tcn.kernel_initializer, 'he_normal')
+        self.assertEqual(tcn.use_batch_norm, False)
+
+        # Check save and restore config
+        tcn_2 = TCN.from_config(tcn.get_config())
+        self.assertEqual(tcn_2.filters, 64)
+        self.assertEqual(tcn_2.kernel_size, 2)
+        self.assertEqual(tcn_2.stacks, 1)
+        self.assertEqual(tcn_2.dilations, [1, 2, 4, 8, 16, 32, 64])
+        self.assertEqual(tcn_2.padding, 'causal')
+        self.assertEqual(tcn_2.use_skip_connections, True)
+        self.assertEqual(tcn_2.dropout_rate, 0.0)
+        self.assertEqual(tcn_2.return_sequences, False)
+        self.assertEqual(tcn_2.activation, 'linear')
+        self.assertEqual(tcn_2.kernel_initializer, 'he_normal')
+        self.assertEqual(tcn_2.use_batch_norm, False)
+
+    def test_config_residual_block(self):
+
+        # test default config
+        residual_block = ResidualBlock()
+        self.assertEqual(residual_block.dilation_rate, 1)
+        self.assertEqual(residual_block.filters, 64)
+        self.assertEqual(residual_block.kernel_size, 2)
+        self.assertEqual(residual_block.padding, 'same')
+        self.assertEqual(residual_block.activation, 'relu')
+        self.assertEqual(residual_block.dropout_rate, 0.0)
+        self.assertEqual(residual_block.kernel_initializer, 'he_normal')
+        self.assertEqual(residual_block.last_block, False)
+        self.assertEqual(residual_block.use_batch_norm, False)
+
+        # Check save and restore config
+        residual_block_2 = ResidualBlock.from_config(
+            residual_block.get_config())
+        self.assertEqual(residual_block_2.dilation_rate, 1)
+        self.assertEqual(residual_block_2.filters, 64)
+        self.assertEqual(residual_block_2.kernel_size, 2)
+        self.assertEqual(residual_block_2.padding, 'same')
+        self.assertEqual(residual_block_2.activation, 'relu')
+        self.assertEqual(residual_block_2.dropout_rate, 0.0)
+        self.assertEqual(residual_block_2.kernel_initializer, 'he_normal')
+        self.assertEqual(residual_block_2.last_block, False)
+        self.assertEqual(residual_block_2.use_batch_norm, False)
+
 
 if __name__ == "__main__":
     tf.test.main()

From b8664d7b4c367314c1d39d2f332ac1d917e25dd4 Mon Sep 17 00:00:00 2001
From: Shun Lin <shunlin@google.com>
Date: Tue, 19 Nov 2019 10:58:07 -0800
Subject: [PATCH 19/20] fix lambda styling error

---
 env/bin/tensorboard             | 8 ++++++++
 tensorflow_addons/layers/tcn.py | 4 ++--
 2 files changed, 10 insertions(+), 2 deletions(-)
 create mode 100755 env/bin/tensorboard

diff --git a/env/bin/tensorboard b/env/bin/tensorboard
new file mode 100755
index 0000000000..5933b25dbc
--- /dev/null
+++ b/env/bin/tensorboard
@@ -0,0 +1,8 @@
+#!/Users/shunlin/addons/env/bin/python3
+# -*- coding: utf-8 -*-
+import re
+import sys
+from tensorboard.main import run_main
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(run_main())
diff --git a/tensorflow_addons/layers/tcn.py b/tensorflow_addons/layers/tcn.py
index ee558efbbb..9519b09a56 100644
--- a/tensorflow_addons/layers/tcn.py
+++ b/tensorflow_addons/layers/tcn.py
@@ -270,8 +270,8 @@ def __init__(self,
                             len(self.residual_blocks))))
 
         if not self.return_sequences:
-            self.last_output_layer = tf.keras.layers.Lambda(
-                lambda tt: tt[:, -1, :])
+            self.last_output_layer = tf.keras.layers.Lambda(lambda x:
+                                                            x[:, -1, :])
 
     def build(self, input_shape):
         self.main_conv1D.build(input_shape)

From 82e90e248aa96991fd77645969de656194a021d8 Mon Sep 17 00:00:00 2001
From: Shun Lin <shunlin@google.com>
Date: Tue, 19 Nov 2019 12:19:31 -0800
Subject: [PATCH 20/20] Remove env/bin/Tensorboard

---
 env/bin/tensorboard | 8 --------
 1 file changed, 8 deletions(-)
 delete mode 100755 env/bin/tensorboard

diff --git a/env/bin/tensorboard b/env/bin/tensorboard
deleted file mode 100755
index 5933b25dbc..0000000000
--- a/env/bin/tensorboard
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/Users/shunlin/addons/env/bin/python3
-# -*- coding: utf-8 -*-
-import re
-import sys
-from tensorboard.main import run_main
-if __name__ == '__main__':
-    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
-    sys.exit(run_main())