From 30c45e2309018b5ce2aeeca41ac5ce49c01200eb Mon Sep 17 00:00:00 2001
From: Moritz Kirschte <moritz.kirschte@gmail.com>
Date: Mon, 18 Feb 2019 19:46:16 +0100
Subject: [PATCH 1/9] finally made cdroupout for keras-contrib available

---
 keras_contrib/wrappers/__init__.py            |   3 +
 keras_contrib/wrappers/cdropout.py            | 165 ++++++++++++++++++
 tests/keras_contrib/wrappers/test_cdropout.py |  59 +++++++
 3 files changed, 227 insertions(+)
 create mode 100644 keras_contrib/wrappers/cdropout.py
 create mode 100644 tests/keras_contrib/wrappers/test_cdropout.py

diff --git a/keras_contrib/wrappers/__init__.py b/keras_contrib/wrappers/__init__.py
index e69de29bb..dffcc152d 100644
--- a/keras_contrib/wrappers/__init__.py
+++ b/keras_contrib/wrappers/__init__.py
@@ -0,0 +1,3 @@
+from __future__ import absolute_import
+
+from .cdropout import ConcreteDropout
\ No newline at end of file
diff --git a/keras_contrib/wrappers/cdropout.py b/keras_contrib/wrappers/cdropout.py
new file mode 100644
index 000000000..0192e83ad
--- /dev/null
+++ b/keras_contrib/wrappers/cdropout.py
@@ -0,0 +1,165 @@
+# -*- coding: utf-8 -*-
+
+import numpy as np
+
+from keras import backend as K
+from keras.engine import InputSpec
+from keras.initializers import RandomUniform
+from keras.layers import Wrapper
+
+
+class ConcreteDropout(Wrapper):
+    """A wrapper automating the dropout rate choice
+       through the 'Concrete Dropout' technique.
+
+    # Example
+
+    ```python
+        # as first layer in a sequential model:
+        model = Sequential()
+        model.add(ConcreteDropout(Dense(8), input_shape=(16)), n_data=5000)
+        # now model.output_shape == (None, 8)
+        # subsequent layers: no need for input shape
+        model.add(ConcreteDropout(Dense(32), n_data=500))
+        # now model.output_shape == (None, 32)
+
+        # Note that the current implementation supports Conv2D Layer as well.
+    ```
+
+    # Arguments
+        layer: The to be wrapped layer.
+        n_data: int. Length of the dataset.
+        length_scale: float. Prior lengthscale.
+        model_precision: float. Model precision parameter is `1` for classification.
+                         Also known as inverse observation noise.
+        prob_init: Tuple[float, float].
+                   Probability  lower / upper bounds of dropout rate initialization.
+        temp: float. Temperature. Not used to be optimized.
+        seed: Seed for random probability  sampling.
+
+    # References
+        - [Concrete Dropout](https://arxiv.org/pdf/1705.07832.pdf)
+    """
+
+    def __init__(self,
+                 layer,
+                 n_data,
+                 length_scale=2e-2,
+                 model_precision=1,
+                 prob_init=(0.1, 0.5),
+                 temp=0.1,
+                 seed=None,
+                 **kwargs):
+        assert 'kernel_regularizer' not in kwargs
+        super(ConcreteDropout, self).__init__(layer, **kwargs)
+        self.weight_regularizer = length_scale**2 / (model_precision * n_data)
+        self.dropout_regularizer = 2 / (model_precision * n_data)
+        self.prob_init = tuple(np.log(prob_init))
+        self.temp = temp
+        self.seed = seed
+
+        self.supports_masking = True
+        self.p_logit = None
+        self.p = None
+
+    def _concrete_dropout(self, inputs, layer_type):
+        """Applies concrete dropout.
+           Used at training time (gradients can be propagated)
+
+        # Arguments
+            inputs: Input.
+            layer_type: str. Either 'dense' or 'conv2d'.
+        # Returns
+            A tensor with the same shape as inputs and dropout applied.
+        """
+        eps = K.cast_to_floatx(K.epsilon())
+
+        noise_shape = K.shape(inputs)
+        if layer_type == 'conv2d':
+            if K.image_data_format() == 'channels_first':
+                noise_shape = (noise_shape[0], noise_shape[1], 1, 1)
+            else:
+                noise_shape = (noise_shape[0], 1, 1, noise_shape[3])
+        unif_noise = K.random_uniform(shape=noise_shape,
+                                      seed=self.seed,
+                                      dtype=inputs.dtype)
+        drop_prob = (
+            K.log(self.p + eps)
+            - K.log(1. - self.p + eps)
+            + K.log(unif_noise + eps)
+            - K.log(1. - unif_noise + eps)
+        )
+        drop_prob = K.sigmoid(drop_prob / self.temp)
+
+        random_tensor = 1. - drop_prob
+        retain_prob = 1. - self.p
+        inputs *= random_tensor
+        inputs /= retain_prob
+
+        return inputs
+
+    def build(self, input_shape=None):
+        if len(input_shape) == 2:  # Dense_layer
+            input_dim = np.prod(input_shape[-1])  # we drop only last dim
+        elif len(input_shape) == 4:  # Conv_layer
+            input_dim = (input_shape[1]
+                         if K.image_data_format() == 'channels_first'
+                         else input_shape[3])  # we drop only channels
+        else:
+            raise ValueError(
+                'concrete_dropout currenty supports only Dense/Conv2D layers')
+
+        self.input_spec = InputSpec(shape=input_shape)
+        if not self.layer.built:
+            self.layer.build(input_shape)
+            self.layer.built = True
+
+        # initialise p
+        self.p_logit = self.layer.add_weight(name='p_logit',
+                                             shape=(1,),
+                                             initializer=RandomUniform(
+                                                 *self.prob_init,
+                                                 seed=self.seed
+                                             ),
+                                             trainable=True)
+        self.p = K.squeeze(K.sigmoid(self.p_logit), axis=0)
+
+        super(ConcreteDropout, self).build(input_shape)
+
+        # initialise regularizer / prior KL term
+        weight = self.layer.kernel
+        kernel_regularizer = (
+            self.weight_regularizer
+            * K.sum(K.square(weight))
+            / (1. - self.p)
+        )
+        dropout_regularizer = (
+            self.p * K.log(self.p)
+            + (1. - self.p) * K.log(1. - self.p)
+        ) * self.dropout_regularizer * input_dim
+        regularizer = K.sum(kernel_regularizer + dropout_regularizer)
+        self.layer.add_loss(regularizer)
+
+    def call(self, inputs, training=None):
+        def relaxed_dropped_inputs():
+            return self.layer.call(self._concrete_dropout(inputs, (
+                'dense'
+                if len(K.int_shape(inputs)) == 2
+                else 'conv2d'
+            )))
+
+        return K.in_train_phase(relaxed_dropped_inputs,
+                                self.layer.call(inputs),
+                                training=training)
+
+    def get_config(self):
+        config = {'weight_regularizer': self.weight_regularizer,
+                  'dropout_regularizer': self.dropout_regularizer,
+                  'prob_init': self.prob_init,
+                  'temp': self.temp,
+                  'seed': self.seed}
+        base_config = super(ConcreteDropout, self).get_config()
+        return dict(list(base_config.items()) + list(config.items()))
+
+    def compute_output_shape(self, input_shape):
+        return self.layer.compute_output_shape(input_shape)
diff --git a/tests/keras_contrib/wrappers/test_cdropout.py b/tests/keras_contrib/wrappers/test_cdropout.py
new file mode 100644
index 000000000..31f263788
--- /dev/null
+++ b/tests/keras_contrib/wrappers/test_cdropout.py
@@ -0,0 +1,59 @@
+import pytest
+import numpy as np
+
+from keras.layers import Input, Dense
+from keras.models import Model
+from numpy.testing import assert_allclose
+from numpy.testing import assert_array_almost_equal
+from numpy.testing import assert_approx_equal
+from numpy.testing import assert_equal
+
+from keras_contrib.wrappers import ConcreteDropout
+
+
+def test_cdropout():
+    #  Data
+    in_dim = 20
+    init_prop = .1
+    np.random.seed(1)
+    X = np.random.randn(1, in_dim)
+
+    #  Model
+    inputs = Input(shape=(in_dim,))
+    dense = Dense(1, use_bias=True, input_shape=(in_dim,))
+    #  Model, normal
+    cd = ConcreteDropout(dense, in_dim, prob_init=(init_prop, init_prop))
+    x = cd(inputs)
+    model = Model(inputs, x)
+    model.compile(loss=None, optimizer='rmsprop')
+    #  Model, reference w/o Dropout
+    x_ref = dense(inputs)
+    model_ref = Model(inputs, x_ref)
+    model_ref.compile(loss='mse', optimizer='rmsprop')
+
+    #  Check about correct 3rd weight (equal to initial value)
+    W = model.get_weights()
+    assert_array_almost_equal(W[2], [np.log(init_prop)])
+
+    #  Check if ConcreteDropout in prediction phase is the same as no dropout
+    out = model.predict(X)
+    out_ref = model_ref.predict(X)
+    assert_allclose(out, out_ref, atol=1e-5)
+
+    #  Check if ConcreteDropout has the right amount of losses deposited
+    assert_equal(len(model.losses), 1)
+
+    #  Check if the loss correspons the the desired value
+    def sigmoid(x):
+        return 1. / (1. + np.exp(-x))
+    p = np.squeeze(sigmoid(W[2]))
+    kernel_regularizer = cd.weight_regularizer * np.sum(np.square(W[0])) / (1. - p)
+    dropout_regularizer = (p * np.log(p) + (1. - p) * np.log(1. - p))
+    dropout_regularizer *= cd.dropout_regularizer * in_dim
+    loss = np.sum(kernel_regularizer + dropout_regularizer)
+    eval_loss = model.evaluate(X)
+    assert_approx_equal(eval_loss, loss)
+
+
+if __name__ == '__main__':
+    pytest.main([__file__])

From fcab9cbc5a52cb870d1d7dc9240adcff3edcfe34 Mon Sep 17 00:00:00 2001
From: Moritz Kirschte <moritz.kirschte@gmail.com>
Date: Mon, 18 Feb 2019 19:48:36 +0100
Subject: [PATCH 2/9] added myself to codeowner

---
 CODEOWNERS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CODEOWNERS b/CODEOWNERS
index afa98df95..1fafadbb1 100644
--- a/CODEOWNERS
+++ b/CODEOWNERS
@@ -52,3 +52,4 @@ keras_contrib/optimizers/padam.py @MFreidank
 
 
 # wrappers
+keras_contrib/wrappers/cdropout.py @moritzmoritz98
\ No newline at end of file

From 45cd51faf9ee75320e19fbf1b9ae42693d066d47 Mon Sep 17 00:00:00 2001
From: Moritz Kirschte <moritz.kirschte@gmail.com>
Date: Mon, 18 Feb 2019 20:30:43 +0100
Subject: [PATCH 3/9] fixing travis build

---
 keras_contrib/wrappers/cdropout.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/keras_contrib/wrappers/cdropout.py b/keras_contrib/wrappers/cdropout.py
index 0192e83ad..7a5e68af9 100644
--- a/keras_contrib/wrappers/cdropout.py
+++ b/keras_contrib/wrappers/cdropout.py
@@ -1,11 +1,12 @@
 # -*- coding: utf-8 -*-
+from __future__ import absolute_import
 
 import numpy as np
 
 from keras import backend as K
 from keras.engine import InputSpec
 from keras.initializers import RandomUniform
-from keras.layers import Wrapper
+from keras.layers.wrappers import Wrapper
 
 
 class ConcreteDropout(Wrapper):

From 900761863540ee9ca8cdc9ba48bea2da6256d8a5 Mon Sep 17 00:00:00 2001
From: Moritz Kirschte <moritz.kirschte@gmail.com>
Date: Mon, 18 Feb 2019 21:25:04 +0100
Subject: [PATCH 4/9] added new test cases for coverage

---
 keras_contrib/wrappers/__init__.py            |  2 +-
 keras_contrib/wrappers/cdropout.py            |  7 +-
 tests/keras_contrib/wrappers/test_cdropout.py | 83 +++++++++++++++++--
 3 files changed, 81 insertions(+), 11 deletions(-)

diff --git a/keras_contrib/wrappers/__init__.py b/keras_contrib/wrappers/__init__.py
index dffcc152d..7490e705c 100644
--- a/keras_contrib/wrappers/__init__.py
+++ b/keras_contrib/wrappers/__init__.py
@@ -1,3 +1,3 @@
 from __future__ import absolute_import
 
-from .cdropout import ConcreteDropout
\ No newline at end of file
+from .cdropout import ConcreteDropout
diff --git a/keras_contrib/wrappers/cdropout.py b/keras_contrib/wrappers/cdropout.py
index 7a5e68af9..d24b547db 100644
--- a/keras_contrib/wrappers/cdropout.py
+++ b/keras_contrib/wrappers/cdropout.py
@@ -2,10 +2,9 @@
 from __future__ import absolute_import
 
 import numpy as np
-
 from keras import backend as K
-from keras.engine import InputSpec
 from keras.initializers import RandomUniform
+from keras.layers import InputSpec
 from keras.layers.wrappers import Wrapper
 
 
@@ -34,7 +33,7 @@ class ConcreteDropout(Wrapper):
         model_precision: float. Model precision parameter is `1` for classification.
                          Also known as inverse observation noise.
         prob_init: Tuple[float, float].
-                   Probability  lower / upper bounds of dropout rate initialization.
+                   Probability lower / upper bounds of dropout rate initialization.
         temp: float. Temperature. Not used to be optimized.
         seed: Seed for random probability  sampling.
 
@@ -156,7 +155,7 @@ def relaxed_dropped_inputs():
     def get_config(self):
         config = {'weight_regularizer': self.weight_regularizer,
                   'dropout_regularizer': self.dropout_regularizer,
-                  'prob_init': self.prob_init,
+                  'prob_init': tuple(np.round(self.prob_init, 8)),
                   'temp': self.temp,
                   'seed': self.seed}
         base_config = super(ConcreteDropout, self).get_config()
diff --git a/tests/keras_contrib/wrappers/test_cdropout.py b/tests/keras_contrib/wrappers/test_cdropout.py
index 31f263788..730dbdfcd 100644
--- a/tests/keras_contrib/wrappers/test_cdropout.py
+++ b/tests/keras_contrib/wrappers/test_cdropout.py
@@ -1,26 +1,25 @@
 import pytest
 import numpy as np
 
-from keras.layers import Input, Dense
-from keras.models import Model
 from numpy.testing import assert_allclose
 from numpy.testing import assert_array_almost_equal
 from numpy.testing import assert_approx_equal
 from numpy.testing import assert_equal
-
+from keras.layers import Input, Dense, Conv1D, Conv2D, Conv3D
+from keras.models import Model
 from keras_contrib.wrappers import ConcreteDropout
 
 
 def test_cdropout():
-    #  Data
+    #  DATA
     in_dim = 20
     init_prop = .1
     np.random.seed(1)
     X = np.random.randn(1, in_dim)
 
-    #  Model
+    #  MODEL
     inputs = Input(shape=(in_dim,))
-    dense = Dense(1, use_bias=True, input_shape=(in_dim,))
+    dense = Dense(1, use_bias=True)
     #  Model, normal
     cd = ConcreteDropout(dense, in_dim, prob_init=(init_prop, init_prop))
     x = cd(inputs)
@@ -31,6 +30,7 @@ def test_cdropout():
     model_ref = Model(inputs, x_ref)
     model_ref.compile(loss='mse', optimizer='rmsprop')
 
+    #  CHECKS
     #  Check about correct 3rd weight (equal to initial value)
     W = model.get_weights()
     assert_array_almost_equal(W[2], [np.log(init_prop)])
@@ -55,5 +55,76 @@ def sigmoid(x):
     assert_approx_equal(eval_loss, loss)
 
 
+def test_cdropout_conv():
+    #  DATA
+    in_dim = 20
+    init_prop = .1
+    np.random.seed(1)
+    X = np.random.randn(1, in_dim, in_dim, 1)
+
+    #  MODEL
+    inputs = Input(shape=(in_dim, in_dim, 1,))
+    conv2d = Conv2D(1, (3, 3))
+    #  Model, normal
+    cd = ConcreteDropout(conv2d, in_dim, prob_init=(init_prop, init_prop))
+    x = cd(inputs)
+    model = Model(inputs, x)
+    model.compile(loss=None, optimizer='rmsprop')
+    #  Model, reference w/o Dropout
+    x_ref = conv2d(inputs)
+    model_ref = Model(inputs, x_ref)
+    model_ref.compile(loss=None, optimizer='rmsprop')
+
+    #  CHECKS
+    #  Check about correct 3rd weight (equal to initial value)
+    W = model.get_weights()
+    assert_array_almost_equal(W[2], [np.log(init_prop)])
+
+    #  Check if ConcreteDropout in prediction phase is the same as no dropout
+    out = model.predict(X)
+    out_ref = model_ref.predict(X)
+    assert_allclose(out, out_ref, atol=1e-5)
+
+    #  Check if ConcreteDropout has the right amount of losses deposited
+    assert_equal(len(model.losses), 1)
+
+    #  Check if the loss correspons the the desired value
+    def sigmoid(x):
+        return 1. / (1. + np.exp(-x))
+    p = np.squeeze(sigmoid(W[2]))
+    kernel_regularizer = cd.weight_regularizer * np.sum(np.square(W[0])) / (1. - p)
+    dropout_regularizer = (p * np.log(p) + (1. - p) * np.log(1. - p))
+    dropout_regularizer *= cd.dropout_regularizer * 1  # only channels are dropped
+    loss = np.sum(kernel_regularizer + dropout_regularizer)
+    eval_loss = model.evaluate(X)
+    assert_approx_equal(eval_loss, loss)
+
+
+def test_cdropout_1d_layer():
+    """To be replaced with a real function test, if implemented.
+    """
+    in_dim = 20
+    init_prop = .1
+
+    with pytest.raises(ValueError):
+        inputs = Input(shape=(in_dim, 1,))
+        ConcreteDropout(Conv1D(1, 3),
+                        in_dim,
+                        prob_init=(init_prop, init_prop))(inputs)
+
+
+def test_cdropout_3d_layer():
+    """To be replaced with a real function test, if implemented.
+    """
+    in_dim = 20
+    init_prop = .1
+
+    with pytest.raises(ValueError):
+        inputs = Input(shape=(in_dim, in_dim, in_dim, 1,))
+        ConcreteDropout(Conv3D(1, 3),
+                        in_dim,
+                        prob_init=(init_prop, init_prop))(inputs)
+
+
 if __name__ == '__main__':
     pytest.main([__file__])

From 4271e9c1e67b9d24b4e37f2d2fb30b43200457d6 Mon Sep 17 00:00:00 2001
From: Moritz Kirschte <moritz.kirschte@gmail.com>
Date: Tue, 19 Feb 2019 12:06:19 +0100
Subject: [PATCH 5/9] refactored tests

---
 keras_contrib/wrappers/cdropout.py            |  15 ++-
 tests/keras_contrib/wrappers/test_cdropout.py | 113 ++++++++++++++++--
 2 files changed, 109 insertions(+), 19 deletions(-)

diff --git a/keras_contrib/wrappers/cdropout.py b/keras_contrib/wrappers/cdropout.py
index d24b547db..57b44acfc 100644
--- a/keras_contrib/wrappers/cdropout.py
+++ b/keras_contrib/wrappers/cdropout.py
@@ -6,6 +6,7 @@
 from keras.initializers import RandomUniform
 from keras.layers import InputSpec
 from keras.layers.wrappers import Wrapper
+from keras_contrib.utils.test_utils import to_tuple
 
 
 class ConcreteDropout(Wrapper):
@@ -34,8 +35,9 @@ class ConcreteDropout(Wrapper):
                          Also known as inverse observation noise.
         prob_init: Tuple[float, float].
                    Probability lower / upper bounds of dropout rate initialization.
-        temp: float. Temperature. Not used to be optimized.
-        seed: Seed for random probability  sampling.
+        temp: float. Temperature.
+              Determines the speed of probability adjustments.
+        seed: Seed for random probability sampling.
 
     # References
         - [Concrete Dropout](https://arxiv.org/pdf/1705.07832.pdf)
@@ -44,10 +46,10 @@ class ConcreteDropout(Wrapper):
     def __init__(self,
                  layer,
                  n_data,
-                 length_scale=2e-2,
+                 length_scale=5e-2,
                  model_precision=1,
                  prob_init=(0.1, 0.5),
-                 temp=0.1,
+                 temp=0.4,
                  seed=None,
                  **kwargs):
         assert 'kernel_regularizer' not in kwargs
@@ -64,7 +66,7 @@ def __init__(self,
 
     def _concrete_dropout(self, inputs, layer_type):
         """Applies concrete dropout.
-           Used at training time (gradients can be propagated)
+           Used at training time (gradients can be propagated).
 
         # Arguments
             inputs: Input.
@@ -99,6 +101,7 @@ def _concrete_dropout(self, inputs, layer_type):
         return inputs
 
     def build(self, input_shape=None):
+        input_shape = to_tuple(input_shape)
         if len(input_shape) == 2:  # Dense_layer
             input_dim = np.prod(input_shape[-1])  # we drop only last dim
         elif len(input_shape) == 4:  # Conv_layer
@@ -126,7 +129,7 @@ def build(self, input_shape=None):
 
         super(ConcreteDropout, self).build(input_shape)
 
-        # initialise regularizer / prior KL term
+        # initialize regularizer / prior KL term
         weight = self.layer.kernel
         kernel_regularizer = (
             self.weight_regularizer
diff --git a/tests/keras_contrib/wrappers/test_cdropout.py b/tests/keras_contrib/wrappers/test_cdropout.py
index 730dbdfcd..c9363c400 100644
--- a/tests/keras_contrib/wrappers/test_cdropout.py
+++ b/tests/keras_contrib/wrappers/test_cdropout.py
@@ -5,12 +5,23 @@
 from numpy.testing import assert_array_almost_equal
 from numpy.testing import assert_approx_equal
 from numpy.testing import assert_equal
+from keras import backend as K
 from keras.layers import Input, Dense, Conv1D, Conv2D, Conv3D
 from keras.models import Model
 from keras_contrib.wrappers import ConcreteDropout
 
 
-def test_cdropout():
+@pytest.fixture
+def clear_session_after_test():
+    """Overridden: make session cleanup manually.
+    """
+    pass
+
+
+@pytest.fixture(scope='module')
+def dense_model():
+    """Initialize to be tested dense model. Executed once.
+    """
     #  DATA
     in_dim = 20
     init_prop = .1
@@ -28,34 +39,74 @@ def test_cdropout():
     #  Model, reference w/o Dropout
     x_ref = dense(inputs)
     model_ref = Model(inputs, x_ref)
-    model_ref.compile(loss='mse', optimizer='rmsprop')
+    model_ref.compile(loss=None, optimizer='rmsprop')
+
+    yield {'model': model,
+           'model_ref': model_ref,
+           'concrete_dropout': cd,
+           'init_prop': init_prop,
+           'in_dim': in_dim,
+           'X': X}
+    if K.backend() == 'tensorflow' or K.backend() == 'cntk':
+        K.clear_session()
+
+
+def test_cdropout_dense_3rdweight(dense_model):
+    """Check about correct 3rd weight (equal to initial value)
+    """
+    model = dense_model['model']
+    init_prop = dense_model['init_prop']
 
-    #  CHECKS
-    #  Check about correct 3rd weight (equal to initial value)
     W = model.get_weights()
     assert_array_almost_equal(W[2], [np.log(init_prop)])
 
-    #  Check if ConcreteDropout in prediction phase is the same as no dropout
+
+def test_cdropout_dense_identity(dense_model):
+    """Check if ConcreteDropout in prediction phase is the same as no dropout
+    """
+    model = dense_model['model']
+    model_ref = dense_model['model_ref']
+    X = dense_model['X']
+
     out = model.predict(X)
     out_ref = model_ref.predict(X)
     assert_allclose(out, out_ref, atol=1e-5)
 
-    #  Check if ConcreteDropout has the right amount of losses deposited
+
+def test_cdropout_dense_loss(dense_model):
+    """Check if ConcreteDropout has the right amount of losses deposited
+    """
+    model = dense_model['model']
+
     assert_equal(len(model.losses), 1)
 
-    #  Check if the loss correspons the the desired value
+
+def test_cdropout_dense_loss_value(dense_model):
+    """Check if the loss corresponds the the desired value
+    """
+    model = dense_model['model']
+    X = dense_model['X']
+    cd = dense_model['concrete_dropout']
+    in_dim = dense_model['in_dim']
+
     def sigmoid(x):
         return 1. / (1. + np.exp(-x))
+
+    W = model.get_weights()
     p = np.squeeze(sigmoid(W[2]))
     kernel_regularizer = cd.weight_regularizer * np.sum(np.square(W[0])) / (1. - p)
     dropout_regularizer = (p * np.log(p) + (1. - p) * np.log(1. - p))
     dropout_regularizer *= cd.dropout_regularizer * in_dim
     loss = np.sum(kernel_regularizer + dropout_regularizer)
+
     eval_loss = model.evaluate(X)
     assert_approx_equal(eval_loss, loss)
 
 
-def test_cdropout_conv():
+@pytest.fixture(scope='module')
+def conv2d_model():
+    """Initialize to be tested conv model. Executed once.
+    """
     #  DATA
     in_dim = 20
     init_prop = .1
@@ -75,27 +126,63 @@ def test_cdropout_conv():
     model_ref = Model(inputs, x_ref)
     model_ref.compile(loss=None, optimizer='rmsprop')
 
-    #  CHECKS
-    #  Check about correct 3rd weight (equal to initial value)
+    yield {'model': model,
+           'model_ref': model_ref,
+           'concrete_dropout': cd,
+           'init_prop': init_prop,
+           'in_dim': in_dim,
+           'X': X}
+    if K.backend() == 'tensorflow' or K.backend() == 'cntk':
+        K.clear_session()
+
+
+def test_cdropout_conv2d_3rdweight(conv2d_model):
+    """Check about correct 3rd weight (equal to initial value)
+    """
+    model = conv2d_model['model']
+    init_prop = conv2d_model['init_prop']
+
     W = model.get_weights()
     assert_array_almost_equal(W[2], [np.log(init_prop)])
 
-    #  Check if ConcreteDropout in prediction phase is the same as no dropout
+
+def test_cdropout_conv2d_identity(conv2d_model):
+    """Check if ConcreteDropout in prediction phase is the same as no dropout
+    """
+    model = conv2d_model['model']
+    model_ref = conv2d_model['model_ref']
+    X = conv2d_model['X']
+
     out = model.predict(X)
     out_ref = model_ref.predict(X)
     assert_allclose(out, out_ref, atol=1e-5)
 
-    #  Check if ConcreteDropout has the right amount of losses deposited
+
+def test_cdropout_conv2d_loss(conv2d_model):
+    """Check if ConcreteDropout has the right amount of losses deposited
+    """
+    model = conv2d_model['model']
+
     assert_equal(len(model.losses), 1)
 
-    #  Check if the loss correspons the the desired value
+
+def test_cdropout_conv2d_loss_value(conv2d_model):
+    """Check if the loss corresponds the the desired value
+    """
+    model = conv2d_model['model']
+    X = conv2d_model['X']
+    cd = conv2d_model['concrete_dropout']
+
     def sigmoid(x):
         return 1. / (1. + np.exp(-x))
+
+    W = model.get_weights()
     p = np.squeeze(sigmoid(W[2]))
     kernel_regularizer = cd.weight_regularizer * np.sum(np.square(W[0])) / (1. - p)
     dropout_regularizer = (p * np.log(p) + (1. - p) * np.log(1. - p))
     dropout_regularizer *= cd.dropout_regularizer * 1  # only channels are dropped
     loss = np.sum(kernel_regularizer + dropout_regularizer)
+
     eval_loss = model.evaluate(X)
     assert_approx_equal(eval_loss, loss)
 

From 088698a3b0d94108c64ed82232ea318af887d5b2 Mon Sep 17 00:00:00 2001
From: Moritz Kirschte <moritz.kirschte@gmail.com>
Date: Tue, 19 Feb 2019 14:17:34 +0100
Subject: [PATCH 6/9] further test coverage increasement

---
 tests/keras_contrib/wrappers/test_cdropout.py | 25 +++++++++++++++----
 1 file changed, 20 insertions(+), 5 deletions(-)

diff --git a/tests/keras_contrib/wrappers/test_cdropout.py b/tests/keras_contrib/wrappers/test_cdropout.py
index c9363c400..4bb8d2762 100644
--- a/tests/keras_contrib/wrappers/test_cdropout.py
+++ b/tests/keras_contrib/wrappers/test_cdropout.py
@@ -103,18 +103,33 @@ def sigmoid(x):
     assert_approx_equal(eval_loss, loss)
 
 
-@pytest.fixture(scope='module')
-def conv2d_model():
-    """Initialize to be tested conv model. Executed once.
+@pytest.fixture(scope='module', params=['channels_first', 'channels_last'])
+def conv2d_model(request):
+    """Initialize to be tested conv model. Executed once per param:
+       The whole tests are repeated for respectively
+       `channels_first` and `channels_last`.
     """
+    assert request.param in {'channels_last', 'channels_first'}
+    K.set_image_data_format(request.param)
+
     #  DATA
     in_dim = 20
     init_prop = .1
     np.random.seed(1)
-    X = np.random.randn(1, in_dim, in_dim, 1)
+    if K.image_data_format() == 'channels_last':
+        X = np.random.randn(1, in_dim, in_dim, 1)
+    elif K.image_data_format() == 'channels_first':
+        X = np.random.randn(1, 1, in_dim, in_dim)
+    else:
+        raise ValueError('Unknown data_format:', K.image_data_format())
 
     #  MODEL
-    inputs = Input(shape=(in_dim, in_dim, 1,))
+    if K.image_data_format() == 'channels_last':
+        inputs = Input(shape=(in_dim, in_dim, 1,))
+    elif K.image_data_format() == 'channels_first':
+        inputs = Input(shape=(1, in_dim, in_dim,))
+    else:
+        raise ValueError('Unknown data_format:', K.image_data_format())
     conv2d = Conv2D(1, (3, 3))
     #  Model, normal
     cd = ConcreteDropout(conv2d, in_dim, prob_init=(init_prop, init_prop))

From 87e4d0f3175404bd06e834b22f90ddc1d50f7297 Mon Sep 17 00:00:00 2001
From: Moritz Kirschte <moritz.kirschte@gmail.com>
Date: Tue, 19 Feb 2019 14:58:30 +0100
Subject: [PATCH 7/9] minor documentary adjustments

---
 keras_contrib/wrappers/cdropout.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/keras_contrib/wrappers/cdropout.py b/keras_contrib/wrappers/cdropout.py
index 57b44acfc..f8e93d073 100644
--- a/keras_contrib/wrappers/cdropout.py
+++ b/keras_contrib/wrappers/cdropout.py
@@ -36,7 +36,7 @@ class ConcreteDropout(Wrapper):
         prob_init: Tuple[float, float].
                    Probability lower / upper bounds of dropout rate initialization.
         temp: float. Temperature.
-              Determines the speed of probability adjustments.
+              Determines the speed of probability (i.e. dropout rate) adjustments.
         seed: Seed for random probability sampling.
 
     # References
@@ -74,6 +74,7 @@ def _concrete_dropout(self, inputs, layer_type):
         # Returns
             A tensor with the same shape as inputs and dropout applied.
         """
+        assert layer_type in {'dense', 'conv2d'}
         eps = K.cast_to_floatx(K.epsilon())
 
         noise_shape = K.shape(inputs)
@@ -93,6 +94,7 @@ def _concrete_dropout(self, inputs, layer_type):
         )
         drop_prob = K.sigmoid(drop_prob / self.temp)
 
+        # apply dropout
         random_tensor = 1. - drop_prob
         retain_prob = 1. - self.p
         inputs *= random_tensor
@@ -104,7 +106,7 @@ def build(self, input_shape=None):
         input_shape = to_tuple(input_shape)
         if len(input_shape) == 2:  # Dense_layer
             input_dim = np.prod(input_shape[-1])  # we drop only last dim
-        elif len(input_shape) == 4:  # Conv_layer
+        elif len(input_shape) == 4:  # Conv2D_layer
             input_dim = (input_shape[1]
                          if K.image_data_format() == 'channels_first'
                          else input_shape[3])  # we drop only channels
@@ -129,7 +131,7 @@ def build(self, input_shape=None):
 
         super(ConcreteDropout, self).build(input_shape)
 
-        # initialize regularizer / prior KL term
+        # initialize regularizer / prior KL term and add to layer-loss
         weight = self.layer.kernel
         kernel_regularizer = (
             self.weight_regularizer
@@ -146,9 +148,7 @@ def build(self, input_shape=None):
     def call(self, inputs, training=None):
         def relaxed_dropped_inputs():
             return self.layer.call(self._concrete_dropout(inputs, (
-                'dense'
-                if len(K.int_shape(inputs)) == 2
-                else 'conv2d'
+                'dense' if len(K.int_shape(inputs)) == 2 else 'conv2d'
             )))
 
         return K.in_train_phase(relaxed_dropped_inputs,

From 1bc09d7e7af8f57fd75d3c841c0f014fdcd63ac0 Mon Sep 17 00:00:00 2001
From: Moritz Kirschte <moritz.kirschte@gmail.com>
Date: Wed, 20 Feb 2019 12:45:03 +0100
Subject: [PATCH 8/9] added layer_test & fixed invalid get_config() response

---
 keras_contrib/wrappers/cdropout.py            | 52 ++++++++++++-------
 tests/keras_contrib/wrappers/test_cdropout.py | 36 ++++++-------
 2 files changed, 51 insertions(+), 37 deletions(-)

diff --git a/keras_contrib/wrappers/cdropout.py b/keras_contrib/wrappers/cdropout.py
index f8e93d073..11bf843c8 100644
--- a/keras_contrib/wrappers/cdropout.py
+++ b/keras_contrib/wrappers/cdropout.py
@@ -29,13 +29,16 @@ class ConcreteDropout(Wrapper):
 
     # Arguments
         layer: The to be wrapped layer.
-        n_data: int. Length of the dataset.
-        length_scale: float. Prior lengthscale.
-        model_precision: float. Model precision parameter is `1` for classification.
+        n_data: int. `n_data > 0`.
+                Length of the dataset.
+        length_scale: float. `length_scale > 0`.
+                      Prior lengthscale.
+        model_precision: float. `model_precision > 0`.
+                         Model precision parameter is `1` for classification.
                          Also known as inverse observation noise.
-        prob_init: Tuple[float, float].
+        prob_init: Tuple[float, float]. `prob_init > 0`
                    Probability lower / upper bounds of dropout rate initialization.
-        temp: float. Temperature.
+        temp: float. Temperature. `temp > 0`.
               Determines the speed of probability (i.e. dropout rate) adjustments.
         seed: Seed for random probability sampling.
 
@@ -53,13 +56,23 @@ def __init__(self,
                  seed=None,
                  **kwargs):
         assert 'kernel_regularizer' not in kwargs
+        assert n_data > 0 and isinstance(n_data, int)
+        assert length_scale > 0.
+        assert prob_init[0] <= prob_init[1] and prob_init[0] > 0.
+        assert temp > 0.
+        assert model_precision > 0.
         super(ConcreteDropout, self).__init__(layer, **kwargs)
-        self.weight_regularizer = length_scale**2 / (model_precision * n_data)
-        self.dropout_regularizer = 2 / (model_precision * n_data)
-        self.prob_init = tuple(np.log(prob_init))
-        self.temp = temp
-        self.seed = seed
 
+        self._n_data = n_data
+        self._length_scale = length_scale
+        self._model_precision = model_precision
+        self._prob_init = prob_init
+        self._temp = temp
+        self._seed = seed
+
+        eps = K.epsilon()
+        self.weight_regularizer = length_scale**2 / (model_precision * n_data + eps)
+        self.dropout_regularizer = 2 / (model_precision * n_data + eps)
         self.supports_masking = True
         self.p_logit = None
         self.p = None
@@ -84,7 +97,7 @@ def _concrete_dropout(self, inputs, layer_type):
             else:
                 noise_shape = (noise_shape[0], 1, 1, noise_shape[3])
         unif_noise = K.random_uniform(shape=noise_shape,
-                                      seed=self.seed,
+                                      seed=self._seed,
                                       dtype=inputs.dtype)
         drop_prob = (
             K.log(self.p + eps)
@@ -92,7 +105,7 @@ def _concrete_dropout(self, inputs, layer_type):
             + K.log(unif_noise + eps)
             - K.log(1. - unif_noise + eps)
         )
-        drop_prob = K.sigmoid(drop_prob / self.temp)
+        drop_prob = K.sigmoid(drop_prob / self._temp)
 
         # apply dropout
         random_tensor = 1. - drop_prob
@@ -123,8 +136,8 @@ def build(self, input_shape=None):
         self.p_logit = self.layer.add_weight(name='p_logit',
                                              shape=(1,),
                                              initializer=RandomUniform(
-                                                 *self.prob_init,
-                                                 seed=self.seed
+                                                 *np.log(self._prob_init),
+                                                 seed=self._seed
                                              ),
                                              trainable=True)
         self.p = K.squeeze(K.sigmoid(self.p_logit), axis=0)
@@ -156,11 +169,12 @@ def relaxed_dropped_inputs():
                                 training=training)
 
     def get_config(self):
-        config = {'weight_regularizer': self.weight_regularizer,
-                  'dropout_regularizer': self.dropout_regularizer,
-                  'prob_init': tuple(np.round(self.prob_init, 8)),
-                  'temp': self.temp,
-                  'seed': self.seed}
+        config = {'n_data': self._n_data,
+                  'length_scale': self._length_scale,
+                  'model_precision': self._model_precision,
+                  'prob_init': self._prob_init,
+                  'temp': self._temp,
+                  'seed': self._seed}
         base_config = super(ConcreteDropout, self).get_config()
         return dict(list(base_config.items()) + list(config.items()))
 
diff --git a/tests/keras_contrib/wrappers/test_cdropout.py b/tests/keras_contrib/wrappers/test_cdropout.py
index 4bb8d2762..6f44852b3 100644
--- a/tests/keras_contrib/wrappers/test_cdropout.py
+++ b/tests/keras_contrib/wrappers/test_cdropout.py
@@ -8,6 +8,7 @@
 from keras import backend as K
 from keras.layers import Input, Dense, Conv1D, Conv2D, Conv3D
 from keras.models import Model
+from keras_contrib.utils.test_utils import layer_test
 from keras_contrib.wrappers import ConcreteDropout
 
 
@@ -202,30 +203,29 @@ def sigmoid(x):
     assert_approx_equal(eval_loss, loss)
 
 
-def test_cdropout_1d_layer():
+@pytest.mark.parametrize('n_data', [1, 60])
+@pytest.mark.parametrize('layer, shape', [(Conv1D(8, 3), (None, 20, 1)),
+                                          (Conv3D(16, 7), (1, 20, 20, 20, 1))])
+def test_cdropout_invalid_layer(layer, shape, n_data):
     """To be replaced with a real function test, if implemented.
     """
-    in_dim = 20
-    init_prop = .1
-
     with pytest.raises(ValueError):
-        inputs = Input(shape=(in_dim, 1,))
-        ConcreteDropout(Conv1D(1, 3),
-                        in_dim,
-                        prob_init=(init_prop, init_prop))(inputs)
+        layer_test(ConcreteDropout,
+                   kwargs={'layer': layer,
+                           'n_data': n_data},
+                   input_shape=shape)
 
 
-def test_cdropout_3d_layer():
-    """To be replaced with a real function test, if implemented.
+@pytest.mark.parametrize('n_data', [1, 60])
+@pytest.mark.parametrize('layer, shape', [(Conv2D(8, 3), (None, 12, 12, 3)),
+                                          (Conv2D(16, 7), (1, 12, 12, 3))])
+def test_cdropout_valid_layer(layer, shape, n_data):
+    """Original layer test with valid parameters.
     """
-    in_dim = 20
-    init_prop = .1
-
-    with pytest.raises(ValueError):
-        inputs = Input(shape=(in_dim, in_dim, in_dim, 1,))
-        ConcreteDropout(Conv3D(1, 3),
-                        in_dim,
-                        prob_init=(init_prop, init_prop))(inputs)
+    layer_test(ConcreteDropout,
+               kwargs={'layer': layer,
+                       'n_data': n_data},
+               input_shape=shape)
 
 
 if __name__ == '__main__':

From 6eb632b42f12f35e4e2d42607ffd24a994db9911 Mon Sep 17 00:00:00 2001
From: Moritz Kirschte <moritz.kirschte@gmail.com>
Date: Wed, 20 Feb 2019 14:45:31 +0100
Subject: [PATCH 9/9] added conv1d & conv2d functionality

---
 keras_contrib/wrappers/cdropout.py            | 33 ++++++++++++-------
 tests/keras_contrib/wrappers/test_cdropout.py | 30 +++++++----------
 2 files changed, 32 insertions(+), 31 deletions(-)

diff --git a/keras_contrib/wrappers/cdropout.py b/keras_contrib/wrappers/cdropout.py
index 11bf843c8..a7c89a8b7 100644
--- a/keras_contrib/wrappers/cdropout.py
+++ b/keras_contrib/wrappers/cdropout.py
@@ -13,6 +13,11 @@ class ConcreteDropout(Wrapper):
     """A wrapper automating the dropout rate choice
        through the 'Concrete Dropout' technique.
 
+       Note that currently only Dense layers with weights
+       and Conv layers (Conv1D, Conv2D, Conv3D) are supported.
+       In the case of Dense Layers, dropout is applied to its complete input,
+       whereas in the Conv case just the input-channels are dropped.
+
     # Example
 
     ```python
@@ -24,7 +29,7 @@ class ConcreteDropout(Wrapper):
         model.add(ConcreteDropout(Dense(32), n_data=500))
         # now model.output_shape == (None, 32)
 
-        # Note that the current implementation supports Conv2D Layer as well.
+        # Note that the current implementation supports Conv layers as well.
     ```
 
     # Arguments
@@ -83,19 +88,21 @@ def _concrete_dropout(self, inputs, layer_type):
 
         # Arguments
             inputs: Input.
-            layer_type: str. Either 'dense' or 'conv2d'.
+            layer_type: str. Either 'dense' or 'conv'.
         # Returns
             A tensor with the same shape as inputs and dropout applied.
         """
-        assert layer_type in {'dense', 'conv2d'}
+        assert layer_type in {'dense', 'conv'}
         eps = K.cast_to_floatx(K.epsilon())
 
         noise_shape = K.shape(inputs)
-        if layer_type == 'conv2d':
+        if layer_type == 'conv':
+            nodrops = np.ones(len(K.int_shape(inputs)) - 2, int)
+            _ = lambda *x: x  # don't ask... py2 can't unpack directly into a tuple
             if K.image_data_format() == 'channels_first':
-                noise_shape = (noise_shape[0], noise_shape[1], 1, 1)
+                noise_shape = _(noise_shape[0], noise_shape[1], *nodrops)
             else:
-                noise_shape = (noise_shape[0], 1, 1, noise_shape[3])
+                noise_shape = _(noise_shape[0], *(_(*nodrops) + (noise_shape[-1],)))
         unif_noise = K.random_uniform(shape=noise_shape,
                                       seed=self._seed,
                                       dtype=inputs.dtype)
@@ -119,13 +126,15 @@ def build(self, input_shape=None):
         input_shape = to_tuple(input_shape)
         if len(input_shape) == 2:  # Dense_layer
             input_dim = np.prod(input_shape[-1])  # we drop only last dim
-        elif len(input_shape) == 4:  # Conv2D_layer
-            input_dim = (input_shape[1]
-                         if K.image_data_format() == 'channels_first'
-                         else input_shape[3])  # we drop only channels
+        elif 3 <= len(input_shape) <= 5:  # Conv_layers
+            input_dim = (
+                input_shape[1]
+                if K.image_data_format() == 'channels_first'
+                else input_shape[-1]  # we drop only channels
+            )
         else:
             raise ValueError(
-                'concrete_dropout currenty supports only Dense/Conv2D layers')
+                'concrete_dropout currenty supports only Dense/Conv layers')
 
         self.input_spec = InputSpec(shape=input_shape)
         if not self.layer.built:
@@ -161,7 +170,7 @@ def build(self, input_shape=None):
     def call(self, inputs, training=None):
         def relaxed_dropped_inputs():
             return self.layer.call(self._concrete_dropout(inputs, (
-                'dense' if len(K.int_shape(inputs)) == 2 else 'conv2d'
+                'dense' if len(K.int_shape(inputs)) == 2 else 'conv'
             )))
 
         return K.in_train_phase(relaxed_dropped_inputs,
diff --git a/tests/keras_contrib/wrappers/test_cdropout.py b/tests/keras_contrib/wrappers/test_cdropout.py
index 6f44852b3..88e15f65d 100644
--- a/tests/keras_contrib/wrappers/test_cdropout.py
+++ b/tests/keras_contrib/wrappers/test_cdropout.py
@@ -6,7 +6,7 @@
 from numpy.testing import assert_approx_equal
 from numpy.testing import assert_equal
 from keras import backend as K
-from keras.layers import Input, Dense, Conv1D, Conv2D, Conv3D
+from keras.layers import Conv1D, Conv2D, Conv3D, Dense, Input
 from keras.models import Model
 from keras_contrib.utils.test_utils import layer_test
 from keras_contrib.wrappers import ConcreteDropout
@@ -204,29 +204,21 @@ def sigmoid(x):
 
 
 @pytest.mark.parametrize('n_data', [1, 60])
-@pytest.mark.parametrize('layer, shape', [(Conv1D(8, 3), (None, 20, 1)),
-                                          (Conv3D(16, 7), (1, 20, 20, 20, 1))])
-def test_cdropout_invalid_layer(layer, shape, n_data):
-    """To be replaced with a real function test, if implemented.
-    """
-    with pytest.raises(ValueError):
-        layer_test(ConcreteDropout,
-                   kwargs={'layer': layer,
-                           'n_data': n_data},
-                   input_shape=shape)
-
-
-@pytest.mark.parametrize('n_data', [1, 60])
-@pytest.mark.parametrize('layer, shape', [(Conv2D(8, 3), (None, 12, 12, 3)),
-                                          (Conv2D(16, 7), (1, 12, 12, 3))])
-def test_cdropout_valid_layer(layer, shape, n_data):
-    """Original layer test with valid parameters.
+@pytest.mark.parametrize('layer, args, shape', [(Dense, (2,), (None, 6)),
+                                                (Conv1D, (4, 3), (None, 6, 1)),
+                                                (Conv2D, (8, 7), (None, 12, 12, 3)),
+                                                (Conv3D, (16, 3), (1, 6, 6, 6, 1))])
+def test_cdropout_valid_layer(layer, args, shape, n_data):
+    """Original layer test with a variety of different valid parameters.
     """
     layer_test(ConcreteDropout,
-               kwargs={'layer': layer,
+               kwargs={'layer': layer(*args),
                        'n_data': n_data},
                input_shape=shape)
 
+    if K.backend() == 'tensorflow' or K.backend() == 'cntk':
+        K.clear_session()
+
 
 if __name__ == '__main__':
     pytest.main([__file__])