diff --git a/keras_contrib/wrappers/__init__.py b/keras_contrib/wrappers/__init__.py index dffcc152d..7490e705c 100644 --- a/keras_contrib/wrappers/__init__.py +++ b/keras_contrib/wrappers/__init__.py @@ -1,3 +1,3 @@ from __future__ import absolute_import -from .cdropout import ConcreteDropout \ No newline at end of file +from .cdropout import ConcreteDropout diff --git a/keras_contrib/wrappers/cdropout.py b/keras_contrib/wrappers/cdropout.py index 7a5e68af9..d24b547db 100644 --- a/keras_contrib/wrappers/cdropout.py +++ b/keras_contrib/wrappers/cdropout.py @@ -2,10 +2,9 @@ from __future__ import absolute_import import numpy as np - from keras import backend as K -from keras.engine import InputSpec from keras.initializers import RandomUniform +from keras.layers import InputSpec from keras.layers.wrappers import Wrapper @@ -34,7 +33,7 @@ class ConcreteDropout(Wrapper): model_precision: float. Model precision parameter is `1` for classification. Also known as inverse observation noise. prob_init: Tuple[float, float]. - Probability lower / upper bounds of dropout rate initialization. + Probability lower / upper bounds of dropout rate initialization. temp: float. Temperature. Not used to be optimized. seed: Seed for random probability sampling. @@ -156,7 +155,7 @@ def relaxed_dropped_inputs(): def get_config(self): config = {'weight_regularizer': self.weight_regularizer, 'dropout_regularizer': self.dropout_regularizer, - 'prob_init': self.prob_init, + 'prob_init': tuple(np.round(self.prob_init, 8)), 'temp': self.temp, 'seed': self.seed} base_config = super(ConcreteDropout, self).get_config() diff --git a/tests/keras_contrib/wrappers/test_cdropout.py b/tests/keras_contrib/wrappers/test_cdropout.py index 31f263788..a5fb0d15b 100644 --- a/tests/keras_contrib/wrappers/test_cdropout.py +++ b/tests/keras_contrib/wrappers/test_cdropout.py @@ -1,13 +1,12 @@ import pytest import numpy as np -from keras.layers import Input, Dense -from keras.models import Model from numpy.testing import assert_allclose from numpy.testing import assert_array_almost_equal from numpy.testing import assert_approx_equal from numpy.testing import assert_equal - +from keras.layers import Input, Dense, Conv2D, Conv1D +from keras.models import Model from keras_contrib.wrappers import ConcreteDropout @@ -20,7 +19,7 @@ def test_cdropout(): # Model inputs = Input(shape=(in_dim,)) - dense = Dense(1, use_bias=True, input_shape=(in_dim,)) + dense = Dense(1, use_bias=True) # Model, normal cd = ConcreteDropout(dense, in_dim, prob_init=(init_prop, init_prop)) x = cd(inputs) @@ -55,5 +54,61 @@ def sigmoid(x): assert_approx_equal(eval_loss, loss) +def test_cdropout_conv(): + # Data + in_dim = 20 + init_prop = .1 + np.random.seed(1) + X = np.random.randn(1, in_dim, in_dim, 1) + + # Model + inputs = Input(shape=(in_dim, in_dim, 1,)) + conv2d = Conv2D(1, (3, 3)) + # Model, normal + cd = ConcreteDropout(conv2d, in_dim, prob_init=(init_prop, init_prop)) + x = cd(inputs) + model = Model(inputs, x) + model.compile(loss=None, optimizer='rmsprop') + # Model, reference w/o Dropout + x_ref = conv2d(inputs) + model_ref = Model(inputs, x_ref) + model_ref.compile(loss=None, optimizer='rmsprop') + + # Check about correct 3rd weight (equal to initial value) + W = model.get_weights() + assert_array_almost_equal(W[2], [np.log(init_prop)]) + + # Check if ConcreteDropout in prediction phase is the same as no dropout + out = model.predict(X) + out_ref = model_ref.predict(X) + assert_allclose(out, out_ref, atol=1e-5) + + # Check if ConcreteDropout has the right amount of losses deposited + assert_equal(len(model.losses), 1) + + # Check if the loss correspons the the desired value + def sigmoid(x): + return 1. / (1. + np.exp(-x)) + p = np.squeeze(sigmoid(W[2])) + kernel_regularizer = cd.weight_regularizer * np.sum(np.square(W[0])) / (1. - p) + dropout_regularizer = (p * np.log(p) + (1. - p) * np.log(1. - p)) + dropout_regularizer *= cd.dropout_regularizer * 1 # only channels are dropped + loss = np.sum(kernel_regularizer + dropout_regularizer) + eval_loss = model.evaluate(X) + assert_approx_equal(eval_loss, loss) + + +def test_cdropout_wrong_layertype(): + """To be replaced with a real function test, if implemented. + """ + in_dim = 20 + init_prop = .1 + with pytest.raises(ValueError): + inputs = Input(shape=(in_dim, in_dim,)) + ConcreteDropout(Conv1D(1, 3), + in_dim, + prob_init=(init_prop, init_prop))(inputs) + + if __name__ == '__main__': pytest.main([__file__])