From 5b44d71b04a4cd21a5b63c081f0254fec7c4e754 Mon Sep 17 00:00:00 2001 From: Zhao HG Date: Sun, 12 May 2019 00:49:50 +0800 Subject: [PATCH] Support tf.keras (#57) * #55 Merge outputs as feature * Update README.md * Support tf.keras * Fix masked layer * Fix masked layer --- .travis.yml | 16 +++++++++--- README.md | 4 +++ README.zh-CN.md | 4 +++ keras_bert/backend.py | 46 +++++++++++++++++++++++++++++++++ keras_bert/bert.py | 8 +++--- keras_bert/layers/conv.py | 4 +-- keras_bert/layers/embedding.py | 6 ++--- keras_bert/layers/extract.py | 2 +- keras_bert/layers/inputs.py | 2 +- keras_bert/layers/masked.py | 11 +++++--- keras_bert/layers/pooling.py | 4 +-- keras_bert/loader.py | 2 +- keras_bert/optimizers/warmup.py | 4 +-- lint.sh | 2 -- requirements.txt | 4 +-- setup.py | 2 +- test.sh | 3 ++- tests/layers/test_conv.py | 2 +- tests/layers/test_embedding.py | 2 +- tests/layers/test_extract.py | 2 +- tests/layers/test_masked.py | 4 +-- tests/layers/test_pooling.py | 2 +- tests/optimizers/test_warmup.py | 2 +- tests/test_bert.py | 2 +- 24 files changed, 104 insertions(+), 36 deletions(-) create mode 100644 keras_bert/backend.py delete mode 100755 lint.sh diff --git a/.travis.yml b/.travis.yml index e6dcb33..104440f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,10 +2,12 @@ dist: xenial language: python python: - 2.7 - - 3.7 + - 3.6 env: - KERAS_BACKEND=tensorflow + - KERAS_BACKEND=tensorflow TF_KERAS=1 - KERAS_BACKEND=theano THEANO_FLAGS=optimizer=fast_compile + # - KERAS_BACKEND=cntk PYTHONWARNINGS=ignore install: - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then wget https://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh; @@ -24,9 +26,17 @@ install: - pip install -r requirements.txt - pip install -r requirements-dev.txt - if [[ $KERAS_BACKEND == "theano" ]]; then pip install theano && conda install mkl mkl-service; fi + - if [[ "$KERAS_BACKEND" == "cntk" ]]; then + set -e && + pip install cntk && + mkdir -p ~/mpi && + pushd ~/mpi && + wget http://cntk.ai/PythonWheel/ForKeras/depends/openmpi_1.10-3.zip && + unzip ./openmpi_1.10-3.zip && + sudo dpkg -i openmpi_1.10-3.deb && + popd; + fi - pip install coveralls -before_script: - - ./lint.sh script: - ./test.sh after_success: diff --git a/README.md b/README.md index 1f233fd..4988dfc 100644 --- a/README.md +++ b/README.md @@ -122,3 +122,7 @@ inputs, output_layer = get_model( Only available when `training` is `False`. ) ``` + +### Use `tensorflow.python.keras` + +Add `TF_KERAS=1` to environment variables to use `tensorflow.python.keras`. diff --git a/README.zh-CN.md b/README.zh-CN.md index 053c168..c8ef33e 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -126,3 +126,7 @@ inputs, output_layer = get_model( output_layer_num=4, # 最后几层的输出将合并在一起作为最终的输出，只有当`training`是`False`有效 ) ``` + +### 使用`tensorflow.python.keras` + +在环境变量里加入`TF_KERAS=1`可以启用`tensorflow.python.keras`。 diff --git a/keras_bert/backend.py b/keras_bert/backend.py new file mode 100644 index 0000000..e6b9f58 --- /dev/null +++ b/keras_bert/backend.py @@ -0,0 +1,46 @@ +import os + +__all__ = [ + 'keras', 'utils', 'activations', 'applications', 'backend', 'datasets', 'engine', + 'layers', 'preprocessing', 'wrappers', 'callbacks', 'constraints', 'initializers', + 'metrics', 'models', 'losses', 'optimizers', 'regularizers', +] + +if 'TF_KERAS' in os.environ and os.environ['TF_KERAS'] != '0': + from tensorflow.python import keras + from tensorflow.python.keras import utils + from tensorflow.python.keras import activations + from tensorflow.python.keras import applications + from tensorflow.python.keras import backend + from tensorflow.python.keras import datasets + from tensorflow.python.keras import engine + from tensorflow.python.keras import layers + from tensorflow.python.keras import preprocessing + from tensorflow.python.keras import wrappers + from tensorflow.python.keras import callbacks + from tensorflow.python.keras import constraints + from tensorflow.python.keras import initializers + from tensorflow.python.keras import metrics + from tensorflow.python.keras import models + from tensorflow.python.keras import losses + from tensorflow.python.keras import optimizers + from tensorflow.python.keras import regularizers +else: + import keras + from keras import utils + from keras import activations + from keras import applications + from keras import backend + from keras import datasets + from keras import engine + from keras import layers + from keras import preprocessing + from keras import wrappers + from keras import callbacks + from keras import constraints + from keras import initializers + from keras import metrics + from keras import models + from keras import losses + from keras import optimizers + from keras import regularizers diff --git a/keras_bert/bert.py b/keras_bert/bert.py index 90e19e3..651b28b 100644 --- a/keras_bert/bert.py +++ b/keras_bert/bert.py @@ -1,11 +1,12 @@ import math -import keras -import keras.backend as K + import numpy as np from keras_pos_embd import PositionEmbedding from keras_layer_normalization import LayerNormalization from keras_transformer import get_encoders from keras_transformer import get_custom_objects as get_encoder_custom_objects +from .backend import keras +from .backend import backend as K from .layers import get_inputs, get_embedding, TokenEmbedding, EmbeddingSimilarity, Masked, Extract from .optimizers import AdamWarmup @@ -25,7 +26,8 @@ def gelu(x): if K.backend() == 'tensorflow': - return 0.5 * x * (1.0 + K.tf.erf(x / K.tf.sqrt(2.0))) + import tensorflow as tf + return 0.5 * x * (1.0 + tf.erf(x / tf.sqrt(2.0))) return 0.5 * x * (1.0 + K.tanh(math.sqrt(2.0 / math.pi) * (x + 0.044715 * K.pow(x, 3)))) diff --git a/keras_bert/layers/conv.py b/keras_bert/layers/conv.py index a19c5cf..2344c64 100644 --- a/keras_bert/layers/conv.py +++ b/keras_bert/layers/conv.py @@ -1,5 +1,5 @@ -import keras -import keras.backend as K +from keras_bert.backend import keras +from keras_bert.backend import backend as K class MaskedConv1D(keras.layers.Conv1D): diff --git a/keras_bert/layers/embedding.py b/keras_bert/layers/embedding.py index a58a85d..1ea3ea2 100644 --- a/keras_bert/layers/embedding.py +++ b/keras_bert/layers/embedding.py @@ -1,5 +1,5 @@ -import keras -import keras.backend as K +from keras_bert.backend import keras +from keras_bert.backend import backend as K from keras_pos_embd import PositionEmbedding from keras_layer_normalization import LayerNormalization @@ -102,7 +102,7 @@ def get_config(self): def build(self, input_shape): self.bias = self.add_weight( - shape=(input_shape[1][0],), + shape=(int(input_shape[1][0]),), initializer=self.initializer, regularizer=self.regularizer, constraint=self.constraint, diff --git a/keras_bert/layers/extract.py b/keras_bert/layers/extract.py index 81bad5e..4c953a7 100644 --- a/keras_bert/layers/extract.py +++ b/keras_bert/layers/extract.py @@ -1,4 +1,4 @@ -import keras +from keras_bert.backend import keras class Extract(keras.layers.Layer): diff --git a/keras_bert/layers/inputs.py b/keras_bert/layers/inputs.py index 529e192..5c2030b 100644 --- a/keras_bert/layers/inputs.py +++ b/keras_bert/layers/inputs.py @@ -1,4 +1,4 @@ -import keras +from keras_bert.backend import keras def get_inputs(seq_len): diff --git a/keras_bert/layers/masked.py b/keras_bert/layers/masked.py index b549f2f..141bbbe 100644 --- a/keras_bert/layers/masked.py +++ b/keras_bert/layers/masked.py @@ -1,5 +1,5 @@ -import keras -import keras.backend as K +from keras_bert.backend import keras +from keras_bert.backend import backend as K class Masked(keras.layers.Layer): @@ -36,9 +36,12 @@ def compute_output_shape(self, input_shape): def compute_mask(self, inputs, mask=None): token_mask = K.not_equal(inputs[1], 0) - return K.all(K.stack([token_mask, mask[0]], axis=0), axis=0) + masked = K.all(K.stack([token_mask, mask[0]], axis=0), axis=0) + if self.return_masked: + return [masked, None] + return masked def call(self, inputs, mask=None, **kwargs): if self.return_masked: - return [inputs[0], K.cast(self.compute_mask(inputs, mask), K.floatx())] + return [inputs[0], K.cast(self.compute_mask(inputs, mask)[0], K.floatx())] return inputs[0] diff --git a/keras_bert/layers/pooling.py b/keras_bert/layers/pooling.py index 195a3ee..eb37eeb 100644 --- a/keras_bert/layers/pooling.py +++ b/keras_bert/layers/pooling.py @@ -1,5 +1,5 @@ -import keras -import keras.backend as K +from keras_bert.backend import keras +from keras_bert.backend import backend as K class MaskedGlobalMaxPool1D(keras.layers.Layer): diff --git a/keras_bert/loader.py b/keras_bert/loader.py index df1ed35..fd73fbc 100644 --- a/keras_bert/loader.py +++ b/keras_bert/loader.py @@ -1,7 +1,7 @@ import json -import keras import numpy as np import tensorflow as tf +from .backend import keras from .bert import get_model diff --git a/keras_bert/optimizers/warmup.py b/keras_bert/optimizers/warmup.py index b8911fa..6adfcdf 100644 --- a/keras_bert/optimizers/warmup.py +++ b/keras_bert/optimizers/warmup.py @@ -1,5 +1,5 @@ -import keras -import keras.backend as K +from keras_bert.backend import keras +from keras_bert.backend import backend as K class AdamWarmup(keras.optimizers.Optimizer): diff --git a/lint.sh b/lint.sh deleted file mode 100755 index 4ddd0ce..0000000 --- a/lint.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env bash -pycodestyle --max-line-length=120 keras_bert tests diff --git a/requirements.txt b/requirements.txt index 1671c4b..8a37efe 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ numpy Keras -keras-pos-embd==0.9.0 -keras-transformer==0.21.0 +keras-pos-embd==0.10.0 +keras-transformer==0.22.0 diff --git a/setup.py b/setup.py index 01d43c7..e10d697 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ setup( name='keras-bert', - version='0.42.0', + version='0.43.0', packages=find_packages(), url='https://github.com/CyberZHG/keras-bert', license='MIT', diff --git a/test.sh b/test.sh index 0eb4e6d..77348b5 100755 --- a/test.sh +++ b/test.sh @@ -1,2 +1,3 @@ #!/usr/bin/env bash -nosetests --with-coverage --cover-erase --cover-html --cover-html-dir=htmlcov --cover-package="keras_bert" tests \ No newline at end of file +pycodestyle --max-line-length=120 keras_bert tests && \ + nosetests --with-coverage --cover-erase --cover-html --cover-html-dir=htmlcov --cover-package=keras_bert tests \ No newline at end of file diff --git a/tests/layers/test_conv.py b/tests/layers/test_conv.py index 3ad6b47..b12b2bd 100644 --- a/tests/layers/test_conv.py +++ b/tests/layers/test_conv.py @@ -1,6 +1,6 @@ from unittest import TestCase -import keras import numpy as np +from keras_bert.backend import keras from keras_bert.layers import MaskedConv1D, MaskedGlobalMaxPool1D diff --git a/tests/layers/test_embedding.py b/tests/layers/test_embedding.py index d832d7e..1e26fb8 100644 --- a/tests/layers/test_embedding.py +++ b/tests/layers/test_embedding.py @@ -1,5 +1,5 @@ import unittest -import keras +from keras_bert.backend import keras from keras_bert.layers import get_inputs, get_embedding diff --git a/tests/layers/test_extract.py b/tests/layers/test_extract.py index f17ed6c..c39beb6 100644 --- a/tests/layers/test_extract.py +++ b/tests/layers/test_extract.py @@ -1,6 +1,6 @@ import unittest -import keras import numpy as np +from keras_bert.backend import keras from keras_bert.layers import Extract diff --git a/tests/layers/test_masked.py b/tests/layers/test_masked.py index c8f1b9f..742e4d5 100644 --- a/tests/layers/test_masked.py +++ b/tests/layers/test_masked.py @@ -1,7 +1,7 @@ import unittest -import keras import numpy as np from keras_transformer import gelu, get_encoders +from keras_bert.backend import keras from keras_bert.layers import get_inputs, get_embedding, Masked @@ -58,6 +58,7 @@ def test_mask_result(self): return_masked=True, name='Masked', )([dense_layer, mask_layer]) + print([masked_layer, mask_result]) model = keras.models.Model( inputs=[input_layer, mask_layer], outputs=[masked_layer, mask_result], @@ -65,7 +66,6 @@ def test_mask_result(self): model.compile( optimizer='adam', loss='mse', - metrics=['mse'], ) model.summary(line_length=150) predicts = model.predict([ diff --git a/tests/layers/test_pooling.py b/tests/layers/test_pooling.py index a007b44..61d8aa8 100644 --- a/tests/layers/test_pooling.py +++ b/tests/layers/test_pooling.py @@ -1,6 +1,6 @@ from unittest import TestCase -import keras import numpy as np +from keras_bert.backend import keras from keras_bert.layers import MaskedGlobalMaxPool1D diff --git a/tests/optimizers/test_warmup.py b/tests/optimizers/test_warmup.py index 96d414e..d26dcee 100644 --- a/tests/optimizers/test_warmup.py +++ b/tests/optimizers/test_warmup.py @@ -1,6 +1,6 @@ from unittest import TestCase -import keras import numpy as np +from keras_bert.backend import keras from keras_bert.optimizers import AdamWarmup diff --git a/tests/test_bert.py b/tests/test_bert.py index 86b3f2a..0c3b147 100644 --- a/tests/test_bert.py +++ b/tests/test_bert.py @@ -1,8 +1,8 @@ import unittest import os import tempfile -import keras import numpy as np +from keras_bert.backend import keras from keras_bert import gelu, get_model, get_custom_objects, get_base_dict, gen_batch_inputs