Add UNetPlusPlus and UNet conversion (#666)

* Add env variable UPSAMPLE_COORDINATE_TRANSFORMATION_MODE

* UNetPlusPlus

applications/nightly_build/test_chatbot.py

@@ -101,7 +101,7 @@ def test_chatbot(self):
         expected = keras_model.predict([data1, data2])
         onnx_model = keras2onnx.convert_keras(keras_model, keras_model.name)
         self.assertTrue(
-            run_keras_and_ort(onnx_model.graph.name, onnx_model, keras_model, [data1, data2], expected, self.model_files, compare_perf=True))
+            run_keras_and_ort(onnx_model.graph.name, onnx_model, keras_model, [data1, data2], expected, self.model_files))
 
 
 if __name__ == "__main__":

applications/nightly_build/test_crnn.py

@@ -209,7 +209,7 @@ def test_CRNN_GRU(self):
         expected = model.predict(data)
         onnx_model = keras2onnx.convert_keras(model, model.name)
         self.assertTrue(
-            run_keras_and_ort(onnx_model.graph.name, onnx_model, model, data, expected, self.model_files, compare_perf=True))
+            run_keras_and_ort(onnx_model.graph.name, onnx_model, model, data, expected, self.model_files))
 
 
 if __name__ == "__main__":

applications/nightly_build/test_deep_rl.py

@@ -78,7 +78,7 @@ def test_DPPG_actor(self):
         expected = keras_model.predict(data)
         onnx_model = keras2onnx.convert_keras(keras_model, keras_model.name)
         self.assertTrue(
-            run_keras_and_ort(onnx_model.graph.name, onnx_model, keras_model, data, expected, self.model_files, compare_perf=True))
+            run_keras_and_ort(onnx_model.graph.name, onnx_model, keras_model, data, expected, self.model_files))
 
     @unittest.skipIf(test_level_0,
                      "Test level 0 only.")

applications/nightly_build/test_efn.py

@@ -32,14 +32,14 @@ def test_custom(self):
         base_model = efn.EfficientNetB0(input_shape=(600, 600, 3), weights=None)
         backbone = keras.Model(base_model.input, base_model.get_layer("top_activation").output)
         res = run_image(backbone, self.model_files, img_path, target_size=(600, 600),
-                        rtol=1e-2, atol=1e-1, tf_v2=True)
+                        rtol=1e-2, atol=1e-1)
         self.assertTrue(*res)
 
     def test_efn(self):
         from efficientnet import tfkeras as efn
         keras.backend.set_learning_phase(0)
         model = efn.EfficientNetB0(weights=None)
-        res = run_image(model, self.model_files, img_path, target_size=(224, 224), rtol=1e-2, tf_v2=True)
+        res = run_image(model, self.model_files, img_path, target_size=(224, 224), rtol=1e-2)
         self.assertTrue(*res)
 
 

applications/nightly_build/test_keras_applications.py

@@ -256,7 +256,7 @@ def test_DeepFashion(self):
 
         ## Create Model
         keras_model = Model(inputs=model_inputs, outputs=[predictions_class, predictions_iou])
-        res = run_image(keras_model, self.model_files, img_path, atol=5e-3, target_size=224, compare_perf=True)
+        res = run_image(keras_model, self.model_files, img_path, atol=5e-3, target_size=224)
         self.assertTrue(*res)
 
     # Model from https://github.com/manicman1999/Keras-BiGAN

applications/nightly_build/test_keras_applications_v2.py

@@ -29,50 +29,50 @@ def tearDown(self):
     def test_DenseNet121(self):
         DenseNet121 = keras.applications.densenet.DenseNet121
         model = DenseNet121(include_top=True, weights=None)
-        res = run_image(model, self.model_files, img_path, tf_v2=True)
+        res = run_image(model, self.model_files, img_path)
         self.assertTrue(*res)
 
     def test_MobileNet(self):
         MobileNet = keras.applications.mobilenet.MobileNet
         model = MobileNet(weights=None)
-        res = run_image(model, self.model_files, img_path, tf_v2=True)
+        res = run_image(model, self.model_files, img_path)
         self.assertTrue(*res)
 
     def test_MobileNetV2(self):
         MobileNetV2 = keras.applications.mobilenet_v2.MobileNetV2
         model = MobileNetV2(weights=None)
-        res = run_image(model, self.model_files, img_path, tf_v2=True)
+        res = run_image(model, self.model_files, img_path)
         self.assertTrue(*res)
 
     def test_NASNetMobile(self):
         NASNetMobile = keras.applications.nasnet.NASNetMobile
         model = NASNetMobile(weights=None)
-        res = run_image(model, self.model_files, img_path, tf_v2=True)
+        res = run_image(model, self.model_files, img_path)
         self.assertTrue(*res)
 
     def test_InceptionV3(self):
         keras.backend.set_learning_phase(0)
         InceptionV3 = keras.applications.inception_v3.InceptionV3
         model = InceptionV3(include_top=True)
-        res = run_image(model, self.model_files, img_path, target_size=299, tf_v2=True)
+        res = run_image(model, self.model_files, img_path, target_size=299)
         self.assertTrue(*res)
 
     def test_InceptionResNetV2(self):
         InceptionResNetV2 = keras.applications.inception_resnet_v2.InceptionResNetV2
         model = InceptionResNetV2(include_top=True)
-        res = run_image(model, self.model_files, img_path, target_size=299, tf_v2=True)
+        res = run_image(model, self.model_files, img_path, target_size=299)
         self.assertTrue(*res)
 
     def test_ResNet50(self):
         ResNet50 = keras.applications.resnet_v2.ResNet50V2
         model = ResNet50(include_top=True, weights=None)
-        res = run_image(model, self.model_files, img_path, tf_v2=True)
+        res = run_image(model, self.model_files, img_path)
         self.assertTrue(*res)
 
     def test_Xception(self):
         Xception = keras.applications.xception.Xception
         model = Xception(include_top=True, weights=None)
-        res = run_image(model, self.model_files, img_path, atol=5e-3, target_size=299, tf_v2=True)
+        res = run_image(model, self.model_files, img_path, atol=5e-3, target_size=299)
         self.assertTrue(*res)
 
 

applications/nightly_build/test_name_entity_recognition.py

@@ -82,7 +82,7 @@ def test_name_entity_recognition(self):
             run_keras_and_ort(onnx_model.graph.name, onnx_model, keras_model,
                               {keras_model.input_names[0]: data1,
                                keras_model.input_names[1]: data2,
-                               keras_model.input_names[2]: data3}, expected, self.model_files, compare_perf=True))
+                               keras_model.input_names[2]: data3}, expected, self.model_files))
 
 
 if __name__ == "__main__":

applications/nightly_build/test_nbeats.py

@@ -264,7 +264,7 @@ def test_NBeats(self):
         expected = keras_model.predict(data)
         onnx_model = keras2onnx.convert_keras(keras_model, keras_model.name)
         self.assertTrue(
-            run_keras_and_ort(onnx_model.graph.name, onnx_model, keras_model, data, expected, self.model_files, compare_perf=True))
+            run_keras_and_ort(onnx_model.graph.name, onnx_model, keras_model, data, expected, self.model_files))
 
 
 if __name__ == "__main__":

applications/nightly_build/test_resnext.py

@@ -353,7 +353,7 @@ def test_ResNext(self):
         inputs = img_input
 
         keras_model = Model(inputs, x, name='resnext')
-        res = run_image(keras_model, self.model_files, img_path, atol=5e-3, target_size=112, tf_v2=True)
+        res = run_image(keras_model, self.model_files, img_path, atol=5e-3, target_size=112)
         self.assertTrue(*res)
 
     # Model from https://github.com/titu1994/keras-squeeze-excite-network
@@ -377,7 +377,7 @@ def test_SEResNext(self):
         inputs = img_input
 
         keras_model = Model(inputs, x, name='se_resnext')
-        res = run_image(keras_model, self.model_files, img_path, atol=5e-3, target_size=112, tf_v2=True)
+        res = run_image(keras_model, self.model_files, img_path, atol=5e-3, target_size=112)
         self.assertTrue(*res)
 
 

applications/nightly_build/test_series_net.py

@@ -114,7 +114,7 @@ def test_series_net(self):
         expected = keras_model.predict(data)
         onnx_model = keras2onnx.convert_keras(keras_model, keras_model.name)
         self.assertTrue(
-            run_keras_and_ort(onnx_model.graph.name, onnx_model, keras_model, data, expected, self.model_files, compare_perf=True))
+            run_keras_and_ort(onnx_model.graph.name, onnx_model, keras_model, data, expected, self.model_files))
 
 
 if __name__ == "__main__":

applications/nightly_build/test_unet.py

@@ -9,20 +9,60 @@
 import keras_segmentation
 from os.path import dirname, abspath
 from keras2onnx.proto import keras, is_keras_older_than
+from onnxconverter_common.onnx_ex import get_maximum_opset_supported
 
 sys.path.insert(0, os.path.join(dirname(abspath(__file__)), '../../tests/'))
 from test_utils import run_image
 img_path = os.path.join(os.path.dirname(__file__), '../data', 'street.jpg')
 
 
 Input = keras.layers.Input
+Concatenate = keras.layers.Concatenate
 concatenate = keras.layers.concatenate
 Conv2D = keras.layers.Conv2D
 Conv2DTranspose = keras.layers.Conv2DTranspose
+Dropout = keras.layers.Dropout
 MaxPooling2D = keras.layers.MaxPooling2D
+UpSampling2D = keras.layers.UpSampling2D
 
 Model = keras.models.Model
 
+def get_unet_model(input_channel_num=3, out_ch=3, start_ch=64, depth=4, inc_rate=2., activation='relu',
+         dropout=0.5, batchnorm=False, maxpool=True, upconv=True, residual=False):
+    def _conv_block(m, dim, acti, bn, res, do=0):
+        n = Conv2D(dim, 3, activation=acti, padding='same')(m)
+        n = BatchNormalization()(n) if bn else n
+        n = Dropout(do)(n) if do else n
+        n = Conv2D(dim, 3, activation=acti, padding='same')(n)
+        n = BatchNormalization()(n) if bn else n
+
+        return Concatenate()([m, n]) if res else n
+
+    def _level_block(m, dim, depth, inc, acti, do, bn, mp, up, res):
+        if depth > 0:
+            #n = _conv_block(m, dim, acti, bn, res)
+            n = m
+            m = MaxPooling2D()(n) if mp else Conv2D(dim, 3, strides=2, padding='same')(n)
+            m = _level_block(m, int(inc * dim), depth - 1, inc, acti, do, bn, mp, up, res)
+            if up:
+                m = UpSampling2D()(m)
+                m = Conv2D(dim, 2, activation=acti, padding='same')(m)
+            else:
+                m = Conv2DTranspose(dim, 3, strides=2, activation=acti, padding='same')(m)
+            n = Concatenate()([n, m])
+            m = _conv_block(n, dim, acti, bn, res)
+        else:
+            m = _conv_block(m, dim, acti, bn, res, do)
+
+        return m
+
+    i = Input(shape=(None, None, input_channel_num))
+    o = _level_block(i, start_ch, depth, inc_rate, activation, dropout, batchnorm, maxpool, upconv, residual)
+    o = Conv2D(out_ch, 1)(o)
+    model = Model(inputs=i, outputs=o)
+
+    return model
+
 class TestUnet(unittest.TestCase):
 
     def setUp(self):
@@ -87,5 +127,13 @@ def test_unet_2(self):
         res = run_image(model, self.model_files, img_path, color_mode="grayscale", target_size=(img_rows, img_cols))
         self.assertTrue(*res)
 
+    @unittest.skipIf(get_maximum_opset_supported() < 14,
+                     "Need ConvTranspose-14 support.")
+    def test_unet_3(self):
+        # From https://github.com/yu4u/noise2noise/blob/master/model.py
+        model = get_unet_model(out_ch=3, upconv=False)
+        res = run_image(model, self.model_files, img_path, target_size=(256, 256, 3))
+        self.assertTrue(*res)
+
 if __name__ == "__main__":
     unittest.main()

applications/nightly_build/test_unet_plus_plus.py

@@ -0,0 +1,162 @@
+###############################################################################
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+###############################################################################
+import os
+import sys
+import unittest
+import numpy as np
+from os.path import dirname, abspath
+from keras2onnx.proto import keras, is_keras_older_than
+from keras.applications.vgg16 import VGG16
+from onnxconverter_common.onnx_ex import get_maximum_opset_supported
+
+sys.path.insert(0, os.path.join(dirname(abspath(__file__)), '../../tests/'))
+from test_utils import run_image
+img_path = os.path.join(os.path.dirname(__file__), '../data', 'street.jpg')
+
+
+Input = keras.layers.Input
+Activation = keras.layers.Activation
+Concatenate = keras.layers.Concatenate
+Conv2D = keras.layers.Conv2D
+Conv2DTranspose = keras.layers.Conv2DTranspose
+MaxPooling2D = keras.layers.MaxPooling2D
+BatchNormalization = keras.layers.BatchNormalization
+
+Model = keras.models.Model
+
+def handle_block_names(stage):
+    conv_name = 'decoder_stage{}_conv'.format(stage)
+    bn_name = 'decoder_stage{}_bn'.format(stage)
+    relu_name = 'decoder_stage{}_relu'.format(stage)
+    up_name = 'decoder_stage{}_upsample'.format(stage)
+    return conv_name, bn_name, relu_name, up_name
+
+def ConvRelu(filters, kernel_size, use_batchnorm=False, conv_name='conv', bn_name='bn', relu_name='relu'):
+    def layer(x):
+        x = Conv2D(filters, kernel_size, padding="same", name=conv_name, use_bias=not(use_batchnorm))(x)
+        if use_batchnorm:
+            x = BatchNormalization(name=bn_name)(x)
+        x = Activation('relu', name=relu_name)(x)
+        return x
+    return layer
+
+
+def Upsample2D_block(filters, stage, kernel_size=(3,3), upsample_rate=(2,2),
+                     use_batchnorm=False, skip=None):
+
+    def layer(input_tensor):
+        conv_name, bn_name, relu_name, up_name = handle_block_names(stage)
+        x = UpSampling2D(size=upsample_rate, name=up_name)(input_tensor)
+        if skip is not None:
+            x = Concatenate()([x, skip])
+        x = ConvRelu(filters, kernel_size, use_batchnorm=use_batchnorm,
+                     conv_name=conv_name + '1', bn_name=bn_name + '1', relu_name=relu_name + '1')(x)
+        x = ConvRelu(filters, kernel_size, use_batchnorm=use_batchnorm,
+                     conv_name=conv_name + '2', bn_name=bn_name + '2', relu_name=relu_name + '2')(x)
+        return x
+    return layer
+
+
+def Transpose2D_block(filters, stage, kernel_size=(3,3), upsample_rate=(2,2),
+                      transpose_kernel_size=(4,4), use_batchnorm=False, skip=None):
+    def layer(input_tensor):
+        conv_name, bn_name, relu_name, up_name = handle_block_names(stage)
+        x = Conv2DTranspose(filters, transpose_kernel_size, strides=upsample_rate,
+                            padding='same', name=up_name, use_bias=not(use_batchnorm))(input_tensor)
+        if use_batchnorm:
+            x = BatchNormalization(name=bn_name+'1')(x)
+        x = Activation('relu', name=relu_name+'1')(x)
+        if skip is not None:
+            x = Concatenate()([x, skip])
+        x = ConvRelu(filters, kernel_size, use_batchnorm=use_batchnorm,
+                     conv_name=conv_name + '2', bn_name=bn_name + '2', relu_name=relu_name + '2')(x)
+        return x
+    return layer
+
+
+def get_layer_number(model, layer_name):
+    for i, l in enumerate(model.layers):
+        if l.name == layer_name:
+            return i
+    raise ValueError('No layer with name {} in  model {}.'.format(layer_name, model.name))
+
+
+def to_tuple(x):
+    if isinstance(x, tuple):
+        if len(x) == 2:
+            return x
+    elif np.isscalar(x):
+        return (x, x)
+    raise ValueError('Value should be tuple of length 2 or int value, got "{}"'.format(x))
+
+
+# From https://github.com/MrGiovanni/UNetPlusPlus
+class TestUnetPlusPlus(unittest.TestCase):
+
+    def setUp(self):
+        self.model_files = []
+
+    def tearDown(self):
+        for fl in self.model_files:
+            os.remove(fl)
+
+    @unittest.skipIf(get_maximum_opset_supported() < 14,
+                     "Need ConvTranspose-14 support.")
+    def test_unet_plus_plus(self):
+        backbone_name = 'vgg16'
+        input_shape = (None, None, 3)
+        input_tensor = None
+        encoder_weights = None#'imagenet'
+
+        backbone = VGG16(input_shape=input_shape,
+                         input_tensor=input_tensor,
+                         weights=encoder_weights,
+                         include_top=False)
+
+        input = backbone.input
+        x = backbone.output
+        block_type = 'transpose'
+
+        if block_type == 'transpose':
+            up_block = Transpose2D_block
+        else:
+            up_block = Upsample2D_block
+
+        skip_connection_layers = ('block5_conv3', 'block4_conv3', 'block3_conv3', 'block2_conv2', 'block1_conv2')
+
+        # convert layer names to indices
+        skip_connection_idx = ([get_layer_number(backbone, l) if isinstance(l, str) else l
+                                for l in skip_connection_layers])
+
+        n_upsample_blocks = 5
+        upsample_rates = (2,2,2,2,2)
+        decoder_filters = (256,128,64,32,16)
+        block_type='upsampling'
+        activation='sigmoid'
+        use_batchnorm=True
+        classes=1
+
+        for i in range(n_upsample_blocks):
+
+            # check if there is a skip connection
+            skip_connection = None
+            if i < len(skip_connection_idx):
+                skip_connection = backbone.layers[skip_connection_idx[i]].output
+
+            upsample_rate = to_tuple(upsample_rates[i])
+
+            x = up_block(decoder_filters[i], i, upsample_rate=upsample_rate,
+                        skip=skip_connection, use_batchnorm=use_batchnorm)(x)
+
+        x = Conv2D(classes, (3,3), padding='same', name='final_conv')(x)
+        x = Activation(activation, name=activation)(x)
+
+        model = Model(input, x)
+        res = run_image(model, self.model_files, img_path, target_size=(256, 256, 3))
+        self.assertTrue(*res)
+
+if __name__ == "__main__":
+    unittest.main()