diff --git a/api/ccapi/include/layer.h b/api/ccapi/include/layer.h index ca0ae19f62..347bda8d29 100644 --- a/api/ccapi/include/layer.h +++ b/api/ccapi/include/layer.h @@ -98,6 +98,7 @@ enum LayerType { LAYER_REDUCE_MEAN, /**< Reduce mean Layer type */ LAYER_LOSS_CONSTANT_DERIVATIVE, /**< Synthetic loss layer to feed constant derivative */ + LAYER_UPSAMPLE2D, /**< Upsample 2D Layer type */ LAYER_UNKNOWN = ML_TRAIN_LAYER_TYPE_UNKNOWN /**< Unknown */ }; @@ -535,6 +536,14 @@ Identity(const std::vector &properties = {}) { return createLayer(LayerType::LAYER_IDENTITY, properties); } +/** + * @brief Helper function to create Upsample2d layer + */ +inline std::unique_ptr +Upsample2D(const std::vector &properties = {}) { + return createLayer(LayerType::LAYER_UPSAMPLE2D, properties); +} + /** * @brief Helper function to create activation layer */ diff --git a/nntrainer/app_context.cpp b/nntrainer/app_context.cpp index a66bc667af..f00f667a45 100644 --- a/nntrainer/app_context.cpp +++ b/nntrainer/app_context.cpp @@ -76,6 +76,7 @@ #include #ifdef ENABLE_TFLITE_BACKBONE +#include "upsample2d_layer.h" #include #endif @@ -306,6 +307,8 @@ static void add_default_object(AppContext &ac) { LayerType::LAYER_POSITIONAL_ENCODING); ac.registerFactory(nntrainer::createLayer, IdentityLayer::type, LayerType::LAYER_IDENTITY); + ac.registerFactory(nntrainer::createLayer, + Upsample2dLayer::type, LayerType::LAYER_UPSAMPLE2D); #ifdef ENABLE_NNSTREAMER_BACKBONE ac.registerFactory(nntrainer::createLayer, diff --git a/nntrainer/layers/common_properties.h b/nntrainer/layers/common_properties.h index 3c5d73300d..9316e79300 100644 --- a/nntrainer/layers/common_properties.h +++ b/nntrainer/layers/common_properties.h @@ -869,10 +869,9 @@ struct ActivationTypeInfo { Enum::ACT_GELU, Enum::ACT_QUICK_GELU, Enum::ACT_NONE, Enum::ACT_UNKNOWN}; - static constexpr const char *EnumStr[] = {"tanh", "sigmoid", "relu", - "softmax", "leaky_relu", "swish", - "gelu", "quick_gelu", "none", - "unknown"}; + static constexpr const char *EnumStr[] = { + "tanh", "sigmoid", "relu", "softmax", "leaky_relu", + "swish", "gelu", "quick_gelu", "none", "unknown"}; }; /** @@ -1068,6 +1067,31 @@ class WeightRegularizer final : public BasicRegularizer { static constexpr const char *key = "weight_regularizer"; }; +/** + * @brief Enumeration of upsample type + * @todo Support torch and keras supported modes like bicubic + */ +struct UpsampleModeInfo { + /** + * @brief Upsampling operation type class + */ + enum class Enum { nearest, bilinear }; + static constexpr std::initializer_list EnumList = {Enum::nearest, + Enum::bilinear}; + + static constexpr const char *EnumStr[] = {"nearest", "bilinear"}; +}; + +/** + * @brief Upsample Type Enumeration Information + * + */ +class UpsampleMode final : public EnumProperty { +public: + using prop_tag = enum_class_prop_tag; + static constexpr const char *key = "upsample"; +}; + /** * @brief Enumeration of pooling type */ diff --git a/nntrainer/layers/meson.build b/nntrainer/layers/meson.build index 9d42a3e34b..75a59605ab 100644 --- a/nntrainer/layers/meson.build +++ b/nntrainer/layers/meson.build @@ -42,7 +42,8 @@ layer_sources = [ 'reshape_layer.cpp', 'reduce_mean_layer.cpp', 'positional_encoding_layer.cpp', - 'identity_layer.cpp' + 'identity_layer.cpp', + 'upsample2d_layer.cpp' ] layer_headers = [ diff --git a/nntrainer/layers/upsample2d_layer.cpp b/nntrainer/layers/upsample2d_layer.cpp new file mode 100644 index 0000000000..2f538b632d --- /dev/null +++ b/nntrainer/layers/upsample2d_layer.cpp @@ -0,0 +1,199 @@ +// SPDX-License-Identifier: Apache-2.0 +/** + * Copyright (C) 2024 heka1024 + * + * @file upsample2d_layer.h + * @date 15 June 2024 + * @brief It is a implementation of upsample layer for given size and + * interpolation method + * @see https://github.com/nnstreamer/nntrainer + * @author heka1024 + * @bug No known bugs except for NYI items + */ + +#include +#include +#include + +namespace nntrainer { +static constexpr size_t SINGLE_INOUT_IDX = 0; + +Upsample2dLayer::Upsample2dLayer() : + Layer(), + upsample2d_props(props::UpsampleMode(), + std::array()) {} + +void Upsample2dLayer::finalize(nntrainer::InitLayerContext &context) { + std::vector dim = context.getInputDimensions(); + + const auto &kernel_size = + std::get>(upsample2d_props); + + for (unsigned int i = 0; i < dim.size(); ++i) { + if (dim[i].getDataLen() == 0) { + throw std::invalid_argument("Input dimension is not set"); + } else { + dim[i].channel(dim[i].channel()); + dim[i].height(dim[i].height() * kernel_size[0]); + dim[i].width(dim[i].width() * kernel_size[1]); + } + } + + context.setOutputDimensions(dim); +} + +void Upsample2dLayer::forwarding(nntrainer::RunLayerContext &context, + bool training) { + nntrainer::Tensor &in = context.getInput(SINGLE_INOUT_IDX); + nntrainer::Tensor &out = context.getOutput(SINGLE_INOUT_IDX); + + const auto &upsampling_type = + std::get(upsample2d_props).get(); + const auto &kernel_size = + std::get>(upsample2d_props); + + switch (upsampling_type) { + case props::UpsampleModeInfo::Enum::nearest: + for (int b = 0; b < (int)out.batch(); b++) { + for (int c = 0; c < (int)out.channel(); c++) { + for (int h = 0; h < (int)out.height(); h++) { + for (int w = 0; w < (int)out.width(); w++) { + out.setValue( + b, c, h, w, + in.getValue(b, c, h / kernel_size[0], w / kernel_size[1])); + } + } + } + } + break; + case props::UpsampleModeInfo::Enum::bilinear: { + float scale_h = kernel_size[0]; + float scale_w = kernel_size[1]; + + for (int b = 0; b < (int)out.batch(); b++) { + for (int c = 0; c < (int)out.channel(); c++) { + for (int h = 0; h < (int)out.height(); h++) { + for (int w = 0; w < (int)out.width(); w++) { + float x_in = (w + 0.5f) / scale_w - 0.5f; + float y_in = (h + 0.5f) / scale_h - 0.5f; + + if (x_in < 0) { + x_in = 0.0f; + } + if (y_in < 0) { + y_in = 0.0f; + } + + int x0 = static_cast(floor(x_in)); + int y0 = static_cast(floor(y_in)); + int x1 = std::min(x0 + 1, (int)in.width() - 1); + int y1 = std::min(y0 + 1, (int)in.height() - 1); + + float dx = x_in - x0; + float dy = y_in - y0; + + float top = (1.0f - dx) * in.getValue(b, c, y1, x0) + + dx * in.getValue(b, c, y1, x1); + float bottom = (1.0f - dx) * in.getValue(b, c, y0, x0) + + dx * in.getValue(b, c, y0, x1); + float v = (1.0f - dy) * bottom + dy * top; + out.setValue(b, c, h, w, v); + } + } + } + } + } break; + default: + throw std::runtime_error("Error: Unknown Upsample Mode Type"); + } +} + +void Upsample2dLayer::calcDerivative(nntrainer::RunLayerContext &context) { + const nntrainer::Tensor &derivative_ = + context.getIncomingDerivative(SINGLE_INOUT_IDX); + + nntrainer::Tensor &dx = context.getOutgoingDerivative(SINGLE_INOUT_IDX); + + const auto &kernel_size = + std::get>(upsample2d_props); + const auto &upsampling_type = + std::get(upsample2d_props).get(); + + switch (upsampling_type) { + case props::UpsampleModeInfo::Enum::nearest: { + float val = 0; + for (int b = 0; b < (int)derivative_.batch(); b++) { + for (int c = 0; c < (int)derivative_.channel(); c++) { + for (int h = 0; h < (int)derivative_.height(); h++) { + for (int w = 0; w < (int)derivative_.width(); w++) { + if (h % kernel_size[0] == 0 && w % kernel_size[1] == 0) { + dx.setValue(b, c, h / kernel_size[0], w / kernel_size[1], 0); + } + + val = dx.getValue(b, c, h / kernel_size[0], w / kernel_size[1]) + + derivative_.getValue(b, c, h, w); + dx.setValue(b, c, h / kernel_size[0], w / kernel_size[1], val); + } + } + } + } + } break; + case props::UpsampleModeInfo::Enum::bilinear: { + dx.setZero(); + + int input_height = dx.height(); + int input_width = dx.width(); + + for (int b = 0; b < (int)derivative_.batch(); b++) { + for (int c = 0; c < (int)derivative_.channel(); c++) { + for (int h = 0; h < (int)derivative_.height(); h++) { + for (int w = 0; w < (int)derivative_.width(); w++) { + float in_h = (h + 0.5f) / kernel_size[0] - 0.5f; + float in_w = (w + 0.5f) / kernel_size[1] - 0.5f; + + if (in_h < 0) { + in_h = 0.0f; + } + if (in_w < 0) { + in_w = 0.0f; + } + + int y0 = static_cast(floor(in_h)); + int x0 = static_cast(floor(in_w)); + int y1 = std::min(y0 + 1, input_height - 1); + int x1 = std::min(x0 + 1, input_width - 1); + + float dx_ = (in_w - x0); // Due to name conflict with dx + float dy_ = (in_h - y0); + + float top_left_weight = (1.0 - dy_) * (1.0 - dx_); + float top_right_weight = (1.0 - dy_) * dx_; + float bottom_left_weight = dy_ * (1.0 - dx_); + float bottom_right_weight = dy_ * dx_; + + float grad = derivative_.getValue(b, c, h, w); + + dx.addValue(b, c, y0, x0, top_left_weight * grad, 1.0f); + dx.addValue(b, c, y0, x1, top_right_weight * grad, 1.0f); + dx.addValue(b, c, y1, x0, bottom_left_weight * grad, 1.0f); + dx.addValue(b, c, y1, x1, bottom_right_weight * grad, 1.0f); + } + } + } + } + } break; + default: + throw std::runtime_error("Error: Unknown Upsample Mode Type"); + } +} + +void Upsample2dLayer::setProperty(const std::vector &values) { + auto remain_props = loadProperties(values, upsample2d_props); + + if (!remain_props.empty()) { + std::string msg = "[Upsample2dLayer] Unknown properties set with count" + + std::to_string(values.size()); + throw exception::not_supported(msg); + } +} +} // namespace nntrainer diff --git a/nntrainer/layers/upsample2d_layer.h b/nntrainer/layers/upsample2d_layer.h new file mode 100644 index 0000000000..d0628a1c8d --- /dev/null +++ b/nntrainer/layers/upsample2d_layer.h @@ -0,0 +1,90 @@ +// SPDX-License-Identifier: Apache-2.0 +/** + * Copyright (C) 2024 heka1024 + * + * @file upsample2d_layer.h + * @date 15 June 2024 + * @brief This is Upsample2d Layer Class of Neural Network + * @see https://github.com/nnstreamer/nntrainer + * @author heka1024 + * @bug No known bugs except for NYI items + * + */ + +#ifndef __UPSAMPLE2D_LAYER_H__ +#define __UPSAMPLE2D_LAYER_H__ +#ifdef __cplusplus + +#include +#include + +#include + +namespace nntrainer { + +constexpr const unsigned int UPSAMPLE2D_DIM = 2; + +/** + * @class Upsample2dLayer + * @brief Upsamle 2d layer + */ +class Upsample2dLayer : public Layer { +public: + /** + * @brief Construct a new Upsample layer object + * + */ + Upsample2dLayer(); + + /** + * @brief Destroy the Upsample layer object + * + */ + ~Upsample2dLayer() {} + + /** + * @copydoc Layer::finalize(InitLayerContext &context) + */ + void finalize(nntrainer::InitLayerContext &context) override; + + /** + * @copydoc Layer::forwarding(RunLayerContext &context, bool training) + */ + void forwarding(nntrainer::RunLayerContext &context, bool training) override; + + /** + * @copydoc Layer::calcDerivative(RunLayerContext &context) + */ + void calcDerivative(nntrainer::RunLayerContext &context) override; + + /** + * @copydoc bool supportBackwarding() const + */ + bool supportBackwarding() const override { return true; }; + + /** + * @copydoc Layer::exportTo(Exporter &exporter, ExportMethods method) + */ + void exportTo(nntrainer::Exporter &exporter, + const ml::train::ExportMethods &method) const override{}; + + /** + * @copydoc Layer::getType() + */ + const std::string getType() const override { return Upsample2dLayer::type; }; + + /** + * @copydoc Layer::setProperty(const std::vector &values) + */ + void setProperty(const std::vector &values) override; + + inline static const std::string type = "upsample2d"; + +private: + std::tuple> + upsample2d_props; /* mode, size of kernel */ +}; +} // namespace nntrainer + +#endif /* __cplusplus */ +#endif /* __UPSAMPLE2D_LAYER_H__ */ diff --git a/test/input_gen/gen_layer_tests.py b/test/input_gen/gen_layer_tests.py index 7a1ed18ec6..f6d762d5f0 100644 --- a/test/input_gen/gen_layer_tests.py +++ b/test/input_gen/gen_layer_tests.py @@ -171,6 +171,21 @@ def call(self, inputs): record_single(conv, (1, 3, 11, 11), "conv2d_sb_same_dilation") record_single(conv, (3, 3, 11, 11), "conv2d_mb_same_dilation") + conv = K.layers.UpSampling2D(size=(2, 2), interpolation="nearest", input_shape=(2, 2, 1)) + record_single(conv, (1, 2, 2, 1), "upsample2d_2x2_nearest") # input_shape: n h w c + + conv = K.layers.UpSampling2D(size=(3, 3), interpolation="nearest", input_shape=(3, 3, 1)) + record_single(conv, (1, 3, 3, 1), "upsample2d_3x3_nearest") + + conv = K.layers.UpSampling2D(size=(2, 2), interpolation="bilinear", input_shape=(2, 2, 1)) + record_single(conv, (1, 2, 2, 1), "upsample2d_2x2_bilinear") # input_shape: n h w c + + conv = K.layers.UpSampling2D(size=(3, 3), interpolation="bilinear", input_shape=(3, 3, 1)) + record_single(conv, (1, 3, 3, 1), "upsample2d_3x3_bilinear") + + conv = K.layers.UpSampling2D(size=(4, 4), interpolation="bilinear", input_shape=(10, 10, 1)) + record_single(conv, (1, 10, 10, 1), "upsample2d_big_bilinear") + # use float data to generate input here attention = K.layers.Attention() record_single( diff --git a/test/unittest/layers/meson.build b/test/unittest/layers/meson.build index 5aae748adf..ce30feb147 100644 --- a/test/unittest/layers/meson.build +++ b/test/unittest/layers/meson.build @@ -1,4 +1,4 @@ -layer_common_test_inc = [ include_directories('./', '../../include') ] +layer_common_test_inc = [include_directories('./', '../../include')] layer_common_test_standalone_files = files('layers_standalone_common_tests.cpp') layer_common_test_dependent_files = files('layers_dependent_common_tests.cpp') @@ -6,27 +6,27 @@ nntrainer_layer_common_standalone_tests_lib = shared_library( 'nntrainer_layer_common_standalone_tests', 'layers_standalone_common_tests.cpp', cpp_args : '-Wno-maybe-uninitialized', - dependencies: [nntrainer_dep, gtest_dep], # nntrainer_devel_dep - include_directories: layer_common_test_inc + dependencies : [nntrainer_dep, gtest_dep], # nntrainer_devel_dep + include_directories : layer_common_test_inc ) nntrainer_layer_common_standalone_tests_dep = declare_dependency( - link_with: nntrainer_layer_common_standalone_tests_lib, - include_directories: layer_common_test_inc - ) + link_with : nntrainer_layer_common_standalone_tests_lib, + include_directories : layer_common_test_inc +) nntrainer_layer_common_dependent_tests_lib = shared_library( 'nntrainer_layer_common_dependent_tests', 'layers_dependent_common_tests.cpp', cpp_args : '-Wno-maybe-uninitialized', - dependencies: [nntrainer_dep, gtest_dep], # nntrainer_devel_dep - include_directories: layer_common_test_inc + dependencies : [nntrainer_dep, gtest_dep], # nntrainer_devel_dep + include_directories : layer_common_test_inc ) nntrainer_layer_common_dependent_tests_dep = declare_dependency( - link_with: nntrainer_layer_common_dependent_tests_lib, - include_directories: layer_common_test_inc - ) + link_with : nntrainer_layer_common_dependent_tests_lib, + include_directories : layer_common_test_inc +) test_target = [ layer_common_test_dependent_files, @@ -64,6 +64,7 @@ test_target = [ # 'unittest_layers_mol_attention.cpp', 'unittest_layers_multi_head_attention.cpp', 'unittest_layers_positional_encoding.cpp', + 'unittest_layers_upsample2d.cpp' ] if get_option('enable-tflite-backbone') @@ -83,16 +84,16 @@ endif exe = executable( 'unittest_layers', test_target, - dependencies: [ + dependencies : [ nntrainer_test_main_deps, ], cpp_args : '-Wno-maybe-uninitialized', - install: get_option('enable-test'), - install_dir: application_install_dir + install : get_option('enable-test'), + install_dir : application_install_dir ) # @todo delete v2 once migration is done test('unittest_layers', exe, - args: '--gtest_output=xml:@0@/@1@.xml'.format(meson.build_root(), 'unittest_layers'), - timeout: test_timeout + args : '--gtest_output=xml:@0@/@1@.xml'.format(meson.build_root(), 'unittest_layers'), + timeout : test_timeout ) diff --git a/test/unittest/layers/unittest_layers_upsample2d.cpp b/test/unittest/layers/unittest_layers_upsample2d.cpp new file mode 100644 index 0000000000..0435c017ed --- /dev/null +++ b/test/unittest/layers/unittest_layers_upsample2d.cpp @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: Apache-2.0 +/** + * Copyright (C) 2024 heka1024 + * + * @file unittest_layers_upsample2d.cppp + * @date 15 June 2024 + * @brief Unit test for upsample2d layer + * @see https://github.com/nnstreamer/nntrainer + * @author heka1024 + * @bug No known bugs except for NYI items + */ +#include + +#include + +#include +#include + +auto semantic_upsample2d = LayerSemanticsParamType( + nntrainer::createLayer, + nntrainer::Upsample2dLayer::type, {"upsample=nearest", "kernel_size=2,2"}, + LayerCreateSetPropertyOptions::AVAILABLE_FROM_APP_CONTEXT, false, 1); + +GTEST_PARAMETER_TEST(Upsample2D, LayerSemantics, + ::testing::Values(semantic_upsample2d)); + +auto upsampling_2x2_nearest = LayerGoldenTestParamType( + nntrainer::createLayer, + {"upsample=nearest", "kernel_size=2,2"}, "1:1:2:2", + "upsample2d_2x2_nearest.nnlayergolden", LayerGoldenTestParamOptions::DEFAULT, + "nchw", "fp32", "fp32"); + +auto upsampling_3x3_nearest = LayerGoldenTestParamType( + nntrainer::createLayer, + {"upsample=nearest", "kernel_size=3,3"}, "1:1:3:3", + "upsample2d_3x3_nearest.nnlayergolden", LayerGoldenTestParamOptions::DEFAULT, + "nchw", "fp32", "fp32"); + +auto upsampling_2x2_bilinear = LayerGoldenTestParamType( + nntrainer::createLayer, + {"upsample=bilinear", "kernel_size=2,2"}, "1:1:2:2", + "upsample2d_2x2_bilinear.nnlayergolden", LayerGoldenTestParamOptions::DEFAULT, + "nchw", "fp32", "fp32"); + +auto upsampling_3x3_bilinear = LayerGoldenTestParamType( + nntrainer::createLayer, + {"upsample=bilinear", "kernel_size=3,3"}, "1:1:3:3", + "upsample2d_3x3_bilinear.nnlayergolden", LayerGoldenTestParamOptions::DEFAULT, + "nchw", "fp32", "fp32"); + +auto upsampling_big_bilinear = LayerGoldenTestParamType( + nntrainer::createLayer, + {"upsample=bilinear", "kernel_size=4,4"}, "1:1:10:10", + "upsample2d_big_bilinear.nnlayergolden", LayerGoldenTestParamOptions::DEFAULT, + "nchw", "fp32", "fp32"); + +GTEST_PARAMETER_TEST(Upsample2D, LayerGoldenTest, + ::testing::Values(upsampling_2x2_nearest, + upsampling_3x3_nearest, + upsampling_2x2_bilinear, + upsampling_3x3_bilinear, + upsampling_big_bilinear));