DRAFT circle schema 06

on-going draft to support new tflite/circle schema.

Signed-off-by: SaeHie Park <[email protected]>

compiler/circle2circle/src/Circle2Circle.cpp

@@ -176,6 +176,12 @@ int entry(int argc, char **argv)
              "This will turn off operator validations. May help input model investigation.");
   add_switch(arser, "--generate_profile_data", "This will turn on profiling data generation.");
 
+  // NOTE Experimental options; these will be removed someday
+  //      Add experimental options here
+  add_switch(arser, "--exp_disable_sep_transposeconv_actfunc",
+             "This will turn off experimental separation of activation function from "
+             "TransposeConv.");
+
   // Convert dynamic batch to single batch
   // Users have to use this option only when the first dimension of rank 4 input (NHWC or NCHW)
   // is dynamic. Remove this comment after non-rank 4 is supported.
@@ -332,6 +338,14 @@ int entry(int argc, char **argv)
   if (arser.get<bool>("--unroll_unidirseqlstm"))
     options->enable(Algorithms::UnrollUnidirSeqLSTM);
 
+  // NOTE Experimental options; these will be removed someday
+  //      Add experimental options here
+  // NOTE XpSepActFromTransposeConv is enabled for default
+  //      exp_disable_sep_act_transposeconv is to turn it off
+  //      which will leave TransposeConv with fused activation
+  if (!arser.get<bool>("--exp_disable_sep_transposeconv_actfunc"))
+    options->enable(Algorithms::XpSepActFromTransposeConv);
+
   if (arser.get<bool>("--mute_warnings"))
     settings->set(luci::UserSettings::Key::MuteWarnings, true);
   if (arser.get<bool>("--disable_validation"))

compiler/circledump/src/OpPrinter.cpp

@@ -665,6 +665,8 @@ class TransposeConvPrinter : public OpPrinter
       os << "Padding(" << params->padding() << ") ";
       os << "Stride.W(" << params->stride_w() << ") ";
       os << "Stride.H(" << params->stride_h() << ") ";
+      os << "Activation(" << EnumNameActivationFunctionType(params->fused_activation_function())
+         << ") ";
       os << std::endl;
     }
   }

compiler/luci-interpreter/src/core/KernelParams.h

@@ -221,6 +221,7 @@ struct TransposeConvParams
   Padding padding;
   int32_t stride_height;
   int32_t stride_width;
+  Activation activation;
 };
 
 struct UnidirectionalSequenceLSTMParams

compiler/luci-interpreter/src/kernels/TransposeConv.test.cpp

@@ -54,6 +54,7 @@ void Check(std::initializer_list<int32_t> output_shape_shape,
   params.padding = padding;
   params.stride_height = stride_height;
   params.stride_width = stride_width;
+  params.activation = luci::FusedActFunc::NONE;
 
   if (bias_data.size() != 0)
   {
@@ -164,6 +165,7 @@ TEST(TransposeConvTest, UInt8)
   params.padding = Padding::VALID;
   params.stride_height = 2;
   params.stride_width = 2;
+  params.activation = luci::FusedActFunc::NONE;
 
   TransposeConv kernel(&output_shape_tensor, &filter_tensor, &input_tensor, &bias_tensor,
                        &output_tensor, &scratch_tensor, params);
@@ -232,6 +234,7 @@ TEST(TransposeConvTest, UInt8_CWQ)
   params.padding = Padding::VALID;
   params.stride_height = 2;
   params.stride_width = 2;
+  params.activation = luci::FusedActFunc::NONE;
 
   TransposeConv kernel(&output_shape_tensor, &filter_tensor, &input_tensor, &bias_tensor,
                        &output_tensor, &scratch_tensor, params);
@@ -278,6 +281,7 @@ TEST(TransposeConvTest, SInt16)
   params.padding = Padding::VALID;
   params.stride_height = 2;
   params.stride_width = 2;
+  params.activation = luci::FusedActFunc::NONE;
 
   TransposeConv kernel(&output_shape_tensor, &filter_tensor, &input_tensor, &bias_tensor,
                        &output_tensor, &scratch_tensor, params);
@@ -336,6 +340,7 @@ TEST(TransposeConvTest, SInt16_CWQ_weights)
   params.padding = Padding::VALID;
   params.stride_height = 2;
   params.stride_width = 2;
+  params.activation = luci::FusedActFunc::NONE;
 
   TransposeConv kernel(&output_shape_tensor, &filter_tensor, &input_tensor, &bias_tensor,
                        &output_tensor, &scratch_tensor, params);

compiler/luci-interpreter/src/loader/KernelBuilder.test.cpp

@@ -1319,6 +1319,7 @@ TEST_F(KernelBuilderTest, TransposeConv)
   op->padding(luci::Padding::SAME);
   op->stride()->h(11);
   op->stride()->w(13);
+  op->fusedActivationFunction(luci::FusedActFunc::NONE);
 
   auto kernel = buildKernel<kernels::TransposeConv>(op);
   ASSERT_THAT(kernel, NotNull());
@@ -1331,6 +1332,7 @@ TEST_F(KernelBuilderTest, TransposeConv)
   EXPECT_THAT(kernel->params().padding, Eq(op->padding()));
   EXPECT_THAT(kernel->params().stride_height, Eq(op->stride()->h()));
   EXPECT_THAT(kernel->params().stride_width, Eq(op->stride()->w()));
+  EXPECT_THAT(kernel->params().activation, Eq(op->fusedActivationFunction()));
 }
 
 TEST_F(KernelBuilderTest, Unpack)

compiler/luci-interpreter/src/loader/nodes/TransposeConv.cpp

@@ -47,6 +47,13 @@ std::unique_ptr<Kernel> build_kernel_CircleTransposeConv(const luci::CircleNode
   params.padding = node->padding();
   params.stride_height = node->stride()->h();
   params.stride_width = node->stride()->w();
+  params.activation = node->fusedActivationFunction();
+
+  // TODO support activation
+  if (params.activation != luci::FusedActFunc::NONE)
+  {
+    throw std::runtime_error("Unsupported activation of TransposeConv");
+  }
 
   return std::make_unique<kernels::TransposeConv>(input_sizes, filter, out_backprop, bias, output,
                                                   tmp, params);

compiler/luci/export/src/CircleBuiltinTypesExtractor.h

@@ -485,7 +485,8 @@ class BuiltinOptionsExtractor final
   flatbuffers::Offset<void> visit(luci::CircleTransposeConv *node)
   {
     return circle::CreateTransposeConvOptions(_builder, getOpPadding(node->padding()),
-                                              node->stride()->w(), node->stride()->h())
+                                              node->stride()->w(), node->stride()->h(),
+                                              to_circle_actfunc(node->fusedActivationFunction()))
       .Union();
   }
   flatbuffers::Offset<void> visit(luci::CircleUnidirectionalSequenceLSTM *node)

compiler/luci/import/src/Nodes/CircleTransposeConv.cpp

@@ -74,6 +74,7 @@ CircleNode *CircleTransposeConvGraphBuilder::build_node(const circle::OperatorT
   node->padding(luci_padding(options->padding));
   node->stride()->w(options->stride_w);
   node->stride()->h(options->stride_h);
+  node->fusedActivationFunction(luci_actfunc(options->fused_activation_function));
 
   return node;
 }

compiler/luci/lang/include/luci/IR/Nodes/CircleTransposeConv.h

@@ -35,6 +35,7 @@ namespace luci
  */
 class CircleTransposeConv final
   : public FixedArityNode<4, CircleNodeImpl<CircleOpcode::TRANSPOSE_CONV>>,
+    public CircleNodeMixin<CircleNodeTrait::FusedActFunc>,
     public CircleNodeMixin<CircleNodeTrait::Bias>
 {
 public:

compiler/luci/logex/src/CircleNodeSummaryBuilder.test.cpp

@@ -298,6 +298,7 @@ TEST_F(CircleNodeSummaryBuilderTest, TransposeConv_validate)
 {
   luci::CircleTransposeConv node;
   node.padding(luci::Padding::SAME);
+  node.fusedActivationFunction(luci::FusedActFunc::RELU);
   EXPECT_TRUE(mock_build(&node));
 }
 
@@ -307,3 +308,10 @@ TEST_F(CircleNodeSummaryBuilderTest, TransposeConv_validate_padding_NEG)
   node.padding(luci::Padding::UNDEFINED);
   EXPECT_FALSE(mock_build(&node));
 }
+
+TEST_F(CircleNodeSummaryBuilderTest, TransposeConv_validate_fused_NEG)
+{
+  luci::CircleTransposeConv node;
+  node.fusedActivationFunction(luci::FusedActFunc::UNDEFINED);
+  EXPECT_FALSE(mock_build(&node));
+}

compiler/luci/logex/src/CircleNodeSummaryBuilders.cpp

@@ -1018,6 +1018,8 @@ bool CircleTransposeConvSummaryBuilder::validate(const luci::CircleNode *node)
   auto transpose_conv = loco::must_cast<const luci::CircleTransposeConv *>(node);
   if (transpose_conv->padding() == luci::Padding::UNDEFINED)
     return false;
+  if (transpose_conv->fusedActivationFunction() == luci::FusedActFunc::UNDEFINED)
+    return false;
 
   return true;
 }
@@ -1034,6 +1036,7 @@ void CircleTransposeConvSummaryBuilder::build_attributes(const luci::CircleNode
   auto transpose_conv = loco::must_cast<const luci::CircleTransposeConv *>(node);
   s.args().append("stride(h,w)", to_str(transpose_conv->stride()));
   s.args().append("padding", to_str(transpose_conv->padding()));
+  s.args().append("fused_activation_function", to_str(transpose_conv->fusedActivationFunction()));
 }
 
 std::vector<std::string>

compiler/luci/partition/src/Nodes/CircleTransposeConv.test.cpp

@@ -38,6 +38,7 @@ class NodeGraphlet : public NodeGraphletT<luci::CircleTransposeConv>
     NodeGraphletT<luci::CircleTransposeConv>::init(g);
 
     _node->padding(luci::Padding::VALID);
+    _node->fusedActivationFunction(luci::FusedActFunc::RELU);
   }
 };
 

compiler/luci/pass/include/luci/CircleOptimizer.h

@@ -90,6 +90,7 @@ class CircleOptimizer final
       RemoveQuantDequantSeq,
       RemoveDuplicateConst,
       UnrollUnidirSeqLSTM,
+      XpSepActFromTransposeConv,
     };
 
     enum AlgorithmParameters

compiler/luci/pass/include/luci/Pass/XpSepActFromTransposeConvPass.h

@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2023 Samsung Electronics Co., Ltd. All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __LUCI_XP_SEP_ACT_FROM_TRANSPOSE_CONV_PASS_H__
+#define __LUCI_XP_SEP_ACT_FROM_TRANSPOSE_CONV_PASS_H__
+
+#include <logo/Pass.h>
+
+namespace luci
+{
+
+/**
+ * @brief  Experimental Class to separate activation functions from TransposeConv
+ */
+struct XpSepActFromTransposeConvPass final : public logo::Pass
+{
+  const char *name(void) const final { return "luci::XpSepActFromTransposeConvPass"; }
+
+  bool run(loco::Graph *g) final;
+};
+
+} // namespace luci
+
+#endif // __LUCI_XP_SEP_ACT_FROM_TRANSPOSE_CONV_PASS_H__

compiler/luci/pass/src/CircleOptimizer.cpp

@@ -73,6 +73,7 @@
 #include "luci/Pass/TransformMinReluToRelu6Pass.h"
 #include "luci/Pass/DecomposeHardSwishPass.h"
 #include "luci/Pass/UnrollUnidirectionalSequenceLSTMPass.h"
+#include "luci/Pass/XpSepActFromTransposeConvPass.h"
 // TODO add more passes
 
 #include "luci/Pass/CircleShapeInferencePass.h"
@@ -442,6 +443,14 @@ void CircleOptimizer::optimize(loco::Graph *g) const
   {
     phase.emplace_back(std::make_unique<luci::UnrollUnidirectionalSequenceLSTMPass>());
   }
+
+  // NOTE Experimental options; these will be removed someday
+  //      Add experimental options here
+  if (_options->query(Options::Algorithm::XpSepActFromTransposeConv))
+  {
+    phase.emplace_back(std::make_unique<luci::XpSepActFromTransposeConvPass>());
+  }
+
   // Forward Reshape/Transpose is done after
   // 1. SubstituteXXXToReshape
   // 2. RemoveRedundantReshape/Transpose

compiler/luci/pass/src/FuseActivationFunctionPass.cpp

@@ -42,6 +42,11 @@ bool fuse_activation_function(luci::CircleNode *node)
   // This will skip fuse for concat as luci-interpreter doesn't support this yet
   if (dynamic_cast<luci::CircleConcatenation *>(pred_node) != nullptr)
     return false;
+  // TODO remove this work-around
+  // This will skip fuse for TransposeConv as backends does not support this yet
+  // NOTE this conflicts with XpSepActFromTransposeConvOpPass when disabled
+  if (dynamic_cast<luci::CircleTransposeConv *>(pred_node) != nullptr)
+    return false;
 
   auto fused_act = node_with_fused_act->fusedActivationFunction();
 

compiler/luci/pass/src/FuseAddWithTConvPass.cpp

@@ -44,6 +44,9 @@ namespace
  */
 bool fuse_add_with_tconv(luci::CircleTransposeConv *tconv)
 {
+  // skip if tconv has fused activation
+  if (tconv->fusedActivationFunction() != luci::FusedActFunc::NONE)
+    return false;
   // check whether it has bias or not. This optimization works only if it doesn't.
   auto bias = dynamic_cast<luci::CircleOutputExclude *>(tconv->bias());
   if (not bias)

compiler/luci/pass/src/FuseBatchNormWithTConvPass.cpp

@@ -87,6 +87,9 @@ bool fused_batch_norm_with_tconv(luci::CircleAdd *add)
     return false;
   if (not luci::fill(&scale, &tconv).with_commutative_args_of(mul))
     return false;
+  // skip if tconv has fused activation
+  if (tconv->fusedActivationFunction() != luci::FusedActFunc::NONE)
+    return false;
 
   // check scale and shift constant attributes
   // TODO maybe rank check is not needed
@@ -215,6 +218,7 @@ bool fused_batch_norm_with_tconv(luci::CircleAdd *add)
   fused_tconv->stride()->h(tconv->stride()->h());
   fused_tconv->stride()->w(tconv->stride()->w());
   fused_tconv->name(name + "/TransposeConv");
+  fused_tconv->fusedActivationFunction(tconv->fusedActivationFunction());
   luci::add_origin(fused_tconv,
                    luci::composite_origin(
                      {luci::get_origin(add), luci::get_origin(mul), luci::get_origin(tconv)}));

compiler/luci/pass/src/QuantizePreCheckerPass.test.cpp

@@ -206,6 +206,7 @@ class SimpleTransposeConvGraph
     transpose_conv->outBackprop(input_1);
     transpose_conv->filter(filter);
     transpose_conv->inputSizes(input_sizes);
+    transpose_conv->fusedActivationFunction(luci::FusedActFunc::NONE);
 
     if (make_valid)
     {

compiler/luci/pass/src/XpSepActFromTransposeConvPass.cpp

@@ -0,0 +1,94 @@
+/*
+ * Copyright (c) 2023 Samsung Electronics Co., Ltd. All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "luci/Pass/XpSepActFromTransposeConvPass.h"
+
+#include <luci/IR/CircleNodes.h>
+#include <luci/IR/CircleNodeMixins.h>
+#include <luci/Profile/CircleNodeOrigin.h>
+
+namespace luci
+{
+
+namespace
+{
+
+bool separate_activation_fuction(luci::CircleTransposeConv *trconv)
+{
+  auto fused_act = trconv->fusedActivationFunction();
+  if (fused_act == luci::FusedActFunc::NONE)
+    return false;
+  if (fused_act == luci::FusedActFunc::UNDEFINED)
+    throw std::runtime_error("ACT UNDEFINED !!!");
+
+  // NOTE features() is call after replace().with();
+  //      calling loco::replace(trconv).with(actnode) will also update actnode itself
+  //      which will make totally wrong result with input of actnode is itself.
+
+  auto name = trconv->name();
+  luci::CircleNode *actnode = nullptr;
+  switch (fused_act)
+  {
+    case luci::FusedActFunc::RELU:
+    {
+      auto af = trconv->graph()->nodes()->create<luci::CircleRelu>();
+      loco::replace(trconv).with(af);
+      af->features(trconv);
+      af->name(name + "/Relu");
+      actnode = af;
+    }
+    break;
+    case luci::FusedActFunc::RELU6:
+    {
+      auto af = trconv->graph()->nodes()->create<luci::CircleRelu6>();
+      loco::replace(trconv).with(af);
+      af->features(trconv);
+      af->name(name + "/Relu6");
+      actnode = af;
+    }
+    break;
+    // TODO support more
+    default:
+      return false;
+  }
+  assert(actnode != nullptr);
+  actnode->dtype(trconv->dtype());
+  luci::add_origin(actnode, luci::get_origin(trconv));
+
+  trconv->fusedActivationFunction(luci::FusedActFunc::NONE);
+
+  return true;
+}
+
+} // namespace
+
+bool XpSepActFromTransposeConvPass::run(loco::Graph *g)
+{
+  bool changed = false;
+  for (auto node : loco::active_nodes(loco::output_nodes(g)))
+  {
+    auto trconv = dynamic_cast<luci::CircleTransposeConv *>(node);
+    if (trconv != nullptr)
+    {
+      if (separate_activation_fuction(trconv))
+        changed = true;
+    }
+  }
+
+  return changed;
+}
+
+} // namespace luci