Merge pull request #1373 from alibaba/feature/sync

Sync Internal Github

CMakeLists.txt

@@ -549,6 +549,20 @@ IF(MNN_BUILD_CODEGEN)
     include(${CMAKE_CURRENT_LIST_DIR}/codegen/CMakeLists.txt)
 ENDIF()
 
+# NPU
+IF(MNN_NPU)
+    add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/source/backend/hiai/)
+    IF(MNN_SEP_BUILD)
+        list(APPEND MNN_DEPS MNN_NPU)
+        list(APPEND MNN_DEPS ${CMAKE_CURRENT_LIST_DIR}/source/backend/hiai/3rdParty/${ANDROID_ABI}/libhiai.so)
+        list(APPEND MNN_DEPS ${CMAKE_CURRENT_LIST_DIR}/source/backend/hiai/3rdParty/${ANDROID_ABI}/libhiai_ir_build.so)
+        list(APPEND MNN_DEPS ${CMAKE_CURRENT_LIST_DIR}/source/backend/hiai/3rdParty/${ANDROID_ABI}/libhiai_ir.so)
+    ELSE()
+        list(APPEND MNN_TARGETS MNN_NPU)
+        list(APPEND MNN_OBJECTS_TO_LINK $<TARGET_OBJECTS:MNN_NPU>)
+    ENDIF()
+ENDIF()
+
 # TensorRT
 IF(MNN_TENSORRT)
   add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/source/backend/tensorrt/)

codegen/OpFuse.cpp

@@ -229,7 +229,10 @@ bool opFuse(CommandBuffer& cmd) {
             plugin_param->attr[0]->i = plugin.getFunctionNum()-1;
             pluginOp->main.type  = OpParameter_Plugin;
             pluginOp->main.value = plugin_param;
-            cmdPlugin = GeometryComputerUtils::makeCommand(pluginOp.get(), inputs, outputs);
+            flatbuffers::FlatBufferBuilder builder;
+            auto lastOffset = Op::Pack(builder, pluginOp.get());
+            builder.Finish(lastOffset);
+            cmdPlugin = GeometryComputerUtils::makeCommand(builder, inputs, outputs);
         }
         for (int i = 0; i < compSet.size(); i++) {
             auto cmd = const_cast<Command*>(compSet[i]->cmd);

codegen/build.sh

@@ -6,5 +6,5 @@ else
     clang -c -O3 kernel.bc -o kernel.o
 fi
 
-g++ -I$root_dir/include -I$root_dir/schema/current -I$root_dir/3rd_party/flatbuffers/include -I./ -std=c++11 -fPIC -O3 -o ./PluginWrapper.cpp.o -c $root_dir/codegen/plugin_wrapper/PluginWrapper.cpp
+g++ -I$root_dir/include -I$root_dir/schema/current -I$root_dir/3rd_party/flatbuffers/include -I./ -std=c++11 -fPIC -O3 -o ./PluginWrapper.cpp.o -c $root_dir/codegen/plugin_wrapper/PluginWrapper.cpp -o ./Plugin.cpp.o
 g++ -fPIC -std=c++11 -O3 -shared -o libplugin_fuse.so ./Plugin.cpp.o ./kernel.o ./libMNN_Plugin.dylib

express/Executor.cpp

@@ -245,7 +245,6 @@ struct Executor::Unit {
     std::vector<Tensor*> outputs;
     const Op* op;
     std::weak_ptr<Expr::Inside> inside;
-    std::shared_ptr<char> extraBuffer;
     std::vector<std::shared_ptr<Tensor>> outputContents;
 };
 Tensor* Executor::getOutput(ComputeCache* cache, int offset) {
@@ -695,7 +694,6 @@ void Executor::_visit(EXPRP expr, std::set<std::shared_ptr<Executor::ComputeCach
     std::shared_ptr<Unit> unitP(new Unit);
     Unit& unit = *unitP;
     unit.op = expr->get();
-    unit.extraBuffer = expr->extra().first;
     unit.inside = std::weak_ptr<Expr::Inside>(expr->inside());
     unit.inputs.resize(inputs.size());
     unit.outputs.resize(expr->inside()->mOutputTensors.size());

express/ExecutorScope.cpp

@@ -7,6 +7,7 @@
 //
 
 #include <thread>
+#include <mutex>
 #include <MNN/expr/Executor.hpp>
 #include <MNN/expr/Scope.hpp>
 #include <MNN/expr/ExecutorScope.hpp>
@@ -16,27 +17,37 @@ namespace Express {
 
 typedef std::shared_ptr<Express::Executor> ExecutorRef;
 #if !defined(__APPLE__)
-thread_local static Scope<ExecutorRef> g_executor_scope;
+thread_local static std::once_flag gInitFlag;
+thread_local static Scope<ExecutorRef>* g_executor_scope = nullptr;
 #else
-static Scope<ExecutorRef> g_executor_scope;
+static std::once_flag gInitFlag;
+static Scope<ExecutorRef>* g_executor_scope = nullptr;
 #endif
 
+static Scope<ExecutorRef>* _getGlobalScope() {
+    std::call_once(gInitFlag,
+                   [&]() {
+        g_executor_scope = new Scope<ExecutorRef>;
+    });
+    return g_executor_scope;
+}
+
 ExecutorScope::ExecutorScope(const std::shared_ptr<Executor>& current) {
-    g_executor_scope.EnterScope(current);
+    _getGlobalScope()->EnterScope(current);
 }
 
 ExecutorScope::ExecutorScope(const std::string& scope_name,
                              const std::shared_ptr<Executor>& current) {
-    g_executor_scope.EnterScope(scope_name, current);
+    _getGlobalScope()->EnterScope(scope_name, current);
 }
 
 ExecutorScope::~ExecutorScope() {
-    g_executor_scope.ExitScope();
+    _getGlobalScope()->ExitScope();
 }
 
 const std::shared_ptr<Executor> ExecutorScope::Current() {
-    if (g_executor_scope.ScopedLevel() > 0) {
-        return g_executor_scope.Current().content;
+    if (_getGlobalScope()->ScopedLevel() > 0) {
+        return _getGlobalScope()->Current().content;
     }
     return Executor::getGlobalExecutor();
 }

express/Expr.cpp

@@ -184,12 +184,10 @@ EXPRP Expr::create(Variable::Info&& info, const void* ptr, VARP::InputType type,
     }
     return expr;
 }
-EXPRP Expr::create(std::pair<std::shared_ptr<char>, int> extra, std::vector<VARP>&& inputs, int outputSize) {
+EXPRP Expr::create(std::shared_ptr<BufferStorage> extra, std::vector<VARP>&& inputs, int outputSize) {
     EXPRP expr(new Expr(outputSize));
-    expr->mExtraBuffer = extra.first;
-    expr->mOpBufferSize = extra.second;
-    expr->mOp = flatbuffers::GetMutableRoot<Op>(extra.first.get());
-    expr->mOpBufferSize = extra.second;
+    expr->mStorage = extra;
+    expr->mOp = flatbuffers::GetRoot<Op>(extra->buffer());
     expr->mInputs   = std::move(inputs);
     expr->mInside->mReq = ExecutorScope::Current()->getRequirement(expr.get());
     _addLinkForInputs(expr);
@@ -239,9 +237,9 @@ EXPRP Expr::create(const OpT* op, std::vector<VARP> inputs, int outputSize) {
     flatbuffers::FlatBufferBuilder builder;
     auto offset = Op::Pack(builder, op);
     builder.Finish(offset);
-    std::shared_ptr<char> extraBuffer(new char[builder.GetSize()], std::default_delete<char[]>());
-    ::memcpy(extraBuffer.get(), builder.GetBufferPointer(), builder.GetSize());
-    auto resExpr = Expr::create(std::make_pair(extraBuffer, builder.GetSize()), std::move(inputs), outputSize);
+    std::shared_ptr<BufferStorage> extra(new BufferStorage);
+    extra->storage.reset(builder.ReleaseRaw(extra->allocated_size, extra->offset));
+    auto resExpr = Expr::create(extra, std::move(inputs), outputSize);
     resExpr->setName(op->name);
     return resExpr;
 }
@@ -356,8 +354,7 @@ void Expr::replace(EXPRP old, EXPRP from) {
     old->mOp = from->mOp;
     old->mName = from->mName;
     old->mOutputNames = from->mOutputNames;
-    old->mExtraBuffer = from->mExtraBuffer;
-    old->mOpBufferSize = from->mOpBufferSize;
+    old->mStorage = from->mStorage;
     old->mType = from->mType;
     old->mValid = from->mValid;
     old->mInside = from->mInside;

express/MathOp.cpp

@@ -11,7 +11,7 @@
 #include <numeric>
 #include <MNN/expr/ExprCreator.hpp>
 #include <MNN/MNNDefine.h>
-#include "MNN_generated.h"
+#include "Utils.hpp"
 
 namespace MNN {
 namespace Express {
@@ -42,45 +42,72 @@ static VARP _checkNC4HW4(VARP x) {
 static VARP _Binary(VARP x, VARP y, BinaryOpOperation operation) {
     x = _checkNC4HW4(x);
     y = _checkNC4HW4(y);
-    std::unique_ptr<OpT> op(new OpT);
-    op->main.type                 = OpParameter_BinaryOp;
-    op->type                      = OpType_BinaryOp;
-    op->main.value                = new BinaryOpT;
-    op->main.AsBinaryOp()->opType = operation;
-    op->main.AsBinaryOp()->T      = DataType_DT_FLOAT;
-    return (Variable::create(Expr::create(op.get(), {x, y})));
+    flatbuffers::FlatBufferBuilder builder;
+    BinaryOpBuilder parameter(builder);
+    parameter.add_opType(operation);
+    auto paOffset = parameter.Finish();
+    OpBuilder opB(builder);
+    opB.add_main(paOffset.Union());
+    opB.add_type(OpType_BinaryOp);
+    opB.add_main_type(OpParameter_BinaryOp);
+    builder.Finish(opB.Finish());
+    std::shared_ptr<BufferStorage> extra(new BufferStorage);
+    extra->storage.reset(builder.ReleaseRaw(extra->allocated_size, extra->offset));
+    return Variable::create(Expr::create(extra, {x, y}, 1));
 }
 static VARP _Unary(VARP x, UnaryOpOperation operation) {
-    std::unique_ptr<OpT> op(new OpT);
-    op->main.type                = OpParameter_UnaryOp;
-    op->type                     = OpType_UnaryOp;
-    op->main.value               = new UnaryOpT;
-    op->main.AsUnaryOp()->opType = operation;
-    op->main.AsUnaryOp()->T      = DataType_DT_FLOAT;
-    return (Variable::create(Expr::create(op.get(), {x})));
+    flatbuffers::FlatBufferBuilder builder;
+    UnaryOpBuilder parameter(builder);
+    parameter.add_opType(operation);
+    auto paOffset = parameter.Finish();
+    OpBuilder opB(builder);
+    opB.add_main(paOffset.Union());
+    opB.add_type(OpType_UnaryOp);
+    opB.add_main_type(OpParameter_UnaryOp);
+    builder.Finish(opB.Finish());
+    std::shared_ptr<BufferStorage> extra(new BufferStorage);
+    extra->storage.reset(builder.ReleaseRaw(extra->allocated_size, extra->offset));
+    return Variable::create(Expr::create(extra, {x}, 1));
 }
 static VARP _Reduce(VARP x, INTS dim, ReductionType type, bool keepDim) {
     x = _checkNC4HW4(x);
-    std::unique_ptr<OpT> op(new OpT);
-    op->main.type                          = OpParameter_ReductionParam;
-    op->type                               = OpType_Reduction;
-    op->main.value                         = new ReductionParamT;
-    op->main.AsReductionParam()->dType     = DataType_DT_FLOAT;
-    op->main.AsReductionParam()->operation = type;
-    op->main.AsReductionParam()->dim       = dim;
-    op->main.AsReductionParam()->keepDims  = keepDim;
-    return (Variable::create(Expr::create(op.get(), {x})));
+    flatbuffers::FlatBufferBuilder builder;
+    flatbuffers::Offset<flatbuffers::Vector<int>> dimOffset;
+    if (!dim.empty()) {
+        dimOffset = builder.CreateVector(dim);
+    }
+    ReductionParamBuilder parameter(builder);
+    parameter.add_operation(type);
+    parameter.add_keepDims(keepDim);
+    if (!dim.empty()) {
+        parameter.add_dim(dimOffset);
+    }
+    auto paOffset = parameter.Finish();
+    OpBuilder opB(builder);
+    opB.add_main(paOffset.Union());
+    opB.add_type(OpType_Reduction);
+    opB.add_main_type(OpParameter_ReductionParam);
+    builder.Finish(opB.Finish());
+    std::shared_ptr<BufferStorage> extra(new BufferStorage);
+    extra->storage.reset(builder.ReleaseRaw(extra->allocated_size, extra->offset));
+    return Variable::create(Expr::create(extra, {x}, 1));
 }
 static VARP _ReduceMutable(VARP x, VARP dim, ReductionType type, bool keepDim) {
     x = _checkNC4HW4(x);
-    std::unique_ptr<OpT> op(new OpT);
-    op->main.type                          = OpParameter_ReductionParam;
-    op->type                               = OpType_Reduction;
-    op->main.value                         = new ReductionParamT;
-    op->main.AsReductionParam()->dType     = DataType_DT_FLOAT;
-    op->main.AsReductionParam()->operation = type;
-    op->main.AsReductionParam()->keepDims  = keepDim;
-    return (Variable::create(Expr::create(op.get(), {x, dim})));
+    flatbuffers::FlatBufferBuilder builder;
+    ReductionParamBuilder parameter(builder);
+    parameter.add_operation(type);
+    parameter.add_keepDims(keepDim);
+    auto paOffset = parameter.Finish();
+    OpBuilder opB(builder);
+    opB.add_main(paOffset.Union());
+    opB.add_type(OpType_Reduction);
+    opB.add_main_type(OpParameter_ReductionParam);
+    builder.Finish(opB.Finish());
+    // TODO: Remove Copy
+    std::shared_ptr<BufferStorage> extra(new BufferStorage);
+    extra->storage.reset(builder.ReleaseRaw(extra->allocated_size, extra->offset));
+    return Variable::create(Expr::create(extra, {x, dim}, 1));
 }
 static VARP _Eltwise(VARP a, VARP b, EltwiseType type, std::vector<float> coeff) {
     std::unique_ptr<OpT> op(new OpT);

express/Utils.hpp

@@ -15,6 +15,18 @@
 #include <MNN/expr/Executor.hpp>
 namespace MNN {
 namespace Express {
+struct BufferStorage {
+    size_t size() const {
+        return allocated_size - offset;
+    }
+
+    const uint8_t* buffer() const {
+        return storage.get() + offset;
+    }
+    size_t allocated_size;
+    size_t offset;
+    std::unique_ptr<uint8_t> storage;
+};
 struct Expr::Inside {
     Inside(int outputSize);
     Inside(Tensor* tensor);

express/module/IfModule.cpp

@@ -81,6 +81,8 @@ IfModule* IfModule::create(const Op* op, const std::map<std::string, SubGraph>&
             }
         }
     }
+    MNN_ASSERT(module->mInputForElse.size() == elseG.inputs.size());
+    MNN_ASSERT(module->mInputForThen.size() == thenG.inputs.size());
     // Map outputs
     auto output = ifParam->aliases_outputs();
     module->mOutputFromThen.resize(output->size());

express/module/PipelineModule.cpp

@@ -399,11 +399,10 @@ void PipelineModule::onClearCache() {
     // Do nothing
 }
 
-static std::map<std::string, SubGraph> _createSubGraph(const MNN::Net* net, const Module::Config* config) {
-    std::map<std::string, SubGraph> subGraphMap;
+void PipelineModule::_createSubGraph(const MNN::Net* net, const Module::Config* config, std::map<std::string, SubGraph>& subGraphMap) {
     auto subGraphs = net->subgraphs();
     if (nullptr == subGraphs) {
-        return subGraphMap;
+        return;
     }
     for (int i=0; i<subGraphs->size(); ++i) {
         auto graph = subGraphs->GetAs<SubGraphProto>(i);
@@ -429,14 +428,7 @@ static std::map<std::string, SubGraph> _createSubGraph(const MNN::Net* net, cons
             flatbuffers::FlatBufferBuilder builder(1024);
             auto offset = Net::Pack(builder, _tempNet.get());
             builder.Finish(offset);
-            // if subgraph contain WhereOp, it's need splite
-            auto iter = _tempNet->oplists.begin();
-            for (; iter != _tempNet->oplists.end() && iter->get()->type != MNN::OpType_Where; iter++);
-            if (config->dynamic || iter != _tempNet->oplists.end()) {
-                submodule.reset(PipelineModule::load(subInputs, subOutputs, (const uint8_t*)builder.GetBufferPointer(), builder.GetSize(), config));
-            } else {
-                submodule.reset(new StaticModule((const uint8_t*)builder.GetBufferPointer(), builder.GetSize(), subInputs, subOutputs, *config));
-            }
+            submodule.reset(PipelineModule::load(subInputs, subOutputs, (const uint8_t*)builder.GetBufferPointer(), builder.GetSize(), config, subGraphMap, true));
             if (graph->name() != nullptr) {
                 submodule->setName(graph->name()->str());
             }
@@ -448,7 +440,7 @@ static std::map<std::string, SubGraph> _createSubGraph(const MNN::Net* net, cons
         subgraph.m = submodule;
         subGraphMap.insert(std::make_pair(key, subgraph));
     }
-    return subGraphMap;
+    return;
 }
 
 struct SubModuleInfo {
@@ -589,7 +581,7 @@ static std::vector<SubModuleInfo> _createSubModuleInfo(const MNN::Net* net, cons
     return submodule;
 }
 
-static Module* _createSubModule(const MNN::Net* net, const SubModuleInfo& info, const std::map<std::string, SubGraph>& subs, const Module::Config& config) {
+static Module* _createSubModule(const MNN::Net* net, const SubModuleInfo& info, const std::map<std::string, SubGraph>& subs, const Module::Config& config, bool inRecurse) {
     if (1 == info.opList.size()) {
         auto op = net->oplists()->GetAs<Op>(info.opList[0]);
         if (OpType_If == op->type()) {
@@ -633,7 +625,7 @@ static Module* _createSubModule(const MNN::Net* net, const SubModuleInfo& info,
     auto offset = Net::Pack(builder, _tempNet.get());
     builder.Finish(offset);
     _tempNet.reset();
-    return new StaticModule((const uint8_t*)builder.GetBufferPointer(), builder.GetSize(), inputNames, outputNames, config);
+    return new StaticModule((const uint8_t*)builder.GetBufferPointer(), builder.GetSize(), inputNames, outputNames, config, inRecurse);
 }
 
 Module* PipelineModule::load(const std::vector<std::string>& inputs, const std::vector<std::string>& outputs, const uint8_t* buffer, size_t length, const Module::Config* config) {
@@ -648,17 +640,34 @@ Module* PipelineModule::load(const std::vector<std::string>& inputs, const std::
         MNN_ERROR("Invalid net, for null oplist or tensorName\n");
         return nullptr;
     }
+    std::map<std::string, SubGraph> subGraphMap;
+    _createSubGraph(net, config, subGraphMap);
+    return load(inputs, outputs, buffer, length, config, subGraphMap);
+}
+Module* PipelineModule::load(const std::vector<std::string>& inputs, const std::vector<std::string>& outputs, const uint8_t* buffer, size_t length, const Module::Config* config, std::map<std::string, SubGraph>& subGraphMap, bool inRecurce) {
+    auto net = GetNet(buffer);
     if (!config->dynamic) {
-        if (nullptr == subGraphs) {
-            auto iter = net->oplists()->begin();
-            for (; iter != net->oplists()->end() && iter->type() != OpType_Where; iter++);
-            if (iter == net->oplists()->end()) {
-                // Has no control flow and WhereOp, can just use static module
-                return new StaticModule(buffer, length, inputs, outputs, *config);
+        bool linear = true;
+        auto iter = net->oplists()->begin();
+        for (; iter != net->oplists()->end(); iter++) {
+            if (iter->type() == OpType_While) {
+                linear = false;
+                break;
             }
+            if (iter->type() == OpType_If) {
+                linear = false;
+                break;
+            }
+            if (iter->type() == OpType_Where) {
+                linear = false;
+                break;
+            }
+        }
+        if (linear) {
+            // Has no control flow and WhereOp, can just use static module
+            return new StaticModule(buffer, length, inputs, outputs, *config, false);
         }
     }
-    auto subGraphMap = _createSubGraph(net, config);
     if (config->dynamic) {
         // For dynamic mode
         auto varMaps = Variable::loadMap(buffer, length);
@@ -705,7 +714,7 @@ Module* PipelineModule::load(const std::vector<std::string>& inputs, const std::
     auto subModulesInfo = _createSubModuleInfo(net, inputIndexes, outputIndexes);
     std::vector<std::shared_ptr<Module>> subModules(subModulesInfo.size());
     for (int i=0; i<subModulesInfo.size(); ++i) {
-        subModules[i].reset(_createSubModule(net, subModulesInfo[i], subGraphMap, *config));
+        subModules[i].reset(_createSubModule(net, subModulesInfo[i], subGraphMap, *config, inRecurce));
     }
     auto result = new PipelineModule;
     /**