diff --git a/nnpackage/schema/circle_schema.fbs b/nnpackage/schema/circle_schema.fbs index 460fa43ee11..0dc86d63fcd 100644 --- a/nnpackage/schema/circle_schema.fbs +++ b/nnpackage/schema/circle_schema.fbs @@ -76,9 +76,22 @@ table CustomQuantization { custom:[ubyte] (force_align: 16); } +// Chunk(block) quantization +table CircleChunkQuantization { + // for debugging, optional + name:string; + // chunk size + size:uint; + // min on 4,5 bit. sum on 8bit + has_min_or_sum:bool; + // super-block quantization + is_super:bool; +} + // Represents a specific quantization technique's parameters. union QuantizationDetails { CustomQuantization, + CircleChunkQuantization } // Parameters for converting a quantized tensor back to float. diff --git a/runtime/libs/circle-schema/include/circle_schema_generated.h b/runtime/libs/circle-schema/include/circle_schema_generated.h index 3da596b1ebc..89e6de2d7cc 100644 --- a/runtime/libs/circle-schema/include/circle_schema_generated.h +++ b/runtime/libs/circle-schema/include/circle_schema_generated.h @@ -35,6 +35,10 @@ struct CustomQuantization; struct CustomQuantizationBuilder; struct CustomQuantizationT; +struct CircleChunkQuantization; +struct CircleChunkQuantizationBuilder; +struct CircleChunkQuantizationT; + struct QuantizationParameters; struct QuantizationParametersBuilder; struct QuantizationParametersT; @@ -756,27 +760,30 @@ enum QuantizationDetails : uint8_t { QuantizationDetails_NONE = 0, QuantizationDetails_CustomQuantization = 1, + QuantizationDetails_CircleChunkQuantization = 2, QuantizationDetails_MIN = QuantizationDetails_NONE, - QuantizationDetails_MAX = QuantizationDetails_CustomQuantization + QuantizationDetails_MAX = QuantizationDetails_CircleChunkQuantization }; -inline const QuantizationDetails (&EnumValuesQuantizationDetails())[2] +inline const QuantizationDetails (&EnumValuesQuantizationDetails())[3] { static const QuantizationDetails values[] = {QuantizationDetails_NONE, - QuantizationDetails_CustomQuantization}; + QuantizationDetails_CustomQuantization, + QuantizationDetails_CircleChunkQuantization}; return values; } inline const char *const *EnumNamesQuantizationDetails() { - static const char *const names[3] = {"NONE", "CustomQuantization", nullptr}; + static const char *const names[4] = {"NONE", "CustomQuantization", "CircleChunkQuantization", + nullptr}; return names; } inline const char *EnumNameQuantizationDetails(QuantizationDetails e) { if (::flatbuffers::IsOutRange(e, QuantizationDetails_NONE, - QuantizationDetails_CustomQuantization)) + QuantizationDetails_CircleChunkQuantization)) return ""; const size_t index = static_cast(e); return EnumNamesQuantizationDetails()[index]; @@ -792,6 +799,11 @@ template <> struct QuantizationDetailsTraits static const QuantizationDetails enum_value = QuantizationDetails_CustomQuantization; }; +template <> struct QuantizationDetailsTraits +{ + static const QuantizationDetails enum_value = QuantizationDetails_CircleChunkQuantization; +}; + template struct QuantizationDetailsUnionTraits { static const QuantizationDetails enum_value = QuantizationDetails_NONE; @@ -802,6 +814,11 @@ template <> struct QuantizationDetailsUnionTraits static const QuantizationDetails enum_value = QuantizationDetails_CustomQuantization; }; +template <> struct QuantizationDetailsUnionTraits +{ + static const QuantizationDetails enum_value = QuantizationDetails_CircleChunkQuantization; +}; + struct QuantizationDetailsUnion { QuantizationDetails type; @@ -862,6 +879,18 @@ struct QuantizationDetailsUnion ? reinterpret_cast(value) : nullptr; } + circle::CircleChunkQuantizationT *AsCircleChunkQuantization() + { + return type == QuantizationDetails_CircleChunkQuantization + ? reinterpret_cast(value) + : nullptr; + } + const circle::CircleChunkQuantizationT *AsCircleChunkQuantization() const + { + return type == QuantizationDetails_CircleChunkQuantization + ? reinterpret_cast(value) + : nullptr; + } }; bool VerifyQuantizationDetails(::flatbuffers::Verifier &verifier, const void *obj, @@ -6240,6 +6269,111 @@ ::flatbuffers::Offset CreateCustomQuantization(::flatbuffers::FlatBufferBuilder &_fbb, const CustomQuantizationT *_o, const ::flatbuffers::rehasher_function_t *_rehasher = nullptr); +struct CircleChunkQuantizationT : public ::flatbuffers::NativeTable +{ + typedef CircleChunkQuantization TableType; + std::string name{}; + uint32_t size = 0; + bool has_min_or_sum = false; + bool is_super = false; +}; + +struct CircleChunkQuantization FLATBUFFERS_FINAL_CLASS : private ::flatbuffers::Table +{ + typedef CircleChunkQuantizationT NativeTableType; + typedef CircleChunkQuantizationBuilder Builder; + enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE + { + VT_NAME = 4, + VT_SIZE = 6, + VT_HAS_MIN_OR_SUM = 8, + VT_IS_SUPER = 10 + }; + const ::flatbuffers::String *name() const + { + return GetPointer(VT_NAME); + } + uint32_t size() const { return GetField(VT_SIZE, 0); } + bool has_min_or_sum() const { return GetField(VT_HAS_MIN_OR_SUM, 0) != 0; } + bool is_super() const { return GetField(VT_IS_SUPER, 0) != 0; } + bool Verify(::flatbuffers::Verifier &verifier) const + { + return VerifyTableStart(verifier) && VerifyOffset(verifier, VT_NAME) && + verifier.VerifyString(name()) && VerifyField(verifier, VT_SIZE, 4) && + VerifyField(verifier, VT_HAS_MIN_OR_SUM, 1) && + VerifyField(verifier, VT_IS_SUPER, 1) && verifier.EndTable(); + } + CircleChunkQuantizationT * + UnPack(const ::flatbuffers::resolver_function_t *_resolver = nullptr) const; + void UnPackTo(CircleChunkQuantizationT *_o, + const ::flatbuffers::resolver_function_t *_resolver = nullptr) const; + static ::flatbuffers::Offset + Pack(::flatbuffers::FlatBufferBuilder &_fbb, const CircleChunkQuantizationT *_o, + const ::flatbuffers::rehasher_function_t *_rehasher = nullptr); +}; + +struct CircleChunkQuantizationBuilder +{ + typedef CircleChunkQuantization Table; + ::flatbuffers::FlatBufferBuilder &fbb_; + ::flatbuffers::uoffset_t start_; + void add_name(::flatbuffers::Offset<::flatbuffers::String> name) + { + fbb_.AddOffset(CircleChunkQuantization::VT_NAME, name); + } + void add_size(uint32_t size) + { + fbb_.AddElement(CircleChunkQuantization::VT_SIZE, size, 0); + } + void add_has_min_or_sum(bool has_min_or_sum) + { + fbb_.AddElement(CircleChunkQuantization::VT_HAS_MIN_OR_SUM, + static_cast(has_min_or_sum), 0); + } + void add_is_super(bool is_super) + { + fbb_.AddElement(CircleChunkQuantization::VT_IS_SUPER, static_cast(is_super), + 0); + } + explicit CircleChunkQuantizationBuilder(::flatbuffers::FlatBufferBuilder &_fbb) : fbb_(_fbb) + { + start_ = fbb_.StartTable(); + } + ::flatbuffers::Offset Finish() + { + const auto end = fbb_.EndTable(start_); + auto o = ::flatbuffers::Offset(end); + return o; + } +}; + +inline ::flatbuffers::Offset +CreateCircleChunkQuantization(::flatbuffers::FlatBufferBuilder &_fbb, + ::flatbuffers::Offset<::flatbuffers::String> name = 0, + uint32_t size = 0, bool has_min_or_sum = false, bool is_super = false) +{ + CircleChunkQuantizationBuilder builder_(_fbb); + builder_.add_size(size); + builder_.add_name(name); + builder_.add_is_super(is_super); + builder_.add_has_min_or_sum(has_min_or_sum); + return builder_.Finish(); +} + +inline ::flatbuffers::Offset +CreateCircleChunkQuantizationDirect(::flatbuffers::FlatBufferBuilder &_fbb, + const char *name = nullptr, uint32_t size = 0, + bool has_min_or_sum = false, bool is_super = false) +{ + auto name__ = name ? _fbb.CreateString(name) : 0; + return circle::CreateCircleChunkQuantization(_fbb, name__, size, has_min_or_sum, is_super); +} + +::flatbuffers::Offset +CreateCircleChunkQuantization(::flatbuffers::FlatBufferBuilder &_fbb, + const CircleChunkQuantizationT *_o, + const ::flatbuffers::rehasher_function_t *_rehasher = nullptr); + struct QuantizationParametersT : public ::flatbuffers::NativeTable { typedef QuantizationParameters TableType; @@ -6293,6 +6427,12 @@ struct QuantizationParameters FLATBUFFERS_FINAL_CLASS : private ::flatbuffers::T ? static_cast(details()) : nullptr; } + const circle::CircleChunkQuantization *details_as_CircleChunkQuantization() const + { + return details_type() == circle::QuantizationDetails_CircleChunkQuantization + ? static_cast(details()) + : nullptr; + } int32_t quantized_dimension() const { return GetField(VT_QUANTIZED_DIMENSION, 0); } bool Verify(::flatbuffers::Verifier &verifier) const { @@ -6322,6 +6462,13 @@ QuantizationParameters::details_as() const return details_as_CustomQuantization(); } +template <> +inline const circle::CircleChunkQuantization * +QuantizationParameters::details_as() const +{ + return details_as_CircleChunkQuantization(); +} + struct QuantizationParametersBuilder { typedef QuantizationParameters Table; @@ -20734,6 +20881,68 @@ CreateCustomQuantization(::flatbuffers::FlatBufferBuilder &_fbb, const CustomQua return circle::CreateCustomQuantization(_fbb, _custom); } +inline CircleChunkQuantizationT * +CircleChunkQuantization::UnPack(const ::flatbuffers::resolver_function_t *_resolver) const +{ + auto _o = std::unique_ptr(new CircleChunkQuantizationT()); + UnPackTo(_o.get(), _resolver); + return _o.release(); +} + +inline void +CircleChunkQuantization::UnPackTo(CircleChunkQuantizationT *_o, + const ::flatbuffers::resolver_function_t *_resolver) const +{ + (void)_o; + (void)_resolver; + { + auto _e = name(); + if (_e) + _o->name = _e->str(); + } + { + auto _e = size(); + _o->size = _e; + } + { + auto _e = has_min_or_sum(); + _o->has_min_or_sum = _e; + } + { + auto _e = is_super(); + _o->is_super = _e; + } +} + +inline ::flatbuffers::Offset +CircleChunkQuantization::Pack(::flatbuffers::FlatBufferBuilder &_fbb, + const CircleChunkQuantizationT *_o, + const ::flatbuffers::rehasher_function_t *_rehasher) +{ + return CreateCircleChunkQuantization(_fbb, _o, _rehasher); +} + +inline ::flatbuffers::Offset +CreateCircleChunkQuantization(::flatbuffers::FlatBufferBuilder &_fbb, + const CircleChunkQuantizationT *_o, + const ::flatbuffers::rehasher_function_t *_rehasher) +{ + (void)_rehasher; + (void)_o; + struct _VectorArgs + { + ::flatbuffers::FlatBufferBuilder *__fbb; + const CircleChunkQuantizationT *__o; + const ::flatbuffers::rehasher_function_t *__rehasher; + } _va = {&_fbb, _o, _rehasher}; + (void)_va; + auto _name = _o->name.empty() ? 0 : _fbb.CreateString(_o->name); + auto _size = _o->size; + auto _has_min_or_sum = _o->has_min_or_sum; + auto _is_super = _o->is_super; + return circle::CreateCircleChunkQuantization(_fbb, _name, _size, _has_min_or_sum, _is_super); +} + inline QuantizationParametersT * QuantizationParameters::UnPack(const ::flatbuffers::resolver_function_t *_resolver) const { @@ -29893,6 +30102,11 @@ inline bool VerifyQuantizationDetails(::flatbuffers::Verifier &verifier, const v auto ptr = reinterpret_cast(obj); return verifier.VerifyTable(ptr); } + case QuantizationDetails_CircleChunkQuantization: + { + auto ptr = reinterpret_cast(obj); + return verifier.VerifyTable(ptr); + } default: return true; } @@ -29929,6 +30143,11 @@ inline void *QuantizationDetailsUnion::UnPack(const void *obj, QuantizationDetai auto ptr = reinterpret_cast(obj); return ptr->UnPack(resolver); } + case QuantizationDetails_CircleChunkQuantization: + { + auto ptr = reinterpret_cast(obj); + return ptr->UnPack(resolver); + } default: return nullptr; } @@ -29946,6 +30165,11 @@ QuantizationDetailsUnion::Pack(::flatbuffers::FlatBufferBuilder &_fbb, auto ptr = reinterpret_cast(value); return CreateCustomQuantization(_fbb, ptr, _rehasher).Union(); } + case QuantizationDetails_CircleChunkQuantization: + { + auto ptr = reinterpret_cast(value); + return CreateCircleChunkQuantization(_fbb, ptr, _rehasher).Union(); + } default: return 0; } @@ -29962,6 +30186,12 @@ inline QuantizationDetailsUnion::QuantizationDetailsUnion(const QuantizationDeta new circle::CustomQuantizationT(*reinterpret_cast(u.value)); break; } + case QuantizationDetails_CircleChunkQuantization: + { + value = new circle::CircleChunkQuantizationT( + *reinterpret_cast(u.value)); + break; + } default: break; } @@ -29977,6 +30207,12 @@ inline void QuantizationDetailsUnion::Reset() delete ptr; break; } + case QuantizationDetails_CircleChunkQuantization: + { + auto ptr = reinterpret_cast(value); + delete ptr; + break; + } default: break; } diff --git a/runtime/onert/core/src/loader/CircleLoader.cc b/runtime/onert/core/src/loader/CircleLoader.cc index f6c822fd442..ea441fe603f 100644 --- a/runtime/onert/core/src/loader/CircleLoader.cc +++ b/runtime/onert/core/src/loader/CircleLoader.cc @@ -91,12 +91,13 @@ class CircleLoader final : public loader::BaseLoader ir::DataType getTensorDataType(const Tensor *tensor) override { auto type = tensor->type(); - // Workaround: No quantization parameter for chunk quantization - // Actual parameter(scale) for each chunk is in data - // TODO Handle custom quantization parameter to represent chunk quantization - if (type == TensorType::TensorType_UINT4 && !tensor->quantization()) + if (type == TensorType::TensorType_UINT4 && tensor->quantization() && + tensor->quantization()->details_type() == + circle::QuantizationDetails::QuantizationDetails_CircleChunkQuantization) return ir::DataType::QUANT_UINT4_SYMM_PER_CHUNK; - if (type == TensorType::TensorType_INT8 && !tensor->quantization()) + if (type == TensorType::TensorType_INT8 && tensor->quantization() && + tensor->quantization()->details_type() == + circle::QuantizationDetails::QuantizationDetails_CircleChunkQuantization) return ir::DataType::QUANT_INT8_SYMM_PER_CHUNK; return tensorTypeToDataType(type); }