From f56171ff5fbdf3ad846109ca1bf4e75d5cda2d7f Mon Sep 17 00:00:00 2001 From: "y01000.you" Date: Thu, 31 Oct 2024 18:49:15 +0900 Subject: [PATCH] Refactoring and clean-up This patch includes minor refactoring to improve overall code quality. ONE-DCO-1.0-Signed-off-by: y01000.you --- .../luci/Pass/QuantizeWeightsWithGPTQPass.h | 14 ++-- .../pass/src/QuantizeWeightsWithGPTQPass.cpp | 76 ++++++++++--------- .../src/QuantizeWeightsWithGPTQPass.test.cpp | 1 + 3 files changed, 49 insertions(+), 42 deletions(-) diff --git a/compiler/luci/pass/include/luci/Pass/QuantizeWeightsWithGPTQPass.h b/compiler/luci/pass/include/luci/Pass/QuantizeWeightsWithGPTQPass.h index afef945bf98..a4ac4620404 100644 --- a/compiler/luci/pass/include/luci/Pass/QuantizeWeightsWithGPTQPass.h +++ b/compiler/luci/pass/include/luci/Pass/QuantizeWeightsWithGPTQPass.h @@ -17,17 +17,19 @@ #ifndef __LUCI_QUANTIZE_WEIGHTS_WITH_GPTQ_PASS_H__ #define __LUCI_QUANTIZE_WEIGHTS_WITH_GPTQ_PASS_H__ -#include +#include +#include #include +#include -#include -#include #include namespace luci { +using HessianMap = std::unordered_map>; + /** * @brief Pass to quantize weights */ @@ -48,9 +50,7 @@ class QuantizeWeightsWithGPTQPass : public logo::Pass // DO NOTHING } - QuantizeWeightsWithGPTQPass( - std::unique_ptr &&ctx, - std::unordered_map> *hessian_map) + QuantizeWeightsWithGPTQPass(std::unique_ptr &&ctx, HessianMap *hessian_map) : _ctx{std::move(ctx)}, _hessian_map{hessian_map} { // DO NOTHING @@ -74,7 +74,7 @@ class QuantizeWeightsWithGPTQPass : public logo::Pass private: std::unique_ptr _ctx; - std::unordered_map> *_hessian_map; + HessianMap *_hessian_map; }; } // namespace luci diff --git a/compiler/luci/pass/src/QuantizeWeightsWithGPTQPass.cpp b/compiler/luci/pass/src/QuantizeWeightsWithGPTQPass.cpp index cd9267117a0..79e5a2bf9c2 100644 --- a/compiler/luci/pass/src/QuantizeWeightsWithGPTQPass.cpp +++ b/compiler/luci/pass/src/QuantizeWeightsWithGPTQPass.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020 Samsung Electronics Co., Ltd. All Rights Reserved + * Copyright (c) 2024 Samsung Electronics Co., Ltd. All Rights Reserved * Copyright 2019 The TensorFlow Authors. All Rights Reserved. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -27,16 +27,19 @@ #include #include #include -#include + +namespace luci +{ namespace { -using namespace luci; using IterFunc = std::function; void iterate_per_channel_with_order(CircleConst *node, IterFunc func, bool reverse) { + assert(node != nullptr); + loco::TensorShape dimension; dimension.rank(4); uint32_t indices[4] = {0}; @@ -44,7 +47,7 @@ void iterate_per_channel_with_order(CircleConst *node, IterFunc func, bool rever uint32_t num_dims[4]; if (!get_channel_dim_index(node, dimension, channel_dim_index)) { - throw std::runtime_error("Failed to get channel dim index."); + throw std::runtime_error("GPTQPass: Failed to get channel dim index."); } auto order = reverse ? std::vector{3, 1, 2, 0} : std::vector{0, 1, 2, 3}; @@ -73,19 +76,15 @@ void iterate_per_channel_with_order(CircleConst *node, IterFunc func, bool rever } } -} // namespace - -namespace luci -{ - -namespace -{ - size_t calculate_qauntized_value(CircleConst *node, uint32_t *indices, loco::TensorShape &dimension, int channel_dim_index, std::vector &scaling_factor, std::vector &max, std::vector &min) { + assert(node != nullptr); + int channel_idx = indices[channel_dim_index]; + + assert(scaling_factor[channel_idx] > 0); const float scaling_factor_inv = 1.0 / scaling_factor[channel_idx]; auto data = node->at(cal_offset(dimension, indices)); auto data_clipped = data < min[channel_idx] ? min[channel_idx] : data; @@ -109,7 +108,7 @@ void cholesky_decomposition(std::vector &src, uint32_t num_size) { if (src[i * num_size + i] - sum <= 0) { - std::cout << "Error: Matrix is not positive definite.\n" << std::endl; + std::cout << "Error: Matrix is not positive definite." << std::endl; return; } src[i * num_size + i] = sqrt(src[i * num_size + i] - sum); @@ -143,6 +142,7 @@ void forward_substitution(const std::vector &L, const std::vector { y[i] -= L[i * num_size + j] * y[j]; } + assert(L[i * num_size + i] != 0); y[i] /= L[i * num_size + i]; } } @@ -157,6 +157,7 @@ void backward_substitution(const std::vector &U, const std::vector { x[i] -= U[i * num_size + j] * x[j]; } + assert(U[i * num_size + i] != 0); x[i] /= U[i * num_size + i]; } } @@ -222,8 +223,7 @@ void cal_minmax_per_channel(CircleConst *node, std::vector &min, std::vec if (!get_channel_dim_index(node, dimension, channel_dim_index)) { - assert(false); - return; + throw std::runtime_error("GPTQPass: Failed to get channel dim index."); } auto size = dimension.dim(channel_dim_index).value(); @@ -262,10 +262,13 @@ void compute_asym_scale_zp(float min, float max, float &scaling_factor, int64_t const double qmax_double = kMaxScale; const double rmin = std::fmin(0, min); const double rmax = std::fmax(0, max); + const double qrange = qmax_double - qmin_double; + assert(qrange > 0); - double scale = (rmax - rmin) / (qmax_double - qmin_double); + double scale = (rmax - rmin) / qrange; double zero_point_double = 0; uint8_t nudged_zero_point = 0; + if (scale == 0) { WARN(l) << "The minimum and maximum values are the same." << std::endl; @@ -280,7 +283,7 @@ void compute_asym_scale_zp(float min, float max, float &scaling_factor, int64_t { assert(min >= 0 && max >= 0); nudged_zero_point = kMinScale; - scale = max / (qmax_double - qmin_double); + scale = max / qrange; if (min > 0 && max > 0) WARN(l) << "The minimum and maximum values are all positive." << std::endl; } @@ -288,7 +291,7 @@ void compute_asym_scale_zp(float min, float max, float &scaling_factor, int64_t { assert(min < 0 && max < 0); nudged_zero_point = kMaxScale; - scale = -min / (qmax_double - qmin_double); + scale = -min / qrange; WARN(l) << "The minimum and maximum values are all negative." << std::endl; } else @@ -318,6 +321,7 @@ void asymmetric_wquant_per_channel(CircleConst *node, std::vector &min, std::vector &hessian) { assert(node->dtype() == loco::DataType::FLOAT32); + assert(output_type == loco::DataType::U8 || output_type != loco::DataType::U4); IterFunc quantize; @@ -333,7 +337,7 @@ void asymmetric_wquant_per_channel(CircleConst *node, std::vector &min, kMaxScale); } - if (hessian.empty()) // Cases where gptq is not applied + if (hessian.empty()) // Case where GPTQ is not applied { quantize = [&](uint32_t *indices, loco::TensorShape &dimension, int channel_dim_index) { quantized_values[cal_offset(dimension, indices)] = calculate_qauntized_value( @@ -341,7 +345,7 @@ void asymmetric_wquant_per_channel(CircleConst *node, std::vector &min, }; iterate_per_channel_with_order(node, quantize, false); } - else // Cases where gptq is applied + else // Case where GPTQ is applied { uint32_t size_hessian = static_cast(sqrt(hessian.size())); float percdamp = .01; @@ -364,7 +368,7 @@ void asymmetric_wquant_per_channel(CircleConst *node, std::vector &min, cholesky_inverse(hessian, size_hessian); cholesky_decomposition(hessian, size_hessian); - // transpose hessian to make upper trangular + // transpose hessian to make upper triangular for (uint32_t i = 0; i < size_hessian; i++) { for (uint32_t j = 0; j < i; j++) @@ -405,10 +409,11 @@ void asymmetric_wquant_per_channel(CircleConst *node, std::vector &min, uint32_t channel_idx = indices[channel_dim_index]; auto data = node->at(cal_offset(dimension, indices)); + auto h_offset = cal_offset_2d(dimension_hessian, indices_diag_hessian); error[cal_offset(dimension, indices)] = (data - (quantized_values[cal_offset(dimension, indices)] - zp[channel_idx]) * scaling_factor[channel_idx]) / - hessian[cal_offset_2d(dimension_hessian, indices_diag_hessian)]; + hessian[h_offset]; if (channel_idx == (dimension.dim(channel_dim_index).value() - 1)) { @@ -426,10 +431,10 @@ void asymmetric_wquant_per_channel(CircleConst *node, std::vector &min, uint32_t idx_ihw = dimension_channel_last[2] * dimension_channel_last[3] * i + dimension_channel_last[3] * h + w; uint32_t indices_hessain[2] = {idx_quant_column, idx_ihw}; + auto _h_offset = cal_offset_2d(dimension_hessian, indices_hessain); node->at(cal_offset(dimension, indices_channel_first)) -= - error[cal_offset(dimension, indices_error)] * - hessian[cal_offset_2d(dimension_hessian, indices_hessain)]; + error[cal_offset(dimension, indices_error)] * hessian[_h_offset]; } } } @@ -439,8 +444,8 @@ void asymmetric_wquant_per_channel(CircleConst *node, std::vector &min, iterate_per_channel_with_order(node, quantize, true); } - node->dtype(loco::DataType::U8); // change the type of tensor - node->size(size); // resize tensor + node->dtype(loco::DataType::U8); // Change the type of tensor + node->size(size); // Resize tensor for (uint32_t i = 0; i < size; ++i) { node->at(i) = std::min(kMaxScale, std::max(kMinScale, quantized_values[i])); @@ -472,27 +477,28 @@ void asymmetric_wdequant_per_channel(CircleConst *node, std::vector &scal } /** - * @brief QuantizeDequantizeWeights quantizes and dequantizes tensors for weights - * @details Find min/max values on the fly, quantize the model, and dequantize the model + * @brief QuantizeWeightsWithGPTQ quantizes and dequantizes tensors for weights uisng GPTQ algorithm + * @details Compensate for the quantization error and update weights using Hessian matrix + * */ -struct QuantizeWeightsWithGPTQ final : public luci::CircleNodeMutableVisitor +class QuantizeWeightsWithGPTQ final : public luci::CircleNodeMutableVisitor { +public: QuantizeWeightsWithGPTQ( loco::DataType input, loco::DataType output, QuantizationGranularity granularity, std::unordered_map> *hessian_map) - : input_type(input), output_type(output), granularity(granularity), hessian_map(hessian_map) + : input_type(input), output_type(output), granularity(granularity), _hessian_map(hessian_map) { } +private: loco::DataType input_type; loco::DataType output_type; QuantizationGranularity granularity; - std::unordered_map> *hessian_map; + std::unordered_map> *_hessian_map; -private: void fake_quantize_cwq(luci::CircleConst *weights, std::vector &hessian) const { - // assert(output_type == loco::DataType::U8); // FIX_CALLER_UNLESS if (output_type != loco::DataType::U8) { throw std::runtime_error("GPTQ quantization supports u8"); @@ -565,7 +571,7 @@ struct QuantizeWeightsWithGPTQ final : public luci::CircleNodeMutableVisitorfilter(new_weights); - auto hessian = (*hessian_map)[node]; + auto hessian = (*_hessian_map)[node]; fake_quantize(new_weights, hessian); } @@ -615,7 +621,7 @@ struct QuantizeWeightsWithGPTQ final : public luci::CircleNodeMutableVisitorweights(new_weights); - auto hessian = (*hessian_map)[node]; + auto hessian = (*_hessian_map)[node]; fake_quantize(new_weights, hessian); } diff --git a/compiler/luci/pass/src/QuantizeWeightsWithGPTQPass.test.cpp b/compiler/luci/pass/src/QuantizeWeightsWithGPTQPass.test.cpp index 367433115e3..8c00e46bcab 100644 --- a/compiler/luci/pass/src/QuantizeWeightsWithGPTQPass.test.cpp +++ b/compiler/luci/pass/src/QuantizeWeightsWithGPTQPass.test.cpp @@ -5,6 +5,7 @@ namespace { + struct QuantizeWeightsWithGPTQPassTest : public ::testing::Test { /**