diff --git a/compiler/q-implant-qparam-test/CMakeLists.txt b/compiler/q-implant-qparam-test/CMakeLists.txt new file mode 100644 index 00000000000..e007bbddd19 --- /dev/null +++ b/compiler/q-implant-qparam-test/CMakeLists.txt @@ -0,0 +1,23 @@ +if(NOT ENABLE_TEST) + return() +endif(NOT ENABLE_TEST) + +unset(Q_IMPLANT_TESTS) + +macro(addeval NAME) + list(APPEND Q_IMPLANT_TESTS ${NAME}) +endmacro(addeval) + +include("test.lst") + +get_target_property(ARTIFACTS_BIN_PATH testDataGenerator BINARY_DIR) + +add_test(NAME q-implant-qparam-test + COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/q_implant_qparam_test.sh" + "${CMAKE_CURRENT_BINARY_DIR}" + "${ARTIFACTS_BIN_PATH}" + "${NNCC_OVERLAY_DIR}/venv_2_12_1" + "$" + "$" + ${Q_IMPLANT_TESTS} +) diff --git a/compiler/q-implant-qparam-test/README.md b/compiler/q-implant-qparam-test/README.md new file mode 100644 index 00000000000..05f09d15dcb --- /dev/null +++ b/compiler/q-implant-qparam-test/README.md @@ -0,0 +1,61 @@ +# q-implant-qparam-test + +`q-implant-qparam-test` validates that q-implant supports common used operators. + +The test proceeds as follows + +Step 1: Generate qparam file(.json) and numpy array(.npy) through the operator python file. +``` +operator file -> qparam file, numpy array +``` + +Step 2: Generate output.circle to use q-implant +``` +"circle file" + "qparam.json" -> q-implant -> "quant circle file" +``` + +Step 3: Dump output.circle to output.h5. +``` +"output.circle" -> circle-tensordump -> "output.h5" +``` + +Step 4: And compare tensor values of h5 file with numpy arrays due to validate q-implant. + +how to make qparam file + +step 1: Choose the recipe in 'res/TensorFlowLiteRecipes' and get name of recipe. + +step 2: Create folder in qparam that name is recipe name + +step 3: Create `__init__.py` follow this sample. + +``` python +from test_utils import TestCase +from test_utils import gen_random_tensor + + +class recipe_name_000_Q8(TestCase): + def __init__(self): + self.name = _name_ + + def generate(self) -> dict: + json_content = dict() + + # Generate operand_name + json_content['operand_name'] = gen_random_tensor( + "uint8", # dtype_str + (1), # scale_shape + (1), # zerop_shape + 0, # quantized_dimension + (3, 3, 3, 3)) # value_shape ( such as weight, bias ) + + ... + + return json_content + + +_name_ = 'recipe_name_000_Q8' + +_test_case_ = recipe_name_000_Q8() + +``` diff --git a/compiler/q-implant-qparam-test/q_implant_qparam_test.py b/compiler/q-implant-qparam-test/q_implant_qparam_test.py new file mode 100644 index 00000000000..fad271b41a2 --- /dev/null +++ b/compiler/q-implant-qparam-test/q_implant_qparam_test.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2023 Samsung Electronics Co., Ltd. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import subprocess +import os +import importlib + +from test_utils import TestRunner +from q_implant_validator import validate + +parser = argparse.ArgumentParser() +parser.add_argument('--input_dir', type=str, required=True) +parser.add_argument('--output_dir', type=str, required=True) +parser.add_argument('--driver', type=str, required=True) +parser.add_argument('--dump', type=str, required=True) +parser.add_argument('--model', type=str, required=True) +args = parser.parse_args() + +input_dir = args.input_dir +output_dir = args.output_dir +driver = args.driver +dump = args.dump +model = args.model + +module = importlib.import_module('qparam.' + model) + +input_circle = input_dir + '.circle' +output_circle = output_dir + f'/{module._name_}/output.circle' +qparam_dir = output_dir + f'/{module._name_}/qparam.json' +h5_path = output_dir + f'/{module._name_}/output.h5' + +if not os.path.exists(input_circle): + print('fail to load input circle') + quit(255) + +# if the previous test dummy file exist, remove it. +if os.path.exists(output_circle): + os.remove(output_circle) + +if os.path.exists(h5_path): + os.remove(h5_path) + +# generate qparam.json and numpys +test_runner = TestRunner(output_dir) + +test_runner.register(module._test_case_) + +test_runner.run() + +if not os.path.exists(qparam_dir): + print('qparam generate fail') + quit(255) + +# run q-implant +process = subprocess.run([driver, input_circle, qparam_dir, output_circle], check=True) + +try: + process.check_returncode() +except: + print('q-implant run failed') + quit(255) + +if not os.path.exists(output_circle): + print('output circle generate fail') + quit(255) + +# dump circle to h5 +process = subprocess.run([dump, '--tensors_to_hdf5', h5_path, output_circle], check=True) + +try: + process.check_returncode() +except: + print('circle-tensordump run failed') + quit(255) + +if not os.path.exists(h5_path): + print('h5 dump failed') + quit(255) + +if not validate(h5_path, output_dir + f'/{module._name_}', qparam_dir): + quit(255) + +quit(0) diff --git a/compiler/q-implant-qparam-test/q_implant_qparam_test.sh b/compiler/q-implant-qparam-test/q_implant_qparam_test.sh new file mode 100755 index 00000000000..fff4e60e318 --- /dev/null +++ b/compiler/q-implant-qparam-test/q_implant_qparam_test.sh @@ -0,0 +1,58 @@ +#!/bin/bash + +VERIFY_SOURCE_PATH="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +VERIFY_SCRIPT_PATH="${VERIFY_SOURCE_PATH}/q_implant_qparam_test.py" +BINDIR="$1"; shift +WORKDIR="$1"; shift +VIRTUALENV="$1"; shift +INTERPRETER_DRIVER_PATH="$1"; shift +H5_DUMP_PATH="$1"; shift + +TESTED=() +PASSED=() +FAILED=() + +for TESTCASE in "$@"; do + TESTED+=("${TESTCASE}") + + TESTCASE_FILE="${WORKDIR}/${TESTCASE}" + TEST_RESULT_FILE="${BINDIR}/${TESTCASE}" + + PASSED_TAG="${TEST_RESULT_FILE}.passed" + rm -f "${PASSED_TAG}" + + cat > "${TEST_RESULT_FILE}.log" <( + exec 2>&1 + set -ex + + source "${VIRTUALENV}/bin/activate" + "${VIRTUALENV}/bin/python" "${VERIFY_SCRIPT_PATH}" \ + --model "${TESTCASE}" \ + --driver "${INTERPRETER_DRIVER_PATH}" \ + --dump "${H5_DUMP_PATH}" \ + --output_dir "${BINDIR}" \ + --input_dir "${TESTCASE_FILE}" + + if [[ $? -eq 0 ]]; then + touch "${PASSED_TAG}" + fi + ) + + if [[ -f "${PASSED_TAG}" ]]; then + PASSED+=("${TESTCASE}") + else + FAILED+=("${TESTCASE}") + fi +done + +if [[ ${#TESTED[@]} -ne ${#PASSED[@]} ]]; then + echo "FAILED" + for TEST in "${FAILED[@]}" + do + echo "- ${TEST}" + done + exit 255 +fi + +echo "PASSED" +exit 0 diff --git a/compiler/q-implant-qparam-test/q_implant_validator.py b/compiler/q-implant-qparam-test/q_implant_validator.py new file mode 100644 index 00000000000..1f3d8b608c9 --- /dev/null +++ b/compiler/q-implant-qparam-test/q_implant_validator.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2023 Samsung Electronics Co., Ltd. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import h5py as h5 +import numpy as np +import json + + +def validate(h5_path, qparam_dir, qparam_json): + valid = True + with open(qparam_json, "r") as qparams: + json_load = json.load(qparams) + with h5.File(h5_path, "r") as model: + for node_name in model.keys(): + # not quantized node exists (reshape, pad...) + if not json_load.get(node_name): + continue + + for tensor_name in json_load[node_name]: + np_path = f"{qparam_dir}/{json_load[node_name][tensor_name]}" + if tensor_name == "value": + expected_weights = np.load(np_path) + h5_weights = model[node_name]["weights"][:] + if np.allclose( + h5_weights, expected_weights, rtol=1.e-5, + atol=1.e-5) == False: + print("Implanted weights of " + node_name + "." + tensor_name + + " (" + str(h5_weights) + + ") do not match with expected value (" + + str(expected_weights) + ").") + valid = False + + if tensor_name == "scale": + expected_scale = np.load(np_path) + h5_scale = model[node_name]["scale"][:] + if np.allclose( + h5_scale, expected_scale, rtol=1.e-5, atol=1.e-5) == False: + print("Implanted scale of " + node_name + "." + tensor_name + + " (" + str(h5_scale) + + ") do not match with expected value (" + + str(expected_scale) + ").") + valid = False + + if tensor_name == "zerop": + expected_zerop = np.load(np_path) + input_zerop = model[node_name]["zero_point"][:] + if np.allclose(input_zerop, expected_zerop, rtol=0, atol=1) == False: + print("Implanted zero point of " + tensor_name + " (" + + str(input_zerop) + ") do not match with expected value (" + + str(expected_zerop) + ").") + valid = False + + return valid diff --git a/compiler/q-implant-qparam-test/requires.cmake b/compiler/q-implant-qparam-test/requires.cmake new file mode 100644 index 00000000000..2e4349c0363 --- /dev/null +++ b/compiler/q-implant-qparam-test/requires.cmake @@ -0,0 +1,2 @@ +require("common-artifacts") +require("q-implant") diff --git a/compiler/q-implant-qparam-test/test.lst b/compiler/q-implant-qparam-test/test.lst new file mode 100644 index 00000000000..e69de29bb2d diff --git a/compiler/q-implant-qparam-test/test_utils.py b/compiler/q-implant-qparam-test/test_utils.py new file mode 100644 index 00000000000..5fabd220831 --- /dev/null +++ b/compiler/q-implant-qparam-test/test_utils.py @@ -0,0 +1,92 @@ +import json +import typing +import numpy as np +import os + + +def _dump_npy_included_json(output_dir: str, json_content: dict): + """ + Dump json and npy files to output_dir + """ + # Create output_dir if not exists + if not os.path.exists(output_dir): + os.makedirs(output_dir) + + # file name for npy data (ex: 0.npy, 1.npy, ...) + _index = 0 + _index_to_value = dict() + + # Replace npy to the path to the npy file + for tensor_name, qparam in json_content.items(): + assert type(tensor_name) == str + assert type(qparam) == dict + for field, value in qparam.items(): + if isinstance(value, np.ndarray): + npy_name = str(_index) + '.npy' + + # Save npy file + np.save(os.path.join(output_dir, npy_name), value) + + # Replace to the path to the npy file + json_content[tensor_name][field] = npy_name + + # Save the mapping from index to tensor name + _index_to_value[_index] = tensor_name + "_" + field + _index += 1 + + # Dump json + with open(os.path.join(output_dir, 'qparam.json'), 'w') as f: + json.dump(json_content, f, indent=2) + + +def _str_to_npy_dtype(dtype_str: str): + if dtype_str == "uint8": + return np.uint8 + if dtype_str == "int16": + return np.int16 + if dtype_str == "int32": + return np.int32 + if dtype_str == "int64": + return np.int64 + raise SystemExit("Unsupported npy dtype", dtype_str) + + +def gen_random_tensor(dtype_str: str, + scale_shape: typing.Tuple[int], + zerop_shape: typing.Tuple[int], + quantized_dimension: int, + value_shape: typing.Optional[typing.Tuple[int]] = None) -> dict: + content = dict() + content['dtype'] = dtype_str + content['scale'] = np.random.rand(scale_shape).astype(np.float32) + # Why 256? To ensure the smallest dtype (uint8) range [0, 256) + content['zerop'] = np.random.randint(256, size=zerop_shape, dtype=np.int64) + content['quantized_dimension'] = quantized_dimension + + if value_shape != None: + dtype = _str_to_npy_dtype(dtype_str) + content['value'] = np.random.randint(256, size=value_shape, dtype=dtype) + return content + + +class TestCase: + def __init__(self): + pass + + def generate(self) -> dict: + pass + + +class TestRunner: + def __init__(self, output_dir: str): + self.test_cases = list() + self.output_dir = output_dir + + def register(self, test_case: TestCase): + self.test_cases.append(test_case) + + def run(self): + for test_case in self.test_cases: + print("Generate test case: " + test_case.name) + _dump_npy_included_json(self.output_dir + '/' + test_case.name, + test_case.generate())