[q-implant-qparam-test] Introduce q-implant-qparam-test (#11677)

This introduces q-implant-qparam-test to check qparam is correctly implanted.

ONE-DCO-1.0-Signed-off-by: Seonguk Park [email protected]

compiler/q-implant-qparam-test/CMakeLists.txt

@@ -0,0 +1,23 @@
+if(NOT ENABLE_TEST)
+  return()
+endif(NOT ENABLE_TEST)
+
+unset(Q_IMPLANT_TESTS)
+
+macro(addeval NAME)
+  list(APPEND Q_IMPLANT_TESTS ${NAME})
+endmacro(addeval)
+
+include("test.lst")
+
+get_target_property(ARTIFACTS_BIN_PATH testDataGenerator BINARY_DIR)
+
+add_test(NAME q-implant-qparam-test
+  COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/q_implant_qparam_test.sh"
+          "${CMAKE_CURRENT_BINARY_DIR}"
+          "${ARTIFACTS_BIN_PATH}"
+          "${NNCC_OVERLAY_DIR}/venv_2_12_1"
+          "$<TARGET_FILE:q-implant>"
+          "$<TARGET_FILE:circle-tensordump>"
+          ${Q_IMPLANT_TESTS}
+)

compiler/q-implant-qparam-test/README.md

@@ -0,0 +1,61 @@
+# q-implant-qparam-test
+
+`q-implant-qparam-test` validates that q-implant supports common used operators.
+
+The test proceeds as follows
+
+Step 1: Generate qparam file(.json) and numpy array(.npy) through the operator python file.
+```
+operator file -> qparam file, numpy array
+```
+
+Step 2: Generate output.circle to use q-implant
+```
+"circle file" + "qparam.json" -> q-implant -> "quant circle file"
+```
+
+Step 3: Dump output.circle to output.h5.
+```
+"output.circle" -> circle-tensordump -> "output.h5"
+```
+
+Step 4: And compare tensor values of h5 file with numpy arrays due to validate q-implant.
+
+how to make qparam file
+
+step 1: Choose the recipe in 'res/TensorFlowLiteRecipes' and get name of recipe.
+
+step 2: Create folder in qparam that name is recipe name
+
+step 3: Create `__init__.py` follow this sample.
+
+``` python
+from test_utils import TestCase
+from test_utils import gen_random_tensor
+
+
+class recipe_name_000_Q8(TestCase):
+    def __init__(self):
+        self.name = _name_
+
+    def generate(self) -> dict:
+        json_content = dict()
+
+        # Generate operand_name
+        json_content['operand_name'] = gen_random_tensor(
+            "uint8",  # dtype_str
+            (1),  # scale_shape
+            (1),  # zerop_shape
+            0,  # quantized_dimension
+            (3, 3, 3, 3)) # value_shape ( such as weight, bias )
+
+         ...
+
+        return json_content
+
+
+_name_ = 'recipe_name_000_Q8'
+
+_test_case_ = recipe_name_000_Q8()
+
+```

compiler/q-implant-qparam-test/q_implant_qparam_test.py

@@ -0,0 +1,97 @@
+#!/usr/bin/env python3
+
+# Copyright (c) 2023 Samsung Electronics Co., Ltd. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import subprocess
+import os
+import importlib
+
+from test_utils import TestRunner
+from q_implant_validator import validate
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--input_dir', type=str, required=True)
+parser.add_argument('--output_dir', type=str, required=True)
+parser.add_argument('--driver', type=str, required=True)
+parser.add_argument('--dump', type=str, required=True)
+parser.add_argument('--model', type=str, required=True)
+args = parser.parse_args()
+
+input_dir = args.input_dir
+output_dir = args.output_dir
+driver = args.driver
+dump = args.dump
+model = args.model
+
+module = importlib.import_module('qparam.' + model)
+
+input_circle = input_dir + '.circle'
+output_circle = output_dir + f'/{module._name_}/output.circle'
+qparam_dir = output_dir + f'/{module._name_}/qparam.json'
+h5_path = output_dir + f'/{module._name_}/output.h5'
+
+if not os.path.exists(input_circle):
+    print('fail to load input circle')
+    quit(255)
+
+# if the previous test dummy file exist, remove it.
+if os.path.exists(output_circle):
+    os.remove(output_circle)
+
+if os.path.exists(h5_path):
+    os.remove(h5_path)
+
+# generate qparam.json and numpys
+test_runner = TestRunner(output_dir)
+
+test_runner.register(module._test_case_)
+
+test_runner.run()
+
+if not os.path.exists(qparam_dir):
+    print('qparam generate fail')
+    quit(255)
+
+# run q-implant
+process = subprocess.run([driver, input_circle, qparam_dir, output_circle], check=True)
+
+try:
+    process.check_returncode()
+except:
+    print('q-implant run failed')
+    quit(255)
+
+if not os.path.exists(output_circle):
+    print('output circle generate fail')
+    quit(255)
+
+# dump circle to h5
+process = subprocess.run([dump, '--tensors_to_hdf5', h5_path, output_circle], check=True)
+
+try:
+    process.check_returncode()
+except:
+    print('circle-tensordump run failed')
+    quit(255)
+
+if not os.path.exists(h5_path):
+    print('h5 dump failed')
+    quit(255)
+
+if not validate(h5_path, output_dir + f'/{module._name_}', qparam_dir):
+    quit(255)
+
+quit(0)

compiler/q-implant-qparam-test/q_implant_qparam_test.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+
+VERIFY_SOURCE_PATH="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+VERIFY_SCRIPT_PATH="${VERIFY_SOURCE_PATH}/q_implant_qparam_test.py"
+BINDIR="$1"; shift
+WORKDIR="$1"; shift
+VIRTUALENV="$1"; shift
+INTERPRETER_DRIVER_PATH="$1"; shift
+H5_DUMP_PATH="$1"; shift
+
+TESTED=()
+PASSED=()
+FAILED=()
+
+for TESTCASE in "$@"; do
+  TESTED+=("${TESTCASE}")
+
+  TESTCASE_FILE="${WORKDIR}/${TESTCASE}"
+  TEST_RESULT_FILE="${BINDIR}/${TESTCASE}"
+
+  PASSED_TAG="${TEST_RESULT_FILE}.passed"
+  rm -f "${PASSED_TAG}"
+
+  cat > "${TEST_RESULT_FILE}.log" <(
+    exec 2>&1
+    set -ex
+
+    source "${VIRTUALENV}/bin/activate"
+    "${VIRTUALENV}/bin/python" "${VERIFY_SCRIPT_PATH}" \
+    --model "${TESTCASE}" \
+    --driver "${INTERPRETER_DRIVER_PATH}" \
+    --dump "${H5_DUMP_PATH}" \
+    --output_dir "${BINDIR}" \
+    --input_dir "${TESTCASE_FILE}"
+
+    if [[ $? -eq 0 ]]; then
+      touch "${PASSED_TAG}"
+    fi
+  )
+
+  if [[ -f "${PASSED_TAG}" ]]; then
+    PASSED+=("${TESTCASE}")
+  else
+    FAILED+=("${TESTCASE}")
+  fi
+done
+
+if [[ ${#TESTED[@]} -ne ${#PASSED[@]} ]]; then
+  echo "FAILED"
+  for TEST in "${FAILED[@]}"
+  do
+    echo "- ${TEST}"
+  done
+  exit 255
+fi
+
+echo "PASSED"
+exit 0

compiler/q-implant-qparam-test/q_implant_validator.py

@@ -0,0 +1,66 @@
+#!/usr/bin/env python3
+
+# Copyright (c) 2023 Samsung Electronics Co., Ltd. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import h5py as h5
+import numpy as np
+import json
+
+
+def validate(h5_path, qparam_dir, qparam_json):
+    valid = True
+    with open(qparam_json, "r") as qparams:
+        json_load = json.load(qparams)
+    with h5.File(h5_path, "r") as model:
+        for node_name in model.keys():
+            # not quantized node exists (reshape, pad...)
+            if not json_load.get(node_name):
+                continue
+
+            for tensor_name in json_load[node_name]:
+                np_path = f"{qparam_dir}/{json_load[node_name][tensor_name]}"
+                if tensor_name == "value":
+                    expected_weights = np.load(np_path)
+                    h5_weights = model[node_name]["weights"][:]
+                    if np.allclose(
+                            h5_weights, expected_weights, rtol=1.e-5,
+                            atol=1.e-5) == False:
+                        print("Implanted weights of " + node_name + "." + tensor_name +
+                              " (" + str(h5_weights) +
+                              ") do not match with expected value (" +
+                              str(expected_weights) + ").")
+                        valid = False
+
+                if tensor_name == "scale":
+                    expected_scale = np.load(np_path)
+                    h5_scale = model[node_name]["scale"][:]
+                    if np.allclose(
+                            h5_scale, expected_scale, rtol=1.e-5, atol=1.e-5) == False:
+                        print("Implanted scale of " + node_name + "." + tensor_name +
+                              " (" + str(h5_scale) +
+                              ") do not match with expected value (" +
+                              str(expected_scale) + ").")
+                        valid = False
+
+                if tensor_name == "zerop":
+                    expected_zerop = np.load(np_path)
+                    input_zerop = model[node_name]["zero_point"][:]
+                    if np.allclose(input_zerop, expected_zerop, rtol=0, atol=1) == False:
+                        print("Implanted zero point of " + tensor_name + " (" +
+                              str(input_zerop) + ") do not match with expected value (" +
+                              str(expected_zerop) + ").")
+                        valid = False
+
+    return valid

compiler/q-implant-qparam-test/requires.cmake

@@ -0,0 +1,2 @@
+require("common-artifacts")
+require("q-implant")