Merge conflicts

docker/OpenCV/Dockerfile

@@ -0,0 +1,112 @@
+FROM ubuntu:20.04
+USER root
+WORKDIR /
+
+# Setup timezone
+ENV TZ=Europe/Moscow
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+
+# Add user
+RUN useradd -ms /bin/bash openvino && \
+    chown openvino -R /home/openvino
+
+# Install dependencies
+ARG DEPENDENCIES="apt-utils \
+                  autoconf \
+                  sudo \
+                  vim \
+                  automake \
+                  build-essential \
+                  cmake \
+                  cpio \
+                  curl \
+                  dialog \
+                  gnupg2 \
+                  libdrm2 \
+                  libglib2.0-0 \
+                  lsb-release \
+                  libgtk-3-0 \
+                  libtool \
+                  python3-pip \
+                  python3-setuptools \
+                  python3-dev \
+                  python3-venv \
+                  pciutils \
+                  libpython3.8 \
+                  udev \
+                  unzip \
+                  wget \
+                  git \
+                  ninja-build"
+RUN apt-get update && \
+    apt-get install -y -qq --no-install-recommends ${DEPENDENCIES} && \
+    rm -rf /var/lib/apt/lists/*
+
+# Install OpenVINO
+ENV INSTALL_DIR /opt/intel/openvino_2022
+ARG OV_SHORT_VERSION=2022.3
+ARG OV_LONG_VERSION="${OV_SHORT_VERSION}.0.9052.9752fafe8eb_x86_64"
+RUN sudo mkdir /opt/intel && \
+    wget https://storage.openvinotoolkit.org/repositories/openvino/packages/${OV_SHORT_VERSION}/linux/l_openvino_toolkit_ubuntu20_${OV_LONG_VERSION}.tgz \
+        -O /tmp/openvino.tgz --no-check-certificate --quiet && \
+    tar -zxvf /tmp/openvino.tgz -C /opt/intel/ && \
+    sudo ln -s /opt/intel/l_openvino_toolkit_ubuntu20_${OV_LONG_VERSION} $INSTALL_DIR && \
+    rm -rf /tmp/*
+
+# Install OpenVINO dependencies
+WORKDIR $INSTALL_DIR/install_dependencies
+RUN ls -la $INSTALL_DIR/install_dependencies && echo y | ./install_openvino_dependencies.sh
+RUN cat /root/.bashrc > tmp && echo 'source $INSTALL_DIR/setupvars.sh' > /root/.bashrc
+
+# Configure for GPU, MYRIAD
+RUN echo y | ./install_NEO_OCL_driver.sh
+RUN usermod -a -G video,users root
+
+# Install openvino-dev
+WORKDIR /tmp/
+RUN pip3 install --upgrade pip
+RUN pip3 install openvino-dev==${OV_SHORT_VERSION}
+
+# Install OpenCV
+ARG OCV_VERSION=4.7.0
+RUN pip uninstall opencv-python -y
+RUN git clone --recurse-submodules https://github.com/opencv/opencv.git  --depth 1 --branch ${OCV_VERSION} --single-branch
+ENV OpenCV_BUILD_DIR=/root/build-opencv
+RUN mkdir $OpenCV_BUILD_DIR
+WORKDIR $OpenCV_BUILD_DIR
+RUN /bin/bash -c 'source $INSTALL_DIR/setupvars.sh && \
+    cmake -G Ninja \
+        -D CMAKE_INSTALL_PREFIX=install \
+        -D CMAKE_BUILD_TYPE=Release \
+        -D BUILD_EXAMPLES=OFF \
+        -D BUILD_TESTS=OFF \
+        -D BUILD_DOCS=OFF \
+        -D WITH_OPENVINO=ON \
+        -D OPENCV_LIB_INSTALL_PATH=lib \
+        -D OPENCV_CONFIG_INSTALL_PATH=cmake \
+        -D PYTHON3_PACKAGES_PATH=install/python/python3 \
+        /tmp/opencv/ && ninja && cmake --install .' && \
+    rm -rf /tmp/*
+ENV OpenCV_INSTALL_DIR="$OpenCV_BUILD_DIR/install"
+ENV OpenCV_DIR="$OpenCV_INSTALL_DIR/cmake"
+ENV LD_LIBRARY_PATH="$OpenCV_INSTALL_DIR/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
+ENV PYTHONPATH="$OpenCV_INSTALL_DIR/python/python3/cv2/python-3.8${PYTHONPATH:+:$PYTHONPATH}"
+
+#accuracy-check
+ARG OMZ_VERSION="${OV_SHORT_VERSION}.0"
+WORKDIR /tmp/
+RUN git clone --recursive https://github.com/openvinotoolkit/open_model_zoo.git --branch ${OMZ_VERSION} --single-branch --depth 1
+WORKDIR /tmp/open_model_zoo/tools/accuracy_checker
+RUN wget https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz
+RUN tar xvf cifar-10-python.tar.gz -C sample
+RUN /bin/bash -c 'source $INSTALL_DIR/setupvars.sh && \
+    accuracy_check -c sample/sample_config.yml -m data/test_models -s sample'
+
+# Download DLI source code
+WORKDIR /tmp/
+RUN git clone https://github.com/itlab-vision/dl-benchmark.git --depth 1 && \
+    pip3 install docker PyYAML
+
+# Download dataset if repository with dataset is set
+ARG DATASET_DOWNLOAD_LINK
+RUN if [ -z "$DATASET_DOWNLOAD_LINK" ] ; then echo Argument DATASET_DOWNLOAD_LINK not provided ; else git clone $DATASET_DOWNLOAD_LINK ; fi

src/benchmark/README.md

@@ -206,7 +206,10 @@ python3 inference_benchmark.py \
 ### Использование
 
 1. В конфигурационном файле (секция `FrameworkDependent`)
-   укажите `Mode`: `ovbenchmark_cpp_latency` или `ovbenchmark_cpp_throughput`.
+   укажите `Mode`: `sync` или `async`;
+           `CodeSource`: `ovbenchmark`;
+           `Runtime`: `cpp`;
+           `Hint`: `none`, `latency` или `throughput`.
 
 1. Найдите исполняемый файл `benchmark_app` по адресу, приведенному ниже.
 
@@ -236,7 +239,10 @@ pip install openvino_dev[mxnet,caffe,caffe2,onnx,pytorch,tensorflow2]==<your ver
 ### Использование
 
 1. В конфигурационном файле (секция `FrameworkDependent`)
-   укажите `Mode`: `ovbenchmark_python_latency` или `ovbenchmark_python_throughput`.
+   укажите `Mode`: `sync` или `async`;
+           `CodeSource`: `ovbenchmark`;
+           `Runtime`: `python`;
+           `Hint`: `none`, `latency` или `throughput`.
 
 1. Запустите скрипт `inference_benchmark.py`.
 

src/benchmark/config_processor.py

@@ -12,6 +12,7 @@ def process_config(config, log):
     test_list = []
 
     tests = test_parser.get_tests_list(config)
+    status = 0
     for idx, curr_test in enumerate(tests):
         try:
             model = test_parser.parse_model(curr_test)
@@ -24,7 +25,8 @@ def process_config(config, log):
                                                                                indep_parameters, dep_parameters))
         except ValueError as valerr:
             log.warning(f'Test {idx + 1} not added to test list: {valerr}')
-    return test_list
+            status = 1
+    return test_list, status
 
 
 class TestConfigParser:

src/benchmark/frameworks/config_parser/dependent_parameters_parser_cpp.py

@@ -6,18 +6,23 @@ class CppParametersParser(DependentParametersParser):
     def parse_parameters(self, curr_test):
         dep_parameters_tag = curr_test.getElementsByTagName('FrameworkDependent')[0]
 
-        _shape = dep_parameters_tag.getElementsByTagName('InputShape')[0].firstChild
+        _backend = None
+        if dep_parameters_tag.getElementsByTagName('Backend'):
+            _backend = dep_parameters_tag.getElementsByTagName('Backend')[0].firstChild
+
+        _input_shape = dep_parameters_tag.getElementsByTagName('InputShape')[0].firstChild
         _layout = dep_parameters_tag.getElementsByTagName('Layout')[0].firstChild
         _mean = dep_parameters_tag.getElementsByTagName('Mean')[0].firstChild
-        _scale = dep_parameters_tag.getElementsByTagName('InputScale')[0].firstChild
+        _input_scale = dep_parameters_tag.getElementsByTagName('InputScale')[0].firstChild
         _thread_count = dep_parameters_tag.getElementsByTagName('ThreadCount')[0].firstChild
         _inference_requests_count = dep_parameters_tag.getElementsByTagName('InferenceRequestsCount')[0].firstChild
 
         return CppParameters(
-            shape=_shape.data if _shape else None,
+            backend=_backend.data if _backend else None,
+            input_shape=_input_shape.data if _input_shape else None,
             layout=_layout.data if _layout else None,
             mean=_mean.data if _mean else None,
-            scale=_scale.data if _scale else None,
+            input_scale=_input_scale.data if _input_scale else None,
             thread_count=_thread_count.data if _thread_count else None,
             inference_requests_count=_inference_requests_count.data if _inference_requests_count else None,
         )

src/benchmark/frameworks/config_parser/framework_independent_parameters.py

@@ -8,7 +8,7 @@ def __init__(self, inference_framework, batch_size, device, iterarion_count, tes
         self.device = None
         self.iteration = None
         self.test_time_limit = None
-        if self._parameter_not_is_none(inference_framework):
+        if self._parameter_is_not_none(inference_framework):
             self.inference_framework = inference_framework
         else:
             raise ValueError('Inference framework is required parameter.')
@@ -17,25 +17,18 @@ def __init__(self, inference_framework, batch_size, device, iterarion_count, tes
                 self.batch_size = int(batch_size)
             else:
                 raise ValueError('Batch size can only take values: integer greater than zero.')
-        if self._device_is_correct(device):
+        if self._parameter_is_not_none(device):
             self.device = device.upper()
         else:
-            raise ValueError('Device is required parameter. '
-                             'Supported values: CPU, GPU, FPGA, MYRIAD.')
-        if self._parameter_not_is_none(iterarion_count) and self._int_value_is_correct(iterarion_count):
+            raise ValueError('Device is required parameter.')
+        if self._parameter_is_not_none(iterarion_count) and self._int_value_is_correct(iterarion_count):
             self.iteration = int(iterarion_count)
         else:
             raise ValueError('Iteration count is required parameter. '
                              'Iteration count can only take values: integer greater than zero.')
-        if self._parameter_not_is_none(test_time_limit) and self._float_value_is_correct(test_time_limit):
+
+        if self._parameter_is_not_none(test_time_limit) and self._float_value_is_correct(test_time_limit):
             self.test_time_limit = float(test_time_limit)
         else:
             raise ValueError('Test time limit is required parameter. '
                              'Test time limit can only `take values: float greater than zero.')
-
-    @staticmethod
-    def _device_is_correct(device):
-        const_correct_devices = ['CPU', 'GPU', 'MYRIAD', 'FPGA']
-        if device.upper() in const_correct_devices:
-            return True
-        return False

src/benchmark/frameworks/config_parser/framework_parameters_cpp.py

@@ -2,23 +2,27 @@
 
 
 class CppParameters(FrameworkParameters):
-    def __init__(self, shape, layout, mean, scale, thread_count, inference_requests_count):
-        self.shape = None
+    def __init__(self, backend, input_shape, layout, mean, input_scale, thread_count, inference_requests_count):
+        self.backend = None
+        self.input_shape = None
         self.layout = None
         self.mean = None
-        self.scale = None
+        self.input_scale = None
         self.thread_count = None
         self.inference_requests_count = None
 
-        if shape is not None:
-            self.shape = shape.strip()
-        if layout is not None:
+        if self._parameter_is_not_none(backend):
+            self.backend = backend.strip()
+        if self._parameter_is_not_none(input_shape):
+            self.input_shape = input_shape.strip()
+        if self._parameter_is_not_none(layout):
             self.layout = layout.strip()
-        if mean is not None:
+        if self._parameter_is_not_none(mean):
             self.mean = mean.strip()
-        if scale is not None:
-            self.scale = scale.strip()
-        if thread_count is not None and self._int_value_is_correct(thread_count):
+        if self._parameter_is_not_none(input_scale):
+            self.input_scale = input_scale.strip()
+        if self._parameter_is_not_none(thread_count) and self._int_value_is_correct(thread_count):
             self.thread_count = thread_count
-        if inference_requests_count is not None and self._int_value_is_correct(inference_requests_count):
+        if self._parameter_is_not_none(inference_requests_count) and self._int_value_is_correct(
+                inference_requests_count):
             self.inference_requests_count = inference_requests_count

src/benchmark/frameworks/config_parser/framework_parameters_parser.py

@@ -1,6 +1,6 @@
 class FrameworkParameters:
     @staticmethod
-    def _parameter_not_is_none(parameter):
+    def _parameter_is_not_none(parameter):
         return parameter is not None
 
     @staticmethod

src/benchmark/frameworks/config_parser/test_reporter.py

@@ -1,4 +1,7 @@
 import abc
+import logging as log
+
+from collections import OrderedDict
 
 
 class Test(metaclass=abc.ABCMeta):
@@ -7,10 +10,25 @@ def __init__(self, model, dataset, indep_parameters, dep_parameters):
         self.dataset = dataset
         self.indep_parameters = indep_parameters
         self.dep_parameters = dep_parameters
+        self._log = log
+
+    def get_report(self, **kwargs):
+        parameters = self.prepare_framework_params()
+        other_param = self._get_optional_parameters_string(parameters)
+
+        report_res = {
+            'task': self.model.task,
+            'model': self.model.name,
+            'dataset': self.dataset.name,
+            'source_framework': self.model.source_framework,
+            'inference_framework': self.indep_parameters.inference_framework,
+            'precision': self.model.precision,
+            'batch_size': self.indep_parameters.batch_size,
+            'mode': 'Sync',
+            'framework_params': other_param,
+        }
 
-    @abc.abstractmethod
-    def get_report(self):
-        pass
+        return report_res
 
     @staticmethod
     def _get_optional_parameters_string(parameters_dict):
@@ -19,3 +37,47 @@ def _get_optional_parameters_string(parameters_dict):
             if parameters_dict[key] is not None:
                 parameter_strings.append(f'{key}: {parameters_dict[key]}')
         return ', '.join(parameter_strings)
+
+    def prepare_framework_params(self):
+        parameters = OrderedDict()
+
+        parameters.update({'Device': self.indep_parameters.device})
+        parameters.update({'Iteration count': self.indep_parameters.iteration})
+
+        match_parameter_description = {}
+
+        match_parameter_description['code_source'] = 'Code Source'
+        match_parameter_description['runtime'] = 'Runtime'
+        match_parameter_description['hint'] = 'Hint'
+        match_parameter_description['frontend'] = 'Frontend'
+        match_parameter_description['backend'] = 'Backend'
+        match_parameter_description['delegate'] = 'Delegate'
+        match_parameter_description['delegate_options'] = 'Delegate options'
+
+        match_parameter_description['async_request'] = 'Async request count'
+        match_parameter_description['nthreads'] = 'Thread count'
+        match_parameter_description['thread_count'] = 'Thread count'
+        match_parameter_description['nstreams'] = 'Stream count'
+        match_parameter_description['kmp_affinity'] = 'KMP_AFFINITY'
+        match_parameter_description['num_inter_threads'] = 'Inter threads'
+        match_parameter_description['num_intra_threads'] = 'Intra threads'
+
+        match_parameter_description['mean'] = 'Mean'
+        match_parameter_description['input_scale'] = 'Scale'
+        match_parameter_description['layout'] = 'Layout'
+        # duplicate because pytorch launcher does not match common template. To be fixed
+        match_parameter_description['output_name'] = 'Output name'
+        match_parameter_description['hybridize'] = 'Hybridization flag'
+        match_parameter_description['normalize'] = 'Normalization flag'
+        match_parameter_description['std'] = 'Standard deviation'
+        match_parameter_description['channel_swap'] = 'Channel swap'
+        match_parameter_description['swapRB'] = 'Channel swap'
+        match_parameter_description['crop'] = 'Crop'
+        match_parameter_description['model_type'] = 'Model type'
+        match_parameter_description['inference_mode'] = 'Inference mode'
+
+        for parameter, description in match_parameter_description.items():
+            if hasattr(self.dep_parameters, parameter) and getattr(self.dep_parameters, parameter) is not None:
+                parameters.update({description: getattr(self.dep_parameters, parameter)})
+
+        return parameters

src/benchmark/frameworks/config_parser/test_reporter_cpp.py

@@ -0,0 +1,39 @@
+from .test_reporter import Test
+
+
+class CppTest(Test):
+    def get_report(self, process):
+        tensors_num = self.dep_parameters.inference_requests_count
+        json_report_content = process.get_json_report_content()
+        if process.get_status() == 0 and not tensors_num:
+            self._log.info('InferenceRequestsCount is not set in XML config, '
+                           'will try to extract it from the launcher JSON report or console output')
+            tensors_num = json_report_content['configurations_setup']['tensors_num']
+
+        batch_size = self.indep_parameters.batch_size
+        if process.get_status() == 0 and not batch_size:
+            self._log.info('BatchSize is not set in XML config, '
+                           'will try to extract it from the launcher JSON report')
+            batch_size = json_report_content['configurations_setup']['batch_size']
+
+        actual_iterations = json_report_content['execution_results']['iterations_num']
+
+        parameters = self.prepare_framework_params()
+        parameters['Infer request count'] = 1
+        parameters['Number of tensors'] = tensors_num
+        parameters['Iteration count'] = actual_iterations
+        optional_parameters_string = self._get_optional_parameters_string(parameters)
+
+        report_res = {
+            'task': self.model.task,
+            'model': self.model.name,
+            'dataset': self.dataset.name,
+            'source_framework': self.model.source_framework,
+            'inference_framework': self.indep_parameters.inference_framework,
+            'precision': self.model.precision,
+            'batch_size': batch_size,
+            'mode': 'Sync',
+            'framework_params': optional_parameters_string,
+        }
+
+        return report_res