refactoring of configuration parameters parsing and *Test method dupl…

…ication remove (#372)

Co-authored-by: Ivan Vikhrev <[email protected]>
Co-authored-by: Zoya Maslova <[email protected]>
Co-authored-by: Ilya Naumov <[email protected]>
Co-authored-by: Zoya Maslova <[email protected]>
Co-authored-by: just_sparta <[email protected]>
Co-authored-by: Alexander Sesorov <[email protected]>
Co-authored-by: Aleksandr Sesorov <[email protected]>
Co-authored-by: Natalya Berezina <[email protected]>

src/benchmark/README.md

@@ -206,7 +206,10 @@ python3 inference_benchmark.py \
 ### Использование
 
 1. В конфигурационном файле (секция `FrameworkDependent`)
-   укажите `Mode`: `ovbenchmark_cpp_latency` или `ovbenchmark_cpp_throughput`.
+   укажите `Mode`: `sync` или `async`;
+           `CodeSource`: `ovbenchmark`;
+           `Runtime`: `cpp`;
+           `Hint`: `none`, `latency` или `throughput`.
 
 1. Найдите исполняемый файл `benchmark_app` по адресу, приведенному ниже.
 
@@ -236,7 +239,10 @@ pip install openvino_dev[mxnet,caffe,caffe2,onnx,pytorch,tensorflow2]==<your ver
 ### Использование
 
 1. В конфигурационном файле (секция `FrameworkDependent`)
-   укажите `Mode`: `ovbenchmark_python_latency` или `ovbenchmark_python_throughput`.
+   укажите `Mode`: `sync` или `async`;
+           `CodeSource`: `ovbenchmark`;
+           `Runtime`: `python`;
+           `Hint`: `none`, `latency` или `throughput`.
 
 1. Запустите скрипт `inference_benchmark.py`.
 

src/benchmark/config_processor.py

@@ -12,6 +12,7 @@ def process_config(config, log):
     test_list = []
 
     tests = test_parser.get_tests_list(config)
+    status = 0
     for idx, curr_test in enumerate(tests):
         try:
             model = test_parser.parse_model(curr_test)
@@ -24,7 +25,8 @@ def process_config(config, log):
                                                                                indep_parameters, dep_parameters))
         except ValueError as valerr:
             log.warning(f'Test {idx + 1} not added to test list: {valerr}')
-    return test_list
+            status = 1
+    return test_list, status
 
 
 class TestConfigParser:

src/benchmark/frameworks/config_parser/dependent_parameters_parser_cpp.py

@@ -6,18 +6,23 @@ class CppParametersParser(DependentParametersParser):
     def parse_parameters(self, curr_test):
         dep_parameters_tag = curr_test.getElementsByTagName('FrameworkDependent')[0]
 
-        _shape = dep_parameters_tag.getElementsByTagName('InputShape')[0].firstChild
+        _backend = None
+        if dep_parameters_tag.getElementsByTagName('Backend'):
+            _backend = dep_parameters_tag.getElementsByTagName('Backend')[0].firstChild
+
+        _input_shape = dep_parameters_tag.getElementsByTagName('InputShape')[0].firstChild
         _layout = dep_parameters_tag.getElementsByTagName('Layout')[0].firstChild
         _mean = dep_parameters_tag.getElementsByTagName('Mean')[0].firstChild
-        _scale = dep_parameters_tag.getElementsByTagName('InputScale')[0].firstChild
+        _input_scale = dep_parameters_tag.getElementsByTagName('InputScale')[0].firstChild
         _thread_count = dep_parameters_tag.getElementsByTagName('ThreadCount')[0].firstChild
         _inference_requests_count = dep_parameters_tag.getElementsByTagName('InferenceRequestsCount')[0].firstChild
 
         return CppParameters(
-            shape=_shape.data if _shape else None,
+            backend=_backend.data if _backend else None,
+            input_shape=_input_shape.data if _input_shape else None,
             layout=_layout.data if _layout else None,
             mean=_mean.data if _mean else None,
-            scale=_scale.data if _scale else None,
+            input_scale=_input_scale.data if _input_scale else None,
             thread_count=_thread_count.data if _thread_count else None,
             inference_requests_count=_inference_requests_count.data if _inference_requests_count else None,
         )

src/benchmark/frameworks/config_parser/framework_independent_parameters.py

@@ -8,7 +8,7 @@ def __init__(self, inference_framework, batch_size, device, iterarion_count, tes
         self.device = None
         self.iteration = None
         self.test_time_limit = None
-        if self._parameter_not_is_none(inference_framework):
+        if self._parameter_is_not_none(inference_framework):
             self.inference_framework = inference_framework
         else:
             raise ValueError('Inference framework is required parameter.')
@@ -17,25 +17,18 @@ def __init__(self, inference_framework, batch_size, device, iterarion_count, tes
                 self.batch_size = int(batch_size)
             else:
                 raise ValueError('Batch size can only take values: integer greater than zero.')
-        if self._device_is_correct(device):
+        if self._parameter_is_not_none(device):
             self.device = device.upper()
         else:
-            raise ValueError('Device is required parameter. '
-                             'Supported values: CPU, GPU, FPGA, MYRIAD.')
-        if self._parameter_not_is_none(iterarion_count) and self._int_value_is_correct(iterarion_count):
+            raise ValueError('Device is required parameter.')
+        if self._parameter_is_not_none(iterarion_count) and self._int_value_is_correct(iterarion_count):
             self.iteration = int(iterarion_count)
         else:
             raise ValueError('Iteration count is required parameter. '
                              'Iteration count can only take values: integer greater than zero.')
-        if self._parameter_not_is_none(test_time_limit) and self._float_value_is_correct(test_time_limit):
+
+        if self._parameter_is_not_none(test_time_limit) and self._float_value_is_correct(test_time_limit):
             self.test_time_limit = float(test_time_limit)
         else:
             raise ValueError('Test time limit is required parameter. '
                              'Test time limit can only `take values: float greater than zero.')
-
-    @staticmethod
-    def _device_is_correct(device):
-        const_correct_devices = ['CPU', 'GPU', 'MYRIAD', 'FPGA']
-        if device.upper() in const_correct_devices:
-            return True
-        return False

src/benchmark/frameworks/config_parser/framework_parameters_cpp.py

@@ -2,23 +2,27 @@
 
 
 class CppParameters(FrameworkParameters):
-    def __init__(self, shape, layout, mean, scale, thread_count, inference_requests_count):
-        self.shape = None
+    def __init__(self, backend, input_shape, layout, mean, input_scale, thread_count, inference_requests_count):
+        self.backend = None
+        self.input_shape = None
         self.layout = None
         self.mean = None
-        self.scale = None
+        self.input_scale = None
         self.thread_count = None
         self.inference_requests_count = None
 
-        if shape is not None:
-            self.shape = shape.strip()
-        if layout is not None:
+        if self._parameter_is_not_none(backend):
+            self.backend = backend.strip()
+        if self._parameter_is_not_none(input_shape):
+            self.input_shape = input_shape.strip()
+        if self._parameter_is_not_none(layout):
             self.layout = layout.strip()
-        if mean is not None:
+        if self._parameter_is_not_none(mean):
             self.mean = mean.strip()
-        if scale is not None:
-            self.scale = scale.strip()
-        if thread_count is not None and self._int_value_is_correct(thread_count):
+        if self._parameter_is_not_none(input_scale):
+            self.input_scale = input_scale.strip()
+        if self._parameter_is_not_none(thread_count) and self._int_value_is_correct(thread_count):
             self.thread_count = thread_count
-        if inference_requests_count is not None and self._int_value_is_correct(inference_requests_count):
+        if self._parameter_is_not_none(inference_requests_count) and self._int_value_is_correct(
+                inference_requests_count):
             self.inference_requests_count = inference_requests_count

src/benchmark/frameworks/config_parser/framework_parameters_parser.py

@@ -1,6 +1,6 @@
 class FrameworkParameters:
     @staticmethod
-    def _parameter_not_is_none(parameter):
+    def _parameter_is_not_none(parameter):
         return parameter is not None
 
     @staticmethod

src/benchmark/frameworks/config_parser/test_reporter.py

@@ -1,4 +1,7 @@
 import abc
+import logging as log
+
+from collections import OrderedDict
 
 
 class Test(metaclass=abc.ABCMeta):
@@ -7,10 +10,25 @@ def __init__(self, model, dataset, indep_parameters, dep_parameters):
         self.dataset = dataset
         self.indep_parameters = indep_parameters
         self.dep_parameters = dep_parameters
+        self._log = log
+
+    def get_report(self, **kwargs):
+        parameters = self.prepare_framework_params()
+        other_param = self._get_optional_parameters_string(parameters)
+
+        report_res = {
+            'task': self.model.task,
+            'model': self.model.name,
+            'dataset': self.dataset.name,
+            'source_framework': self.model.source_framework,
+            'inference_framework': self.indep_parameters.inference_framework,
+            'precision': self.model.precision,
+            'batch_size': self.indep_parameters.batch_size,
+            'mode': 'Sync',
+            'framework_params': other_param,
+        }
 
-    @abc.abstractmethod
-    def get_report(self):
-        pass
+        return report_res
 
     @staticmethod
     def _get_optional_parameters_string(parameters_dict):
@@ -19,3 +37,49 @@ def _get_optional_parameters_string(parameters_dict):
             if parameters_dict[key] is not None:
                 parameter_strings.append(f'{key}: {parameters_dict[key]}')
         return ', '.join(parameter_strings)
+
+    def prepare_framework_params(self):
+        parameters = OrderedDict()
+
+        parameters.update({'Device': self.indep_parameters.device})
+        parameters.update({'Iteration count': self.indep_parameters.iteration})
+
+        match_parameter_description = {}
+
+        match_parameter_description['code_source'] = 'Code Source'
+        match_parameter_description['runtime'] = 'Runtime'
+        match_parameter_description['hint'] = 'Hint'
+        match_parameter_description['frontend'] = 'Frontend'
+        match_parameter_description['backend'] = 'Backend'
+        match_parameter_description['delegate'] = 'Delegate'
+        match_parameter_description['delegate_options'] = 'Delegate options'
+
+        match_parameter_description['async_request'] = 'Async request count'
+        match_parameter_description['nthreads'] = 'Thread count'
+        match_parameter_description['thread_count'] = 'Thread count'
+        match_parameter_description['nstreams'] = 'Stream count'
+        match_parameter_description['kmp_affinity'] = 'KMP_AFFINITY'
+        match_parameter_description['num_inter_threads'] = 'Inter threads'
+        match_parameter_description['num_intra_threads'] = 'Intra threads'
+
+        match_parameter_description['mean'] = 'Mean'
+        match_parameter_description['input_scale'] = 'Scale'
+        match_parameter_description['layout'] = 'Layout'
+        match_parameter_description['input_shape'] = 'Input shape'
+        match_parameter_description['input_name'] = 'Input name'
+        match_parameter_description['output_names'] = 'Output names'
+        # duplicate because pytorch launcher does not match common template. To be fixed
+        match_parameter_description['output_name'] = 'Output name'
+        match_parameter_description['normalize'] = 'Normalization flag'
+        match_parameter_description['std'] = 'Standard deviation'
+        match_parameter_description['channel_swap'] = 'Channel swap'
+        match_parameter_description['swapRB'] = 'Channel swap'
+        match_parameter_description['crop'] = 'Crop'
+        match_parameter_description['model_type'] = 'Model type'
+        match_parameter_description['inference_mode'] = 'Inference mode'
+
+        for parameter, description in match_parameter_description.items():
+            if hasattr(self.dep_parameters, parameter) and getattr(self.dep_parameters, parameter) is not None:
+                parameters.update({description: getattr(self.dep_parameters, parameter)})
+
+        return parameters

src/benchmark/frameworks/config_parser/test_reporter_cpp.py

@@ -0,0 +1,39 @@
+from .test_reporter import Test
+
+
+class CppTest(Test):
+    def get_report(self, process):
+        tensors_num = self.dep_parameters.inference_requests_count
+        json_report_content = process.get_json_report_content()
+        if process.get_status() == 0 and not tensors_num:
+            self._log.info('InferenceRequestsCount is not set in XML config, '
+                           'will try to extract it from the launcher JSON report or console output')
+            tensors_num = json_report_content['configurations_setup']['tensors_num']
+
+        batch_size = self.indep_parameters.batch_size
+        if process.get_status() == 0 and not batch_size:
+            self._log.info('BatchSize is not set in XML config, '
+                           'will try to extract it from the launcher JSON report')
+            batch_size = json_report_content['configurations_setup']['batch_size']
+
+        actual_iterations = json_report_content['execution_results']['iterations_num']
+
+        parameters = self.prepare_framework_params()
+        parameters['Infer request count'] = 1
+        parameters['Number of tensors'] = tensors_num
+        parameters['Iteration count'] = actual_iterations
+        optional_parameters_string = self._get_optional_parameters_string(parameters)
+
+        report_res = {
+            'task': self.model.task,
+            'model': self.model.name,
+            'dataset': self.dataset.name,
+            'source_framework': self.model.source_framework,
+            'inference_framework': self.indep_parameters.inference_framework,
+            'precision': self.model.precision,
+            'batch_size': batch_size,
+            'mode': 'Sync',
+            'framework_params': optional_parameters_string,
+        }
+
+        return report_res

src/benchmark/frameworks/intel_caffe/intel_caffe_parameters_parser.py

@@ -41,25 +41,25 @@ def __init__(self, channel_swap, mean, input_scale, thread_count, kmp_affinity):
         self.nthreads = None
         self.kmp_affinity = None
 
-        if self._parameter_not_is_none(channel_swap):
+        if self._parameter_is_not_none(channel_swap):
             if self._channel_swap_is_correct(channel_swap):
                 self.channel_swap = channel_swap
             else:
                 raise ValueError('Channel swap can only take values: list of unique values 0, 1, 2.')
-        if self._parameter_not_is_none(mean):
+        if self._parameter_is_not_none(mean):
             if self._mean_is_correct(mean):
                 self.mean = mean
             else:
                 raise ValueError('Mean can only take values: list of 3 float elements.')
-        if self._parameter_not_is_none(input_scale):
+        if self._parameter_is_not_none(input_scale):
             if self._float_value_is_correct(input_scale):
                 self.input_scale = input_scale
             else:
                 raise ValueError('Input scale can only take values: float greater than zero.')
-        if self._parameter_not_is_none(thread_count):
+        if self._parameter_is_not_none(thread_count):
             if self._int_value_is_correct(thread_count):
                 self.nthreads = thread_count
             else:
                 raise ValueError('Threads count can only take integer value')
-        if self._parameter_not_is_none(kmp_affinity):
+        if self._parameter_is_not_none(kmp_affinity):
             self.kmp_affinity = kmp_affinity

src/benchmark/frameworks/intel_caffe/intel_caffe_wrapper.py

@@ -1,5 +1,5 @@
 from .intel_caffe_process import IntelCaffeProcess
-from .intel_caffe_test import IntelCaffeTest
+from ..config_parser.test_reporter import Test
 from ..framework_wrapper import FrameworkWrapper
 from ..known_frameworks import KnownFrameworks
 
@@ -13,4 +13,4 @@ def create_process(test, executor, log, cpp_benchmarks_dir=None):
 
     @staticmethod
     def create_test(model, dataset, indep_parameters, dep_parameters):
-        return IntelCaffeTest(model, dataset, indep_parameters, dep_parameters)
+        return Test(model, dataset, indep_parameters, dep_parameters)

src/benchmark/frameworks/mxnet/mxnet_parameters_parser.py

@@ -46,15 +46,15 @@ def __init__(self, input_name, input_shape, normalize, mean, std, channel_swap):
         self.std = None
         self.channel_swap = None
 
-        if self._parameter_not_is_none(input_name):
+        if self._parameter_is_not_none(input_name):
             self.input_name = input_name
-        if self._parameter_not_is_none(input_shape):
+        if self._parameter_is_not_none(input_shape):
             self.input_shape = input_shape
-        if self._parameter_not_is_none(normalize):
+        if self._parameter_is_not_none(normalize):
             self.normalize = normalize
-        if self._parameter_not_is_none(mean):
+        if self._parameter_is_not_none(mean):
             self.mean = mean
-        if self._parameter_not_is_none(std):
+        if self._parameter_is_not_none(std):
             self.std = std
-        if self._parameter_not_is_none(channel_swap):
+        if self._parameter_is_not_none(channel_swap):
             self.channel_swap = channel_swap