diff --git a/src/benchmark/README.md b/src/benchmark/README.md index 5524c8d38..2e78c4b04 100644 --- a/src/benchmark/README.md +++ b/src/benchmark/README.md @@ -114,9 +114,9 @@ Intel® Optimization for TensorFlow, TensorFlow Lite, OpenCV, MXNet и PyTorch осуществляется последовательный и независимый запуск запросов. Запуск очередного запроса выполняется после завершения предыдущего. Для каждого запроса осуществляется замер времени его выполнения. -Для множества полученных времен определяется стандартное среднеквадратичное -отклонение, и отбрасываются времена, выходящие за пределы трех стандартных -отклонений от среднего времени вывода. Результирующий набор времен +Для множества полученных времен определяется стандартное среднеквадратичное +отклонение, и отбрасываются времена, выходящие за пределы трех стандартных +отклонений от среднего времени вывода. Результирующий набор времен используется для вычисления показателя латентности. Остальные показатели вычисляются для всего множества времен. diff --git a/src/benchmark/frameworks/config_parser/test_reporter.py b/src/benchmark/frameworks/config_parser/test_reporter.py index 76c9f1102..7f16c7022 100644 --- a/src/benchmark/frameworks/config_parser/test_reporter.py +++ b/src/benchmark/frameworks/config_parser/test_reporter.py @@ -62,19 +62,7 @@ def prepare_framework_params(self): match_parameter_description['num_inter_threads'] = 'Inter threads' match_parameter_description['num_intra_threads'] = 'Intra threads' - match_parameter_description['mean'] = 'Mean' - match_parameter_description['input_scale'] = 'Scale' - match_parameter_description['layout'] = 'Layout' - match_parameter_description['input_shape'] = 'Input shape' - match_parameter_description['input_name'] = 'Input name' - match_parameter_description['output_names'] = 'Output names' - # duplicate because pytorch launcher does not match common template. To be fixed - match_parameter_description['output_name'] = 'Output name' - match_parameter_description['normalize'] = 'Normalization flag' - match_parameter_description['std'] = 'Standard deviation' - match_parameter_description['channel_swap'] = 'Channel swap' - match_parameter_description['swapRB'] = 'Channel swap' - match_parameter_description['crop'] = 'Crop' + match_parameter_description['hybridize'] = 'Hybridization flag' match_parameter_description['model_type'] = 'Model type' match_parameter_description['inference_mode'] = 'Inference mode' diff --git a/src/benchmark/frameworks/mxnet/mxnet_parameters_parser.py b/src/benchmark/frameworks/mxnet/mxnet_parameters_parser.py index 37663ab93..4c9d4b820 100644 --- a/src/benchmark/frameworks/mxnet/mxnet_parameters_parser.py +++ b/src/benchmark/frameworks/mxnet/mxnet_parameters_parser.py @@ -7,6 +7,7 @@ def parse_parameters(self, curr_test): CONFIG_FRAMEWORK_DEPENDENT_TAG = 'FrameworkDependent' CONFIG_FRAMEWORK_DEPENDENT_INPUT_NAME_TAG = 'InputName' CONFIG_FRAMEWORK_DEPENDENT_INPUT_SHAPE_TAG = 'InputShape' + CONFIG_FRAMEWORK_DEPENDENT_HYBRIDIZE_TAG = 'Hybridize' CONFIG_FRAMEWORK_DEPENDENT_NORMALIZE_TAG = 'Normalize' CONFIG_FRAMEWORK_DEPENDENT_MEAN_TAG = 'Mean' CONFIG_FRAMEWORK_DEPENDENT_STD_TAG = 'Std' @@ -18,6 +19,9 @@ def parse_parameters(self, curr_test): CONFIG_FRAMEWORK_DEPENDENT_INPUT_NAME_TAG)[0].firstChild _input_shape = dep_parameters_tag.getElementsByTagName( CONFIG_FRAMEWORK_DEPENDENT_INPUT_SHAPE_TAG)[0].firstChild + _hybridize = dep_parameters_tag.getElementsByTagName( + CONFIG_FRAMEWORK_DEPENDENT_HYBRIDIZE_TAG)[0].firstChild + print(_hybridize.data) _normalize = dep_parameters_tag.getElementsByTagName( CONFIG_FRAMEWORK_DEPENDENT_NORMALIZE_TAG)[0].firstChild _mean = dep_parameters_tag.getElementsByTagName( @@ -30,6 +34,7 @@ def parse_parameters(self, curr_test): return MXNetParameters( input_name=_input_name.data if _input_name else None, input_shape=_input_shape.data if _input_shape else None, + hybridize=_hybridize.data if _hybridize else None, normalize=_normalize.data if _normalize else None, mean=_mean.data if _mean else None, std=_std.data if _std else None, @@ -38,9 +43,11 @@ def parse_parameters(self, curr_test): class MXNetParameters(FrameworkParameters): - def __init__(self, input_name, input_shape, normalize, mean, std, channel_swap): + def __init__(self, input_name, input_shape, hybridize, normalize, + mean, std, channel_swap): self.input_name = None self.input_shape = None + self.hybridize = None self.normalize = None self.mean = None self.std = None @@ -50,6 +57,8 @@ def __init__(self, input_name, input_shape, normalize, mean, std, channel_swap): self.input_name = input_name if self._parameter_is_not_none(input_shape): self.input_shape = input_shape + if self._parameter_is_not_none(hybridize): + self.hybridize = hybridize if self._parameter_is_not_none(normalize): self.normalize = normalize if self._parameter_is_not_none(mean): diff --git a/src/benchmark/frameworks/mxnet/mxnet_process.py b/src/benchmark/frameworks/mxnet/mxnet_process.py index bae5a28c8..9fd0ed197 100644 --- a/src/benchmark/frameworks/mxnet/mxnet_process.py +++ b/src/benchmark/frameworks/mxnet/mxnet_process.py @@ -48,6 +48,11 @@ def _fill_command_line(self): common_params = MXNetProcess._add_optional_argument_to_cmd_line( common_params, '--input_name', input_name) + hybridize = self._test.dep_parameters.hybridize + if hybridize == 'True': + common_params = MXNetProcess._add_flag_to_cmd_line( + common_params, '--hybrid') + normalize = self._test.dep_parameters.normalize if normalize == 'True': common_params = MXNetProcess._add_flag_to_cmd_line( diff --git a/src/configs/benchmark_configuration_file_template.xml b/src/configs/benchmark_configuration_file_template.xml index 49160f425..4de3cb5cb 100644 --- a/src/configs/benchmark_configuration_file_template.xml +++ b/src/configs/benchmark_configuration_file_template.xml @@ -177,6 +177,7 @@ + diff --git a/src/csv2xlsx/benchmark_table_creator.py b/src/csv2xlsx/benchmark_table_creator.py index 3be222e54..d166c7800 100644 --- a/src/csv2xlsx/benchmark_table_creator.py +++ b/src/csv2xlsx/benchmark_table_creator.py @@ -97,7 +97,8 @@ def _get_devices(self): for inference_framework in machine_inference_frameworks: framework_devices = [] for key, value in self._data_dictionary[self._KEY_PARAMETERS].items(): - pattern = re.compile(r'[.]*Device:[ ]*(?P[^,]+)[,]+[.]*') + pattern = re.compile(r'[.]*Device:[ ]*(?P[^,]+)[,]+[ ]*' + r'(?P[\w\W\d\.,\[\];:]+)') matcher = re.match(pattern, value) device_name = matcher.group('device_name') if (self._data_dictionary[self._KEY_INFRASTRUCTURE][key] == machine @@ -124,7 +125,8 @@ def _get_precisions(self): for device in framework_devices: device_precisions = [] for key, value in self._data_dictionary[self._KEY_PARAMETERS].items(): - pattern = re.compile(r'[.]*Device:[ ]*(?P[^,]+)[,]+[.]*') + pattern = re.compile(r'[.]*Device:[ ]*(?P[^,]+)[,]+[ ]*' + r'(?P[\w\W\d\.,\[\];:]+)') matcher = re.match(pattern, value) device_name = matcher.group('device_name') if (self._data_dictionary[self._KEY_INFRASTRUCTURE][key] == machine @@ -157,7 +159,8 @@ def _get_execution_modes(self): for precision in device_precisions: device_precision_modes = [] for key, value in self._data_dictionary[self._KEY_PARAMETERS].items(): - pattern = re.compile(r'[.]*Device:[ ]*(?P[^,]+)[,]+[.]*') + pattern = re.compile(r'[.]*Device:[ ]*(?P[^,]+)[,]+[ ]*' + r'(?P[\w\W\d\.,\[\];:]+)') matcher = re.match(pattern, value) device_name = matcher.group('device_name') if (self._data_dictionary[self._KEY_INFRASTRUCTURE][key] == machine @@ -175,105 +178,180 @@ def _get_execution_modes(self): logging.info(f'FINISH: _get_execution_modes(). {self._execution_modes}') + def _get_execution_parameters(self): + logging.info('START: _get_execution_parameters()') + + self._execution_parameters = [] + for idx in range(len(self._infrastructure)): + machine = self._infrastructure[idx] + machine_frameworks = self._inference_frameworks[idx] + machine_execparams = [] + for idx2 in range(len(machine_frameworks)): + framework = machine_frameworks[idx2] + machine_framework_devices = self._devices[idx][idx2] + machine_framework_execparams = [] + for idx3 in range(len(machine_framework_devices)): + device = machine_framework_devices[idx3] + machine_framework_device_precisions = self._precisions[idx][idx2][idx3] + framework_device_execparams = [] + for idx4 in range(len(machine_framework_device_precisions)): + precision = machine_framework_device_precisions[idx4] + execution_modes = self._execution_modes[idx][idx2][idx3][idx4] + framework_device_precision_execparams = [] + for mode in execution_modes: + framework_device_precision_mode_execparams = [] + for key, value in self._data_dictionary[self._KEY_PARAMETERS].items(): + pattern = re.compile(r'[.]*Device:[ ]*(?P[^,]+)[,]+[ ]*' + r'(?P[\w\W\d\.,\[\];:]+)') + matcher = re.match(pattern, value) + device_name = matcher.group('device_name') + parameters = matcher.group('parameters') + if (self._data_dictionary[self._KEY_INFRASTRUCTURE][key] == machine + and self._data_dictionary[self._KEY_INFERENCE_FRAMEWORK][key] == framework + and device_name == device + and self._data_dictionary[self._KEY_PRECISION][key] == precision + and self._data_dictionary[self._KEY_EXECUTION_MODE][key] == mode + and parameters not in framework_device_precision_mode_execparams): + framework_device_precision_mode_execparams.append(parameters) + framework_device_precision_execparams.append(framework_device_precision_mode_execparams) + framework_device_execparams.append(framework_device_precision_execparams) + machine_framework_execparams.append(framework_device_execparams) + machine_execparams.append(machine_framework_execparams) + self._execution_parameters.append(machine_execparams) + + logging.info(f'FINISH: _get_execution_parameters(). {self._execution_parameters}') + def _fill_horizontal_title(self): logging.info('START: _fill_horizontal_title()') - rel_row_idx = 4 + rel_row_idx = 5 rel_col_idx = 5 self._col_indeces = [] for idx in range(len(self._infrastructure)): - row_idx = rel_row_idx - col_idx = rel_col_idx - num_cols = 0 machine = self._infrastructure[idx] - machine_inference_frameworks = self._inference_frameworks[idx] - col_idx3 = col_idx + machine_frameworks = self._inference_frameworks[idx] + + row_idx = rel_row_idx + + col_idx2 = rel_col_idx + num_cols2 = 0 col_indeces2 = [] - for idx2 in range(len(machine_inference_frameworks)): - machine_framework = machine_inference_frameworks[idx2] + for idx2 in range(len(machine_frameworks)): + machine_framework = machine_frameworks[idx2] machine_framework_devices = self._devices[idx][idx2] - col_idx2 = col_idx - num_cols2 = 0 + + col_idx3 = col_idx2 + num_cols3 = 0 col_indeces3 = [] for idx3 in range(len(machine_framework_devices)): machine_framework_device = machine_framework_devices[idx3] - framework_device_precisions = self._precisions[idx][idx2][idx3] - col_idx1 = col_idx - num_cols1 = 0 + machine_framework_device_precisions = self._precisions[idx][idx2][idx3] + + col_idx4 = col_idx3 + num_cols4 = 0 col_indeces4 = [] - for idx4 in range(len(framework_device_precisions)): - framework_device_precision = framework_device_precisions[idx4] - framework_device_precision_modes = self._execution_modes[idx][idx2][idx3][idx4] + for idx4 in range(len(machine_framework_device_precisions)): + machine_framework_device_precision = machine_framework_device_precisions[idx4] + machine_framework_device_precision_modes = self._execution_modes[idx][idx2][idx3][idx4] + + col_idx5 = col_idx4 + num_cols5 = 0 col_indeces5 = [] - for idx5 in range(len(framework_device_precision_modes)): - framework_device_precision_mode = framework_device_precision_modes[idx5] - self._sheet.write(row_idx, col_idx + idx5, - framework_device_precision_mode, - self._cell_format_title1) - col_indeces5.append(col_idx + idx5) - k = len(framework_device_precision_modes) - if k > 1: - self._sheet.merge_range(row_idx - 1, col_idx1, - row_idx - 1, col_idx1 + k - 1, - framework_device_precision, + for idx5 in range(len(machine_framework_device_precision_modes)): + machine_framework_device_precision_mode = machine_framework_device_precision_modes[idx5] + machine_framework_device_precision_mode_execparams = ( + self._execution_parameters[idx][idx2][idx3][idx4][idx5]) + + col_idx6 = col_idx5 + num_cols6 = len(machine_framework_device_precision_mode_execparams) + col_indeces6 = [] + for idx6 in range(num_cols6): + execparams = machine_framework_device_precision_mode_execparams[idx6] + self._sheet.write(row_idx, col_idx6 + idx6, + execparams, + self._cell_format_title1) + col_indeces6.append(col_idx6 + idx6) + + if num_cols6 > 1: + self._sheet.merge_range(row_idx - 1, col_idx5, + row_idx - 1, col_idx5 + num_cols6 - 1, + machine_framework_device_precision_mode, + self._cell_format_title1) + elif num_cols6 == 1: + self._sheet.write(row_idx - 1, col_idx5, + machine_framework_device_precision_mode, + self._cell_format_title1) + else: + msg = 'Incorrect number of execution modes' + logging.error(msg) + raise ValueError(msg) + col_idx5 += num_cols6 + num_cols5 += num_cols6 + col_indeces5.append(col_indeces6) + if num_cols5 > 1: + self._sheet.merge_range(row_idx - 2, col_idx4, + row_idx - 2, col_idx4 + num_cols5 - 1, + machine_framework_device_precision, self._cell_format_title1) - elif k == 1: - self._sheet.write(row_idx - 1, col_idx1, - framework_device_precision, + elif num_cols5 == 1: + self._sheet.write(row_idx - 2, col_idx4, + machine_framework_device_precision, self._cell_format_title1) else: - msg = 'Incorrect number of device precision modes' + msg = 'Incorrect number of precisions' logging.error(msg) raise ValueError(msg) - col_idx += k - col_idx1 += k - num_cols1 += k - num_cols2 += k - num_cols += k + col_idx4 += num_cols5 + num_cols4 += num_cols5 col_indeces4.append(col_indeces5) - if num_cols1 > 1: - self._sheet.merge_range(row_idx - 2, col_idx2, - row_idx - 2, col_idx2 + num_cols1 - 1, + if num_cols4 > 1: + self._sheet.merge_range(row_idx - 3, col_idx3, + row_idx - 3, col_idx3 + num_cols4 - 1, machine_framework_device, self._cell_format_title1) - elif num_cols1 == 1: - self._sheet.write(row_idx - 2, col_idx2, + elif num_cols4 == 1: + self._sheet.write(row_idx - 3, col_idx3, machine_framework_device, self._cell_format_title1) else: msg = 'Incorrect number of devices' logging.error(msg) raise ValueError(msg) - col_idx2 += num_cols1 + col_idx3 += num_cols4 + num_cols3 += num_cols4 col_indeces3.append(col_indeces4) - if num_cols2 > 1: - self._sheet.merge_range(row_idx - 3, col_idx3, - row_idx - 3, col_idx3 + num_cols2 - 1, - machine_framework, self._cell_format_title1) - elif num_cols2 == 1: - self._sheet.write(row_idx - 3, col_idx3, - machine_framework, self._cell_format_title1) + if num_cols3 > 1: + self._sheet.merge_range(row_idx - 4, col_idx2, + row_idx - 4, col_idx2 + num_cols3 - 1, + machine_framework, + self._cell_format_title1) + elif num_cols3 == 1: + self._sheet.write(row_idx - 4, col_idx2 + num_cols3 - 1, + machine_framework, + self._cell_format_title1) else: msg = 'Incorrect number of frameworks' logging.error(msg) raise ValueError(msg) - col_idx3 += num_cols2 + col_idx2 += num_cols3 + num_cols2 += num_cols3 col_indeces2.append(col_indeces3) - if num_cols > 1: - self._sheet.merge_range(row_idx - 4, rel_col_idx, - row_idx - 4, rel_col_idx + num_cols - 1, + if num_cols2 > 1: + self._sheet.merge_range(row_idx - 5, rel_col_idx, + row_idx - 5, rel_col_idx + num_cols2 - 1, machine, self._cell_format_title1) - elif num_cols == 1: - self._sheet.write(row_idx - 4, rel_col_idx, + elif num_cols2 == 1: + self._sheet.write(row_idx - 5, rel_col_idx, machine, self._cell_format_title1) else: msg = 'Incorrect number of machines' logging.error(msg) raise ValueError(msg) - rel_col_idx += num_cols + rel_col_idx += num_cols2 self._col_indeces.append(col_indeces2) + logging.info(self._col_indeces) logging.info('FINISH: _fill_horizontal_title()') def _add_new_line(self, values, sep=','): @@ -286,33 +364,34 @@ def create_table_header(self): self._init_xlsx_parameters() # Freeze title panes - self._sheet.freeze_panes(5, 5) + self._sheet.freeze_panes(6, 5) # Write horizontal title (first cells before infrastructure) - self._sheet.merge_range('A1:A5', self._KEY_TASK_TYPE, self._cell_format_title1) + self._sheet.merge_range('A1:A6', self._KEY_TASK_TYPE, self._cell_format_title1) col_width = XlsxTable._get_column_width( self._data_dictionary[self._KEY_TOPOLOGY_NAME], self._cell_format) self._sheet.set_column(1, 1, col_width) - self._sheet.merge_range('B1:B5', self._KEY_TOPOLOGY_NAME, self._cell_format_title1) + self._sheet.merge_range('B1:B6', self._KEY_TOPOLOGY_NAME, self._cell_format_title1) col_width = XlsxTable._get_column_width( self._data_dictionary[self._KEY_TRAIN_FRAMEWORK], self._cell_format) self._sheet.set_column(2, 2, col_width) - self._sheet.merge_range('C1:C5', self._KEY_TRAIN_FRAMEWORK, self._cell_format_title1) + self._sheet.merge_range('C1:C6', self._KEY_TRAIN_FRAMEWORK, self._cell_format_title1) self._add_new_line(self._data_dictionary[self._KEY_BLOB_SIZE]) col_width = XlsxTable._get_column_width( self._data_dictionary[self._KEY_BLOB_SIZE], self._cell_format) self._sheet.set_column(3, 3, col_width) - self._sheet.merge_range('D1:D5', self._KEY_BLOB_SIZE, self._cell_format_title1) - self._sheet.merge_range('E1:E5', self._KEY_BATCH_SIZE, self._cell_format_title1) + self._sheet.merge_range('D1:D6', self._KEY_BLOB_SIZE, self._cell_format_title1) + self._sheet.merge_range('E1:E6', self._KEY_BATCH_SIZE, self._cell_format_title1) self._get_infrastructure() self._get_inference_frameworks() self._get_devices() self._get_precisions() self._get_execution_modes() + self._get_execution_parameters() # Write horizontal title (cells corresponding infrastructure) self._fill_horizontal_title() @@ -324,15 +403,18 @@ def _find_column_idx(self, value): self._infrastructure) idx2 = self._find_inference_framework_idx(value[self._KEY_INFERENCE_FRAMEWORK], self._inference_frameworks[idx1]) - pattern = re.compile(r'[.]*Device:[ ]*(?P[^,]+)[,]+[.]*') + pattern = re.compile(r'[.]*Device:[ ]*(?P[^,]+)[,]+[ ]*(?P[\w\W\d\.,\[\];:]+)') matcher = re.match(pattern, value[self._KEY_PARAMETERS]) device_name = matcher.group('device_name') + parameters = matcher.group('parameters') idx3 = self._find_device_idx(device_name, self._devices[idx1][idx2]) idx4 = self._find_precision_idx(value[self._KEY_PRECISION], self._precisions[idx1][idx2][idx3]) idx5 = self._find_execution_mode_idx(value[self._KEY_EXECUTION_MODE], self._execution_modes[idx1][idx2][idx3][idx4]) - return self._col_indeces[idx1][idx2][idx3][idx4][idx5] + idx6 = self._find_execparams_idx(parameters, + self._execution_parameters[idx1][idx2][idx3][idx4][idx5]) + return self._col_indeces[idx1][idx2][idx3][idx4][idx5][idx6] def _find_row_records(self, task_type, topology_name, train_framework, blob_size, batch_size, experiments, @@ -392,14 +474,13 @@ def create_table_rows(self): logging.info('FINISH: create_table_rows()') def _get_records_group(self, task_records, topology_name, train_framework, - blob_size, processed_records_idxs): + processed_records_idxs): records_group = [] for idx in range(len(task_records)): record = task_records[idx] if (idx not in processed_records_idxs and record[self._KEY_TOPOLOGY_NAME] == topology_name - and record[self._KEY_TRAIN_FRAMEWORK] == train_framework - and record[self._KEY_BLOB_SIZE] == blob_size): + and record[self._KEY_TRAIN_FRAMEWORK] == train_framework): processed_records_idxs.append(idx) records_group.append(record) return records_group @@ -407,7 +488,7 @@ def _get_records_group(self, task_records, topology_name, train_framework, def write_test_results(self): logging.info('START: write_test_results()') - row_idx = 5 + row_idx = 6 for task_type, task_records in self._table_records.items(): # loop by tasks if len(task_records) <= 0: continue @@ -421,11 +502,9 @@ def write_test_results(self): for record in task_records: # searching for records for the same topologies topology_name = record[self._KEY_TOPOLOGY_NAME] train_framework = record[self._KEY_TRAIN_FRAMEWORK] - blob_size = record[self._KEY_BLOB_SIZE] records_group = self._get_records_group(task_records, topology_name, train_framework, - blob_size, processed_records_idxs) topology_num_records = len(records_group) if topology_num_records == 0: @@ -439,15 +518,12 @@ def write_test_results(self): row_idx + topology_num_records - 1, 2, train_framework, self._cell_format_title2) - self._sheet.merge_range(row_idx, 3, - row_idx + topology_num_records - 1, - 3, blob_size, - self._cell_format) else: self._sheet.write(row_idx, 1, topology_name, self._cell_format_title2) self._sheet.write(row_idx, 2, train_framework, self._cell_format_title2) - self._sheet.write(row_idx, 3, blob_size, self._cell_format) for topology_record in records_group: + self._sheet.write(row_idx, 3, topology_record[self._KEY_BLOB_SIZE], + self._cell_format_title2) self._sheet.write(row_idx, 4, topology_record[self._KEY_BATCH_SIZE], self._cell_format_title2) for key, value in topology_record[self._KEY_FPS].items(): @@ -476,18 +552,18 @@ def beautify_table(self): rel_col_idx = 5 # task type, topology, framework, blob sizes, batch size rel_row_idx = 0 - num_header_rows = 5 # infrastructure, framework, device, precision, mode - col_depth = 3 + num_header_rows = 6 # infrastructure, framework, device, precision, mode, parameters + col_depth = 4 self._draw_bold_bolder(0, 0, num_header_rows, rel_col_idx) - self._draw_bold_bolder(num_header_rows - 1, 0, - self._full_num_rows - num_header_rows + 1, + self._draw_bold_bolder(num_header_rows, 0, + self._full_num_rows - num_header_rows, rel_col_idx) for idx in range(len(self._infrastructure)): - execution_modes = list(deepflatten(self._execution_modes[idx], depth=col_depth)) - num_cols = len(execution_modes) + execution_parameters = list(deepflatten(self._execution_parameters[idx], depth=col_depth)) + num_cols = len(execution_parameters) self._draw_bold_bolder(rel_row_idx, rel_col_idx, num_header_rows, num_cols) - self._draw_bold_bolder(rel_row_idx + num_header_rows - 1, rel_col_idx, - self._full_num_rows - num_header_rows + 1, num_cols) + self._draw_bold_bolder(rel_row_idx + num_header_rows, rel_col_idx, + self._full_num_rows - num_header_rows, num_cols) rel_col_idx += num_cols logging.info('FINISH: beautify_table()') diff --git a/src/csv2xlsx/table_creator.py b/src/csv2xlsx/table_creator.py index 46b3bf116..bda78685e 100644 --- a/src/csv2xlsx/table_creator.py +++ b/src/csv2xlsx/table_creator.py @@ -117,48 +117,20 @@ def _find_execution_mode_idx(self, execution_mode, return self._find_idx(execution_mode, available_execution_modes, 'Unknown execution mode') + def _find_execparams_idx(self, parameters, available_parameters): + return self._find_idx(parameters, available_parameters, + 'Unknown param') + @abc.abstractmethod def _find_column_idx(self, value): pass def _draw_bold_bolder(self, rel_row_idx, rel_col_idx, num_rows, num_cols): - # top left corner - self._sheet.conditional_format(rel_row_idx, rel_col_idx, - rel_row_idx, rel_col_idx, - {'type': 'formula', 'criteria': 'True', - 'format': self._book.add_format({'top': 5, - 'bottom': 1, - 'left': 5, - 'right': 1})}) - # bottom left corner - self._sheet.conditional_format(rel_row_idx + num_rows - 1, rel_col_idx, - rel_row_idx + num_rows - 1, rel_col_idx, - {'type': 'formula', 'criteria': 'True', - 'format': self._book.add_format({'top': 1, - 'bottom': 5, - 'left': 5, - 'right': 1})}) - # top right corner - self._sheet.conditional_format(rel_row_idx, rel_col_idx + num_cols - 1, - rel_row_idx, rel_col_idx + num_cols - 1, - {'type': 'formula', 'criteria': 'True', - 'format': self._book.add_format({'top': 5, - 'bottom': 1, - 'left': 1, - 'right': 5})}) - # bottom right corner - self._sheet.conditional_format(rel_row_idx + num_rows - 1, rel_col_idx + num_cols - 1, - rel_row_idx + num_rows - 1, rel_col_idx + num_cols - 1, - {'type': 'formula', 'criteria': 'True', - 'format': self._book.add_format({'top': 1, - 'bottom': 5, - 'left': 1, - 'right': 5})}) if (num_cols > 2): # top self._sheet.conditional_format(rel_row_idx, rel_col_idx + 1, rel_row_idx, rel_col_idx + num_cols - 2, - {'type': 'formula', 'criteria': 'True', + {'type': 'no_blanks', 'format': self._book.add_format({'top': 5, 'bottom': 1, 'left': 1, @@ -166,26 +138,89 @@ def _draw_bold_bolder(self, rel_row_idx, rel_col_idx, num_rows, num_cols): # bottom self._sheet.conditional_format(rel_row_idx + num_rows - 1, rel_col_idx + 1, rel_row_idx + num_rows - 1, rel_col_idx + num_cols - 2, - {'type': 'formula', 'criteria': 'True', + {'type': 'no_blanks', 'format': self._book.add_format({'top': 1, 'bottom': 5, 'left': 1, 'right': 1})}) if (num_rows > 2): + # right + self._sheet.conditional_format(rel_row_idx + 1, rel_col_idx + num_cols - 1, + rel_row_idx + num_rows - 2, rel_col_idx + num_cols - 1, + {'type': 'no_blanks', + 'format': self._book.add_format({'top': 1, + 'bottom': 1, + 'left': 1, + 'right': 5})}) # left self._sheet.conditional_format(rel_row_idx + 1, rel_col_idx, rel_row_idx + num_rows - 2, rel_col_idx, - {'type': 'formula', 'criteria': 'True', + {'type': 'no_blanks', 'format': self._book.add_format({'top': 1, 'bottom': 1, 'left': 5, 'right': 1})}) - # right - self._sheet.conditional_format(rel_row_idx + 1, rel_col_idx + num_cols - 1, - rel_row_idx + num_rows - 2, rel_col_idx + num_cols - 1, - {'type': 'formula', 'criteria': 'True', + if num_cols == 1: + self._sheet.conditional_format(rel_row_idx, rel_col_idx, + rel_row_idx, rel_col_idx, + {'type': 'no_blanks', + 'format': self._book.add_format({'top': 5, + 'bottom': 1, + 'left': 5, + 'right': 5})}) + self._sheet.conditional_format(rel_row_idx + num_rows - 1, rel_col_idx, + rel_row_idx + num_rows - 1, rel_col_idx, + {'type': 'no_blanks', 'format': self._book.add_format({'top': 1, + 'bottom': 5, + 'left': 5, + 'right': 5})}) + if num_rows == 1: + self._sheet.conditional_format(rel_row_idx, rel_col_idx, + rel_row_idx, rel_col_idx, + {'type': 'no_blanks', + 'format': self._book.add_format({'top': 5, + 'bottom': 5, + 'left': 5, + 'right': 1})}) + self._sheet.conditional_format(rel_row_idx, rel_col_idx + num_cols - 1, + rel_row_idx, rel_col_idx + num_cols - 1, + {'type': 'no_blanks', + 'format': self._book.add_format({'top': 5, + 'bottom': 5, + 'left': 1, + 'right': 5})}) + if num_rows > 1 and num_cols > 1: + # top left corner + self._sheet.conditional_format(rel_row_idx, rel_col_idx, + rel_row_idx, rel_col_idx, + {'type': 'no_blanks', + 'format': self._book.add_format({'top': 5, 'bottom': 1, + 'left': 5, + 'right': 1})}) + # bottom left corner + self._sheet.conditional_format(rel_row_idx + num_rows - 1, rel_col_idx, + rel_row_idx + num_rows - 1, rel_col_idx, + {'type': 'no_blanks', + 'format': self._book.add_format({'top': 1, + 'bottom': 5, + 'left': 5, + 'right': 1})}) + # top right corner + self._sheet.conditional_format(rel_row_idx, rel_col_idx + num_cols - 1, + rel_row_idx, rel_col_idx + num_cols - 1, + {'type': 'no_blanks', + 'format': self._book.add_format({'top': 5, + 'bottom': 1, + 'left': 1, + 'right': 5})}) + # bottom right corner + self._sheet.conditional_format(rel_row_idx + num_rows - 1, rel_col_idx + num_cols - 1, + rel_row_idx + num_rows - 1, rel_col_idx + num_cols - 1, + {'type': 'no_blanks', + 'format': self._book.add_format({'top': 1, + 'bottom': 5, 'left': 1, 'right': 5})}) diff --git a/src/inference/inference_async_mode.py b/src/inference/inference_async_mode.py index b1f67daad..db7b5cbb8 100644 --- a/src/inference/inference_async_mode.py +++ b/src/inference/inference_async_mode.py @@ -179,22 +179,6 @@ def infer_async(compiled_model, number_iter, num_request, get_slice): return result, inference_time -def process_result(inference_time, batch_size, iteration_count): - average_time = inference_time / iteration_count - fps = pp.calculate_fps(batch_size * iteration_count, inference_time) - - return average_time, fps - - -def result_output(average_time, fps, log): - log.info('Average time of single pass : {0:.3f}'.format(average_time)) - log.info('FPS : {0:.3f}'.format(fps)) - - -def raw_result_output(average_time, fps): - print('{0:.3f},{1:.3f}'.format(average_time, fps)) - - def main(): log.basicConfig( format='[ %(levelname)s ] %(message)s', @@ -225,11 +209,9 @@ def main(): utils.reshape_input(model, args.batch_size) log.info('Prepare input data') - io.prepare_input(model, args.input) log.info('Create executable network') - compiled_model = utils.compile_model(core, model, args.device, args.priority) log.info('Runtime parameters') @@ -244,13 +226,24 @@ def main(): log.info(f'Starting inference ({args.number_iter} iterations) with {args.requests} requests on {args.device}') result, time = infer_async(compiled_model, args.number_iter, args.requests, io.get_slice_input) - average_time, fps = process_result(time, args.batch_size, args.number_iter) + + log.info('Computing performance metrics') + average_time, fps = pp.calculate_performance_metrics_async_mode(time, + args.batch_size, + args.number_iter) if not args.raw_output: - io.process_output(result, log) - result_output(average_time, fps, log) + if args.number_iter == 1: + try: + log.info('Inference results') + io.process_output(result, log) + except Exception as ex: + log.warning('Error when printing inference results. {0}'.format(str(ex))) + + log.info('Performance results') + pp.log_performance_metrics_async_mode(log, average_time, fps) else: - raw_result_output(average_time, fps) + pp.print_performance_metrics_async_mode(average_time, fps) del model del compiled_model del core diff --git a/src/inference/inference_caffe.py b/src/inference/inference_caffe.py index fd20d13a2..e83199fd0 100644 --- a/src/inference/inference_caffe.py +++ b/src/inference/inference_caffe.py @@ -162,24 +162,6 @@ def inference_caffe(net, number_iter, get_slice): return result, time_infer -def process_result(batch_size, inference_time): - inference_time = pp.three_sigma_rule(inference_time) - average_time = pp.calculate_average_time(inference_time) - latency = pp.calculate_latency(inference_time) - fps = pp.calculate_fps(batch_size, latency) - return average_time, latency, fps - - -def result_output(average_time, fps, latency, log): - log.info('Average time of single pass : {0:.3f}'.format(average_time)) - log.info('FPS : {0:.3f}'.format(fps)) - log.info('Latency : {0:.3f}'.format(latency)) - - -def raw_result_output(average_time, fps, latency): - print('{0:.3f},{1:.3f},{2:.3f}'.format(average_time, fps, latency)) - - def create_dict_for_transformer(args): dictionary = { 'channel_swap': args.channel_swap, @@ -206,28 +188,37 @@ def main(): set_device_to_infer(args.device) log.info('The device has been assigned: {0}'.format(args.device)) - log.info('Loading network files:\n\t {0}\n\t {1}'.format(args.model_prototxt, args.model_caffemodel)) - + log.info('Loading network files:\n\t {0}\n\t {1}'.format( + args.model_prototxt, args.model_caffemodel)) net = load_network(args.model_prototxt, args.model_caffemodel) net = network_input_reshape(net, args.batch_size) - input_shapes = get_input_shape(model_wrapper, net) + input_shapes = get_input_shape(model_wrapper, net) for layer in input_shapes: log.info('Shape for input layer {0}: {1}'.format(layer, input_shapes[layer])) - log.info('Prepare input data') + log.info('Prepare input data') io.prepare_input(net, args.input) log.info(f'Starting inference ({args.number_iter} iterations)') - result, inference_time = inference_caffe(net, args.number_iter, io.get_slice_input) - time, latency, fps = process_result(args.batch_size, inference_time) + + log.info('Computing performance metrics') + average_time, latency, fps = pp.calculate_performance_metrics_sync_mode(args.batch_size, + inference_time) if not args.raw_output: - io.process_output(result, log) - result_output(time, fps, latency, log) + if args.number_iter == 1: + try: + log.info('Inference results') + io.process_output(result, log) + except Exception as ex: + log.warning('Error when printing inference results. {0}'.format(str(ex))) + + log.info('Performance results') + pp.log_performance_metrics_sync_mode(log, average_time, fps, latency) else: - raw_result_output(time, fps, latency) + pp.print_performance_metrics_sync_mode(average_time, fps, latency) except Exception: log.error(traceback.format_exc()) sys.exit(1) diff --git a/src/inference/inference_mxnet.py b/src/inference/inference_mxnet.py index afd73cbed..1ffebe3d8 100644 --- a/src/inference/inference_mxnet.py +++ b/src/inference/inference_mxnet.py @@ -30,6 +30,11 @@ def cli_argument_parser(): help='Model name to download using GluonCV package.', type=str, dest='model_name') + parser.add_argument('--hybrid', + help='Flag to enable symbolic computations.' + 'Default value is false.', + action='store_true', + dest='hybrid') parser.add_argument('-i', '--input', help='Path to data.', required=True, @@ -158,7 +163,7 @@ def load_network_gluon(model_json, model_params, context, input_name): return deserialized_net -def load_network_gluon_model_zoo(model_name, context, save_model, path_save_model): +def load_network_gluon_model_zoo(model_name, hybrid, context, save_model, path_save_model): log.info(f'Loading network \"{model_name}\" from GluonCV model zoo') net = gluoncv.model_zoo.get_model(model_name, pretrained=True, ctx=context) @@ -174,8 +179,9 @@ def load_network_gluon_model_zoo(model_name, context, save_model, path_save_mode log.info(f'Info about the network:\n{net}') - log.info('Hybridizing model to accelerate inference') - net.hybridize() + log.info(f'Hybridizing model to accelerate inference: {hybrid}') + if hybrid is True: + net.hybridize() return net @@ -194,13 +200,12 @@ def create_dict_for_transformer(args): def create_dict_for_modelwrapper(args): dictionary = { 'input_name': args.input_name, - 'input_shape': args.input_shape, + 'input_shape': [args.batch_size] + args.input_shape[1:4], } return dictionary -def inference_mxnet(net, num_iterations, get_slice, input_name, - k=5, file_labels='image_net_labels.json'): +def inference_mxnet(net, num_iterations, get_slice, input_name): predictions = None time_infer = [] slice_input = None @@ -223,24 +228,6 @@ def inference_mxnet(net, num_iterations, get_slice, input_name, return predictions, time_infer -def process_result(batch_size, inference_time): - inference_time = pp.three_sigma_rule(inference_time) - average_time = pp.calculate_average_time(inference_time) - latency = pp.calculate_latency(inference_time) - fps = pp.calculate_fps(batch_size, latency) - return average_time, latency, fps - - -def result_output(average_time, fps, latency): - log.info('Average time of single pass : {0:.3f}'.format(average_time)) - log.info('FPS : {0:.3f}'.format(fps)) - log.info('Latency : {0:.3f}'.format(latency)) - - -def raw_result_output(average_time, fps, latency): - print('{0:.3f},{1:.3f},{2:.3f}'.format(average_time, fps, latency)) - - def prepare_output(result, output_names, task): if task == 'feedforward': return {} @@ -269,7 +256,7 @@ def main(): if ((args.model_name is not None) and (args.model_json is None) and (args.model_params is None)): - net = load_network_gluon_model_zoo(args.model_name, context, + net = load_network_gluon_model_zoo(args.model_name, args.hybrid, context, args.save_model, args.path_save_model) elif (args.model_json is not None) and (args.model_params is not None): net = load_network_gluon(args.model_json, args.model_params, context, @@ -287,7 +274,8 @@ def main(): io.get_slice_input, args.input_name) log.info('Computing performance metrics') - average_time, latency, fps = process_result(args.batch_size, inference_time) + average_time, latency, fps = pp.calculate_performance_metrics_sync_mode(args.batch_size, + inference_time) if not args.raw_output: if args.number_iter == 1: @@ -301,9 +289,9 @@ def main(): log.warning('Error when printing inference results. {0}'.format(str(ex))) log.info('Performance results') - result_output(average_time, fps, latency) + pp.log_performance_metrics_sync_mode(log, average_time, fps, latency) else: - raw_result_output(average_time, fps, latency) + pp.print_performance_metrics_sync_mode(average_time, fps, latency) except Exception: log.error(traceback.format_exc()) sys.exit(1) diff --git a/src/inference/inference_opencv.py b/src/inference/inference_opencv.py index fcb99f828..d41f1abee 100644 --- a/src/inference/inference_opencv.py +++ b/src/inference/inference_opencv.py @@ -202,15 +202,6 @@ def inference_opencv(net, input_name, output_names, number_iter, get_slice): return result, time_infer -def process_result(batch_size, inference_time): - inference_time = pp.three_sigma_rule(inference_time) - average_time = pp.calculate_average_time(inference_time) - latency = pp.calculate_latency(inference_time) - fps = pp.calculate_fps(batch_size, latency) - - return average_time, latency, fps - - def prepare_output(result, output_names, task, args): if task == 'feedforward': return {} @@ -222,16 +213,6 @@ def prepare_output(result, output_names, task, args): raise ValueError(f'Unsupported task {task} to print inference results') -def result_output(average_time, fps, latency, log): - log.info('Average time of single pass : {0:.3f}'.format(average_time)) - log.info('FPS : {0:.3f}'.format(fps)) - log.info('Latency : {0:.3f}'.format(latency)) - - -def raw_result_output(average_time, fps, latency): - print('{0:.3f},{1:.3f},{2:.3f}'.format(average_time, fps, latency)) - - def create_dict_for_transformer(args): dictionary = { 'scalefactor': 1 / args.inv_scale_factor, @@ -292,7 +273,8 @@ def main(): net, args.input_name, args.output_names, args.number_iter, io.get_slice_input) log.info('Computing performance metrics') - average_time, latency, fps = process_result(args.batch_size, inference_time) + average_time, latency, fps = pp.calculate_performance_metrics_sync_mode(args.batch_size, + inference_time) if not args.raw_output: if args.number_iter == 1: @@ -306,9 +288,9 @@ def main(): log.warning('Error when printing inference results. {0}'.format(str(ex))) log.info('Performance results') - result_output(average_time, fps, latency, log) + pp.log_performance_metrics_sync_mode(log, average_time, fps, latency) else: - raw_result_output(average_time, fps, latency) + pp.print_performance_metrics_sync_mode(average_time, fps, latency) except Exception: log.error(traceback.format_exc()) sys.exit(1) diff --git a/src/inference/inference_pytorch.py b/src/inference/inference_pytorch.py index 50c5b036f..54ae6b21f 100644 --- a/src/inference/inference_pytorch.py +++ b/src/inference/inference_pytorch.py @@ -210,24 +210,6 @@ def inference_pytorch(model, num_iterations, get_slice, input_name, inference_mo return predictions, time_infer -def process_result(batch_size, inference_time): - inference_time = pp.three_sigma_rule(inference_time) - average_time = pp.calculate_average_time(inference_time) - latency = pp.calculate_latency(inference_time) - fps = pp.calculate_fps(batch_size, latency) - return average_time, latency, fps - - -def result_output(average_time, fps, latency): - log.info('Average time of single pass : {0:.3f}'.format(average_time)) - log.info('FPS : {0:.3f}'.format(fps)) - log.info('Latency : {0:.3f}'.format(latency)) - - -def raw_result_output(average_time, fps, latency): - print('{0:.3f},{1:.3f},{2:.3f}'.format(average_time, fps, latency)) - - def prepare_output(result, output_names, task): if task == 'feedforward': return {} @@ -271,7 +253,8 @@ def main(): io.get_slice_input, args.input_name, args.inference_mode) log.info('Computing performance metrics') - average_time, latency, fps = process_result(args.batch_size, inference_time) + average_time, latency, fps = pp.calculate_performance_metrics_sync_mode(args.batch_size, + inference_time) if not args.raw_output: if args.number_iter == 1: @@ -285,9 +268,9 @@ def main(): log.warning('Error when printing inference results. {0}'.format(str(ex))) log.info('Performance results') - result_output(average_time, fps, latency) + pp.log_performance_metrics_sync_mode(log, average_time, fps, latency) else: - raw_result_output(average_time, fps, latency) + pp.print_performance_metrics_sync_mode(average_time, fps, latency) except Exception: log.error(traceback.format_exc()) sys.exit(1) diff --git a/src/inference/inference_sync_mode.py b/src/inference/inference_sync_mode.py index b5c051cba..6a8319ffe 100644 --- a/src/inference/inference_sync_mode.py +++ b/src/inference/inference_sync_mode.py @@ -156,26 +156,6 @@ def infer_sync(compiled_model, number_iter, get_slice): return result, time_infer -def process_result(inference_time, batch_size, min_infer_time): - correct_time = pp.delete_incorrect_time(inference_time, min_infer_time) - correct_time = pp.three_sigma_rule(correct_time) - average_time = pp.calculate_average_time(correct_time) - latency = pp.calculate_latency(correct_time) - fps = pp.calculate_fps(batch_size, latency) - - return average_time, latency, fps - - -def result_output(average_time, fps, latency, log): - log.info('Average time of single pass : {0:.3f}'.format(average_time)) - log.info('FPS : {0:.3f}'.format(fps)) - log.info('Latency : {0:.3f}'.format(latency)) - - -def raw_result_output(average_time, fps, latency): - print('{0:.3f},{1:.3f},{2:.3f}'.format(average_time, fps, latency)) - - def main(): log.basicConfig( format='[ %(levelname)s ] %(message)s', @@ -207,23 +187,31 @@ def main(): utils.reshape_input(model, args.batch_size) log.info('Prepare input data') - io.prepare_input(model, args.input) log.info('Create executable network') - compiled_model = utils.compile_model(core, model, args.device, args.priority) log.info(f'Starting inference ({args.number_iter} iterations) on {args.device}') + result, inference_time = infer_sync(compiled_model, args.number_iter, io.get_slice_input) - result, time = infer_sync(compiled_model, args.number_iter, io.get_slice_input) - average_time, latency, fps = process_result(time, args.batch_size, args.mininfer) + log.info('Computing performance metrics') + average_time, latency, fps = pp.calculate_performance_metrics_sync_mode(args.batch_size, + inference_time, + args.mininfer) if not args.raw_output: - io.process_output(result, log) - result_output(average_time, fps, latency, log) + if args.number_iter == 1: + try: + log.info('Inference results') + io.process_output(result, log) + except Exception as ex: + log.warning('Error when printing inference results. {0}'.format(str(ex))) + + log.info('Performance results') + pp.log_performance_metrics_sync_mode(log, average_time, fps, latency) else: - raw_result_output(average_time, fps, latency) + pp.print_performance_metrics_sync_mode(average_time, fps, latency) del model del compiled_model del core diff --git a/src/inference/inference_tensorflow.py b/src/inference/inference_tensorflow.py index e496137cb..88d76d024 100644 --- a/src/inference/inference_tensorflow.py +++ b/src/inference/inference_tensorflow.py @@ -246,25 +246,6 @@ def create_config_for_inference(num_intra_threads, num_inter_threads): return config -def process_result(batch_size, inference_time): - inference_time = pp.three_sigma_rule(inference_time) - average_time = pp.calculate_average_time(inference_time) - latency = pp.calculate_latency(inference_time) - fps = pp.calculate_fps(batch_size, latency) - - return average_time, latency, fps - - -def result_output(average_time, fps, latency, log): - log.info('Average time of single pass : {0:.3f}'.format(average_time)) - log.info('FPS : {0:.3f}'.format(fps)) - log.info('Latency : {0:.3f}'.format(latency)) - - -def raw_result_output(average_time, fps, latency): - print('{0:.3f},{1:.3f},{2:.3f}'.format(average_time, fps, latency)) - - def create_dict_for_transformer(args): dictionary = {'channel_swap': args.channel_swap, 'mean': args.mean, 'input_scale': args.input_scale} @@ -301,15 +282,24 @@ def main(): config) log.info('Computing performance metrics') - average_time, latency, fps = process_result(args.batch_size, inference_time) + average_time, latency, fps = pp.calculate_performance_metrics_sync_mode(args.batch_size, + inference_time) if not args.raw_output: if args.number_iter == 1: - result = prepare_output(result, outputs_names, args.task) - io.process_output(result, log) - result_output(average_time, fps, latency, log) + try: + log.info('Converting output tensor to print results') + result = prepare_output(result, outputs_names, args.task) + + log.info('Inference results') + io.process_output(result, log) + except Exception as ex: + log.warning('Error when printing inference results. {0}'.format(str(ex))) + + log.info('Performance results') + pp.log_performance_metrics_sync_mode(log, average_time, fps, latency) else: - raw_result_output(average_time, fps, latency) + pp.print_performance_metrics_sync_mode(average_time, fps, latency) except Exception as ex: log.error(str(ex)) sys.exit(1) diff --git a/src/inference/inference_tensorflowlite.py b/src/inference/inference_tensorflowlite.py index 75acaae33..be8d4076d 100644 --- a/src/inference/inference_tensorflowlite.py +++ b/src/inference/inference_tensorflowlite.py @@ -199,25 +199,6 @@ def reshape_model_input(io_model_wrapper, model, log): model.resize_tensor_input(model_input['index'], shape) -def process_result(batch_size, inference_time): - inference_time = pp.three_sigma_rule(inference_time) - average_time = pp.calculate_average_time(inference_time) - latency = pp.calculate_latency(inference_time) - fps = pp.calculate_fps(batch_size, latency) - - return average_time, latency, fps - - -def result_output(average_time, fps, latency, log): - log.info(f'Average time of single pass : {average_time:.3f}') - log.info(f'FPS : {fps:.3f}') - log.info(f'Latency : {latency:.3f}') - - -def raw_result_output(average_time, fps, latency): - print(f'{average_time:.3f},{fps:.3f},{latency:.3f}') - - def create_dict_for_transformer(args): dictionary = {} for name in args.input_names: @@ -317,7 +298,8 @@ def main(): log.info(f'Starting inference ({args.number_iter} iterations)') result, inference_time = inference_tflite(interpreter, args.number_iter, io.get_slice_input) - time, latency, fps = process_result(args.batch_size, inference_time) + average_time, latency, fps = pp.calculate_performance_metrics_sync_mode(args.batch_size, + inference_time) if not args.raw_output: if args.number_iter == 1: try: @@ -330,9 +312,9 @@ def main(): log.warning('Error when printing inference results. {0}'.format(str(ex))) log.info('Performance results') - result_output(time, fps, latency, log) + pp.log_performance_metrics_sync_mode(log, average_time, fps, latency) else: - raw_result_output(time, fps, latency) + pp.print_performance_metrics_sync_mode(average_time, fps, latency) except Exception: log.error(traceback.format_exc()) sys.exit(1) diff --git a/src/inference/postprocessing_data.py b/src/inference/postprocessing_data.py index ea4cc23da..856ba453f 100644 --- a/src/inference/postprocessing_data.py +++ b/src/inference/postprocessing_data.py @@ -36,3 +36,39 @@ def calculate_fps(pictures, time): if time == 0: return -1 return pictures / time + + +def calculate_performance_metrics_sync_mode(batch_size, inference_time, + min_infer_time=0.0): + inference_time = delete_incorrect_time(inference_time, min_infer_time) + inference_time = three_sigma_rule(inference_time) + average_time = calculate_average_time(inference_time) + latency = calculate_latency(inference_time) + fps = calculate_fps(batch_size, latency) + return average_time, latency, fps + + +def log_performance_metrics_sync_mode(log, average_time, fps, latency): + log.info(f'Average time of single pass : {average_time:.3f}') + log.info(f'FPS : {fps:.3f}') + log.info(f'Latency : {latency:.3f}') + + +def print_performance_metrics_sync_mode(average_time, fps, latency): + print(f'{average_time:.3f},{fps:.3f},{latency:.3f}') + + +def calculate_performance_metrics_async_mode(inference_time, batch_size, iteration_count): + average_time = inference_time / iteration_count + fps = calculate_fps(batch_size * iteration_count, inference_time) + + return average_time, fps + + +def log_performance_metrics_async_mode(log, average_time, fps): + log.info('Average time of single pass : {0:.3f}'.format(average_time)) + log.info('FPS : {0:.3f}'.format(fps)) + + +def print_performance_metrics_async_mode(average_time, fps): + print('{0:.3f},{1:.3f}'.format(average_time, fps)) diff --git a/src/inference/transformer.py b/src/inference/transformer.py index a473b4c72..fea99518c 100644 --- a/src/inference/transformer.py +++ b/src/inference/transformer.py @@ -217,6 +217,7 @@ def _transform(self, image): def transform_images(self, images, shape, element_type, *args): import mxnet + dataset_size = images.shape[0] new_shape = [dataset_size] + shape[1:] transformed_images = mxnet.nd.zeros(shape=new_shape, dtype=element_type) diff --git a/test/smoke_test/smoke_config.xml b/test/smoke_test/smoke_config.xml index f78ddc09b..aeb6aa1ec 100644 --- a/test/smoke_test/smoke_config.xml +++ b/test/smoke_test/smoke_config.xml @@ -299,6 +299,7 @@ data 1 3 224 224 + True True 0.485 0.456 0.406 0.229 0.224 0.225