From 2dc4c857f4f8574c0593ce30a7f2caea9281e722 Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Tue, 31 Jan 2023 15:14:22 +0300 Subject: [PATCH 01/63] MXNet inference for classification models using Gluon API --- src/inference/inference_mxnet.py | 240 ++++++++++++++++++++++++++++++ src/inference/io_model_wrapper.py | 17 +++ src/inference/transformer.py | 44 ++++++ 3 files changed, 301 insertions(+) create mode 100644 src/inference/inference_mxnet.py diff --git a/src/inference/inference_mxnet.py b/src/inference/inference_mxnet.py new file mode 100644 index 000000000..2699dda46 --- /dev/null +++ b/src/inference/inference_mxnet.py @@ -0,0 +1,240 @@ +import argparse +import logging as log +import sys +from time import time +import numpy as np +import json + +import mxnet +import mxnet.gluon.model_zoo.vision as model_zoo + +import postprocessing_data as pp +from io_adapter import IOAdapter +from io_model_wrapper import MXNetIOModelWrapper +from transformer import MXNetTransformer + + +def cli_argument_parser(): + parser = argparse.ArgumentParser() + + parser.add_argument('-m', '--model', + help='Path to an .json file with a trained model.', + type=str, + dest='model_json') + parser.add_argument('-w', '--weights', + help='Path to an .params file with a trained weights.', + type=str, + dest='model_params') + parser.add_argument('-n', '--model_name', + help='Model name to download using Gluon package.', + type=str, + required=True, + dest='model_name') + parser.add_argument('-i', '--input', + help='Path to data', + required=True, + type=str, + nargs='+', + dest='input') + parser.add_argument('-in', '--input_name', + help='Input name', + default='data', + type=str, + dest='input_name') + parser.add_argument('-is', '--input_shape', + help='Input shape WxHxC, W is an input tensor width, \ + H is an input tensor height, C is an input tensor \ + number of channnels', + default=None, + type=int, + nargs=4, + dest='input_shape') + parser.add_argument('--mean', + help='Parameter mean', + default=[0, 0, 0], + type=float, + nargs=3, + dest='mean') + parser.add_argument('--std', + help='Parameter standard deviation', + default=[1., 1., 1.], + type=float, + nargs=3, + dest='std') + parser.add_argument('--norm', + help='Flag to normalize input images', + default=True, + type=bool, + dest='norm') + parser.add_argument('--channel_swap', + help='Parameter channel swap', + default=[2, 0, 1], + type=int, + nargs=3, + dest='channel_swap') + parser.add_argument('-b', '--batch_size', + help='Size of the processed pack', + default=1, + type=int, + dest='batch_size') + parser.add_argument('-l', '--labels', + help='Labels mapping file', + default=None, + type=str, + dest='labels') + parser.add_argument('-nt', '--number_top', + help='Number of top results', + default=10, + type=int, + dest='number_top') + parser.add_argument('-t', '--task', + help='Output processing method. Default: without postprocess', + choices=['classification'], + default='classification', + type=str, + dest='task') + parser.add_argument('-ni', '--number_iter', + help='Number of inference iterations', + default=1, + type=int, + dest='number_iter') + parser.add_argument('--raw_output', + help='Raw output without logs', + default=False, + type=bool, + dest='raw_output') + parser.add_argument('-d', '--device', + help='Specify the target device to infer on CPU or GPU (CPU by default)', + default='CPU', + type=str, + dest='device') + + args = parser.parse_args() + + return args + + +def get_device_to_infer(device): + log.info('Get device for inference') + if device == 'CPU': + log.info(f'Inference will be executed on {device}') + return mxnet.cpu() + elif device == 'GPU': + log.info(f'Inference will be executed on {device}') + return mxnet.gpu() + else: + log.info(f'The device {device} is not supported') + raise ValueError('The device is not supported') + + +def load_network(model_json, model_params, context): + raise ValueError('Function is not implemented') + + +def load_network_gluon(model_name, context, input_name, input_shape): + log.info(f'Loading network \"{model_name}\"') + net = model_zoo.get_model(model_name, pretrained=True, ctx=context) + log.info(f'Info about the network:\n{net}') + + log.info('Hybridize model to accelerate inference') + net.hybridize() + + return net + +def create_dict_for_transformer(args): + dictionary = { + 'channel_swap': args.channel_swap, + 'mean': args.mean, + 'std': args.std, + 'norm': args.norm, + 'input_shape': args.input_shape, + 'batch_size': args.batch_size, + } + return dictionary + +def create_dict_for_modelwrapper(args): + dictionary = { + 'input_name': args.input_name, + 'input_shape': args.input_shape, + } + return dictionary + +def print_topk_predictions(predictions, k): + mxnet.test_utils.download('https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/doc/tutorials/onnx/image_net_labels.json') + categories = np.array(json.load(open('image_net_labels.json', 'r'))) + top_pred = predictions.topk(k=k)[0].asnumpy() + for index in top_pred: + probability = predictions[0][int(index)] + category = categories[int(index)] + print("{}: {:.2f}%".format(category, probability.asscalar()*100)) + +def inference_mxnet(net, num_iterations, get_slice, input_name): + predictions = None + time_infer = [] + slice_input = None + if num_iterations == 1: + slice_input = get_slice(0) + t0 = time() + predictions = net(slice_input[input_name]).softmax() + t1 = time() + print_topk_predictions(predictions, 5) + time_infer.append(t1 - t0) + else: + for i in range(num_iterations): + slice_input = get_slice(i) + t0 = time() + net(slice_input[input_name]).softmax() + t1 = time() + time_infer.append(t1 - t0) + + return predictions, time_infer + +def process_result(batch_size, inference_time): + inference_time = pp.three_sigma_rule(inference_time) + average_time = pp.calculate_average_time(inference_time) + latency = pp.calculate_latency(inference_time) + fps = pp.calculate_fps(batch_size, latency) + return average_time, latency, fps + +def main(): + log.basicConfig( + format='[ %(levelname)s ] %(message)s', + level=log.INFO, + stream=sys.stdout, + ) + args = cli_argument_parser() + try: + model_wrapper = MXNetIOModelWrapper(create_dict_for_modelwrapper(args)) + data_transformer = MXNetTransformer(create_dict_for_transformer(args)) + io = IOAdapter.get_io_adapter(args, model_wrapper, data_transformer) + + context = get_device_to_infer(args.device) + + if (args.model_name is not None) and \ + (args.model_json is None) and \ + (args.model_params is None): + net = load_network_gluon(args.model_name, context, args.input_name, + args.input_shape) + elif (args.model_json is not None) and (args.model_params is not None): + net = load_network(args.model_json, args.model_params, context) + else: + raise ValueError('Incorrect arguments.') + + log.info('Prepare input data') + io.prepare_input(net, args.input) + + log.info('Start inference') + result, inference_time = inference_mxnet(net, args.number_iter, + io.get_slice_input, args.input_name) + + log.info('Compute performance metrics') + time, latency, fps = process_result(args.batch_size, inference_time) + log.info(f'Performance metrics:\n\tAverage time = {time} s\n\tLatency={latency}\n\tFPS={fps} fps') + except Exception as ex: + log.error(str(ex)) + sys.exit(1) + + +if __name__ == '__main__': + sys.exit(main() or 0) + diff --git a/src/inference/io_model_wrapper.py b/src/inference/io_model_wrapper.py index d06f3c3f7..a5c081f5f 100644 --- a/src/inference/io_model_wrapper.py +++ b/src/inference/io_model_wrapper.py @@ -162,3 +162,20 @@ def get_input_layer_dtype(self, interpreter, layer_name): for input_ in inputs: if layer_name == input_['name']: return input_['dtype'] + +class MXNetIOModelWrapper(IOModelWrapper): + def __init__(self, args): + self._input_names = [args['input_name']] + self._input_shapes = [args['input_shape']] + + def get_input_layer_names(self, model): + #raise ValueError('Method is not implemented') + return self._input_names + + def get_input_layer_shape(self, model, layer_name): + #raise ValueError('Method is not implemented') + return self._input_shapes[0] + + def get_input_layer_dtype(self, model, layer_name): + import numpy as np + return np.float32 diff --git a/src/inference/transformer.py b/src/inference/transformer.py index 96370942f..e9f4a0319 100644 --- a/src/inference/transformer.py +++ b/src/inference/transformer.py @@ -165,3 +165,47 @@ def transform_images(self, images, shape, element_type, input_name): transformed_images = np.zeros(shape=shape, dtype=element_type) transformed_images = self._transform(images, input_name) return transformed_images + +class MXNetTransformer(Transformer): + def __init__(self, converting): + self._converting = converting + + def __set_norm(self, image): + import mxnet + + if self._converting['norm'] is True: + mean = mxnet.nd.array([self._converting['mean'][0], + self._converting['mean'][1], + self._converting['mean'][2]]) + std = mxnet.nd.array([self._converting['std'][0], + self._converting['std'][1], + self._converting['std'][2]]) + normalized_image = mxnet.image.color_normalize( + mxnet.nd.array(image).astype(np.float32)/255, mean=mean, std=std) + return normalized_image + return image + + def __set_channel_swap(self, image): + if self._converting['channel_swap'] is not None: + transposing_form = (self._converting['channel_swap'][0], + self._converting['channel_swap'][1], + self._converting['channel_swap'][2]) + transposed_image = image.transpose(transposing_form) # WxHxC -> CxWXH + return transposed_image + + return image + + def _transform(self, image): + normalized_image = self.__set_norm(image) + transposed_image = self.__set_channel_swap(normalized_image) + return transposed_image + + def transform_images(self, images, shape, element_type, *args): + import mxnet + + b = shape[0] + transformed_images = mxnet.nd.zeros(shape=shape, dtype=element_type) + for i in range(b): + transformed_images[i] = self._transform(images[i]) + return transformed_images + From 0bce56ed40e82d72edb14151a3e46c67d2cc14b5 Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Wed, 1 Feb 2023 06:13:49 +0300 Subject: [PATCH 02/63] Fix loading label file and logging predictions --- src/inference/inference_mxnet.py | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/src/inference/inference_mxnet.py b/src/inference/inference_mxnet.py index 2699dda46..128c7c569 100644 --- a/src/inference/inference_mxnet.py +++ b/src/inference/inference_mxnet.py @@ -79,7 +79,7 @@ def cli_argument_parser(): dest='batch_size') parser.add_argument('-l', '--labels', help='Labels mapping file', - default=None, + default='image_net_labels.json', type=str, dest='labels') parser.add_argument('-nt', '--number_top', @@ -134,11 +134,11 @@ def load_network(model_json, model_params, context): def load_network_gluon(model_name, context, input_name, input_shape): log.info(f'Loading network \"{model_name}\"') net = model_zoo.get_model(model_name, pretrained=True, ctx=context) + log.info(f'Info about the network:\n{net}') log.info('Hybridize model to accelerate inference') net.hybridize() - return net def create_dict_for_transformer(args): @@ -159,16 +159,18 @@ def create_dict_for_modelwrapper(args): } return dictionary -def print_topk_predictions(predictions, k): - mxnet.test_utils.download('https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/doc/tutorials/onnx/image_net_labels.json') - categories = np.array(json.load(open('image_net_labels.json', 'r'))) +def print_topk_predictions(predictions, k, file_labels): + categories = np.array(json.load(open(file_labels, 'r'))) top_pred = predictions.topk(k=k)[0].asnumpy() + log.info(f'Top-{k} predictions:') for index in top_pred: - probability = predictions[0][int(index)] - category = categories[int(index)] - print("{}: {:.2f}%".format(category, probability.asscalar()*100)) + idx = int(index) + probability = predictions[0][idx] + category = categories[idx] + log.info('\t{}: {:.2f}%'.format(category, probability.asscalar()*100)) -def inference_mxnet(net, num_iterations, get_slice, input_name): +def inference_mxnet(net, num_iterations, get_slice, input_name, + k=5, file_labels='image_net_labels.json'): predictions = None time_infer = [] slice_input = None @@ -177,7 +179,7 @@ def inference_mxnet(net, num_iterations, get_slice, input_name): t0 = time() predictions = net(slice_input[input_name]).softmax() t1 = time() - print_topk_predictions(predictions, 5) + print_topk_predictions(predictions, k, file_labels) time_infer.append(t1 - t0) else: for i in range(num_iterations): @@ -229,7 +231,11 @@ def main(): log.info('Compute performance metrics') time, latency, fps = process_result(args.batch_size, inference_time) - log.info(f'Performance metrics:\n\tAverage time = {time} s\n\tLatency={latency}\n\tFPS={fps} fps') + + log.info('Performance metrics:') + log.info('\tAverage time = {:.4f} s'.format(time)) + log.info('\tLatency={:.4f}'. format(latency)) + log.info('\tFPS={:.4f} fps'.format(fps)) except Exception as ex: log.error(str(ex)) sys.exit(1) From 67f1c94e51557aa76749c583014f746fe7121d13 Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Thu, 2 Feb 2023 15:36:48 +0300 Subject: [PATCH 03/63] Code cleanings --- src/inference/inference_mxnet.py | 62 +++++++++++++++++++------------ src/inference/io_model_wrapper.py | 4 +- src/inference/transformer.py | 8 ++-- 3 files changed, 44 insertions(+), 30 deletions(-) diff --git a/src/inference/inference_mxnet.py b/src/inference/inference_mxnet.py index 128c7c569..f90d63b67 100644 --- a/src/inference/inference_mxnet.py +++ b/src/inference/inference_mxnet.py @@ -42,9 +42,10 @@ def cli_argument_parser(): type=str, dest='input_name') parser.add_argument('-is', '--input_shape', - help='Input shape WxHxC, W is an input tensor width, \ - H is an input tensor height, C is an input tensor \ - number of channnels', + help='Input shape BxWxHxC, B is a batch size,\ + W is an input tensor width, \ + H is an input tensor height, \ + C is an input tensor number of channels', default=None, type=int, nargs=4, @@ -67,7 +68,7 @@ def cli_argument_parser(): type=bool, dest='norm') parser.add_argument('--channel_swap', - help='Parameter channel swap', + help='Parameter channel swap (WxHxC to CxWxH by default)', default=[2, 0, 1], type=int, nargs=3, @@ -84,7 +85,7 @@ def cli_argument_parser(): dest='labels') parser.add_argument('-nt', '--number_top', help='Number of top results', - default=10, + default=5, type=int, dest='number_top') parser.add_argument('-t', '--task', @@ -137,7 +138,7 @@ def load_network_gluon(model_name, context, input_name, input_shape): log.info(f'Info about the network:\n{net}') - log.info('Hybridize model to accelerate inference') + log.info('Hybridizing model to accelerate inference') net.hybridize() return net @@ -161,13 +162,15 @@ def create_dict_for_modelwrapper(args): def print_topk_predictions(predictions, k, file_labels): categories = np.array(json.load(open(file_labels, 'r'))) - top_pred = predictions.topk(k=k)[0].asnumpy() - log.info(f'Top-{k} predictions:') - for index in top_pred: - idx = int(index) - probability = predictions[0][idx] - category = categories[idx] - log.info('\t{}: {:.2f}%'.format(category, probability.asscalar()*100)) + log.info(f'Top-{k} results:') + for prediction_idx in range(len(predictions)): + log.info(f'Result for image {prediction_idx}') + top_pred = predictions.topk(k=k)[prediction_idx].asnumpy() + for index in top_pred: + idx = int(index) + probability = predictions[prediction_idx][idx] + category = categories[idx] + log.info('\t{:.7f} {}'.format(probability.asscalar(), category)) def inference_mxnet(net, num_iterations, get_slice, input_name, k=5, file_labels='image_net_labels.json'): @@ -179,7 +182,6 @@ def inference_mxnet(net, num_iterations, get_slice, input_name, t0 = time() predictions = net(slice_input[input_name]).softmax() t1 = time() - print_topk_predictions(predictions, k, file_labels) time_infer.append(t1 - t0) else: for i in range(num_iterations): @@ -198,6 +200,15 @@ def process_result(batch_size, inference_time): fps = pp.calculate_fps(batch_size, latency) return average_time, latency, fps +def result_output(average_time, fps, latency): + log.info('Average time of single pass : {0:.3f}'.format(average_time)) + log.info('FPS : {0:.3f}'.format(fps)) + log.info('Latency : {0:.3f}'.format(latency)) + +def raw_result_output(average_time, fps, latency): + print('{0:.3f},{1:.3f},{2:.3f}'.format(average_time, fps, latency)) + + def main(): log.basicConfig( format='[ %(levelname)s ] %(message)s', @@ -222,20 +233,23 @@ def main(): else: raise ValueError('Incorrect arguments.') - log.info('Prepare input data') + log.info('Preparing input data') io.prepare_input(net, args.input) - log.info('Start inference') + log.info(f'Starting inference ({args.number_iter} iterations) on {args.device}') result, inference_time = inference_mxnet(net, args.number_iter, io.get_slice_input, args.input_name) - - log.info('Compute performance metrics') - time, latency, fps = process_result(args.batch_size, inference_time) - - log.info('Performance metrics:') - log.info('\tAverage time = {:.4f} s'.format(time)) - log.info('\tLatency={:.4f}'. format(latency)) - log.info('\tFPS={:.4f} fps'.format(fps)) + + log.info('Computing performance metrics') + average_time, latency, fps = process_result(args.batch_size, inference_time) + + if not args.raw_output: + # print_topk_predictions should be implemented as io.process_output(result, log) + print_topk_predictions(result, args.number_top, args.labels) + result_output(average_time, fps, latency) + else: + raw_result_output(average_time, fps, latency) + except Exception as ex: log.error(str(ex)) sys.exit(1) diff --git a/src/inference/io_model_wrapper.py b/src/inference/io_model_wrapper.py index a5c081f5f..8f08cb759 100644 --- a/src/inference/io_model_wrapper.py +++ b/src/inference/io_model_wrapper.py @@ -163,17 +163,17 @@ def get_input_layer_dtype(self, interpreter, layer_name): if layer_name == input_['name']: return input_['dtype'] + class MXNetIOModelWrapper(IOModelWrapper): def __init__(self, args): + # model wrapper supports only one input (batch of images) self._input_names = [args['input_name']] self._input_shapes = [args['input_shape']] def get_input_layer_names(self, model): - #raise ValueError('Method is not implemented') return self._input_names def get_input_layer_shape(self, model, layer_name): - #raise ValueError('Method is not implemented') return self._input_shapes[0] def get_input_layer_dtype(self, model, layer_name): diff --git a/src/inference/transformer.py b/src/inference/transformer.py index e9f4a0319..0f887c53d 100644 --- a/src/inference/transformer.py +++ b/src/inference/transformer.py @@ -166,6 +166,7 @@ def transform_images(self, images, shape, element_type, input_name): transformed_images = self._transform(images, input_name) return transformed_images + class MXNetTransformer(Transformer): def __init__(self, converting): self._converting = converting @@ -190,9 +191,8 @@ def __set_channel_swap(self, image): transposing_form = (self._converting['channel_swap'][0], self._converting['channel_swap'][1], self._converting['channel_swap'][2]) - transposed_image = image.transpose(transposing_form) # WxHxC -> CxWXH + transposed_image = image.transpose(transposing_form) return transposed_image - return image def _transform(self, image): @@ -203,9 +203,9 @@ def _transform(self, image): def transform_images(self, images, shape, element_type, *args): import mxnet - b = shape[0] + batch_size = shape[0] transformed_images = mxnet.nd.zeros(shape=shape, dtype=element_type) - for i in range(b): + for i in range(batch_size): transformed_images[i] = self._transform(images[i]) return transformed_images From f55f91a25be71404304699434db56eba6193aa7c Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Thu, 2 Feb 2023 15:51:06 +0300 Subject: [PATCH 04/63] Fix formatting --- src/inference/inference_mxnet.py | 13 +++++++++---- src/inference/transformer.py | 3 +-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/inference/inference_mxnet.py b/src/inference/inference_mxnet.py index f90d63b67..1e35f7b76 100644 --- a/src/inference/inference_mxnet.py +++ b/src/inference/inference_mxnet.py @@ -2,8 +2,8 @@ import logging as log import sys from time import time -import numpy as np import json +import numpy as np import mxnet import mxnet.gluon.model_zoo.vision as model_zoo @@ -137,11 +137,12 @@ def load_network_gluon(model_name, context, input_name, input_shape): net = model_zoo.get_model(model_name, pretrained=True, ctx=context) log.info(f'Info about the network:\n{net}') - + log.info('Hybridizing model to accelerate inference') net.hybridize() return net + def create_dict_for_transformer(args): dictionary = { 'channel_swap': args.channel_swap, @@ -153,6 +154,7 @@ def create_dict_for_transformer(args): } return dictionary + def create_dict_for_modelwrapper(args): dictionary = { 'input_name': args.input_name, @@ -160,6 +162,7 @@ def create_dict_for_modelwrapper(args): } return dictionary + def print_topk_predictions(predictions, k, file_labels): categories = np.array(json.load(open(file_labels, 'r'))) log.info(f'Top-{k} results:') @@ -172,6 +175,7 @@ def print_topk_predictions(predictions, k, file_labels): category = categories[idx] log.info('\t{:.7f} {}'.format(probability.asscalar(), category)) + def inference_mxnet(net, num_iterations, get_slice, input_name, k=5, file_labels='image_net_labels.json'): predictions = None @@ -193,6 +197,7 @@ def inference_mxnet(net, num_iterations, get_slice, input_name, return predictions, time_infer + def process_result(batch_size, inference_time): inference_time = pp.three_sigma_rule(inference_time) average_time = pp.calculate_average_time(inference_time) @@ -200,11 +205,13 @@ def process_result(batch_size, inference_time): fps = pp.calculate_fps(batch_size, latency) return average_time, latency, fps + def result_output(average_time, fps, latency): log.info('Average time of single pass : {0:.3f}'.format(average_time)) log.info('FPS : {0:.3f}'.format(fps)) log.info('Latency : {0:.3f}'.format(latency)) + def raw_result_output(average_time, fps, latency): print('{0:.3f},{1:.3f},{2:.3f}'.format(average_time, fps, latency)) @@ -249,7 +256,6 @@ def main(): result_output(average_time, fps, latency) else: raw_result_output(average_time, fps, latency) - except Exception as ex: log.error(str(ex)) sys.exit(1) @@ -257,4 +263,3 @@ def main(): if __name__ == '__main__': sys.exit(main() or 0) - diff --git a/src/inference/transformer.py b/src/inference/transformer.py index 0f887c53d..b18b35c06 100644 --- a/src/inference/transformer.py +++ b/src/inference/transformer.py @@ -182,7 +182,7 @@ def __set_norm(self, image): self._converting['std'][1], self._converting['std'][2]]) normalized_image = mxnet.image.color_normalize( - mxnet.nd.array(image).astype(np.float32)/255, mean=mean, std=std) + mxnet.nd.array(image).astype(np.float32) / 255, mean=mean, std=std) return normalized_image return image @@ -208,4 +208,3 @@ def transform_images(self, images, shape, element_type, *args): for i in range(batch_size): transformed_images[i] = self._transform(images[i]) return transformed_images - From 3587f4df179c0295c291003d6d7bd1fc04599252 Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Sun, 5 Feb 2023 15:25:46 +0300 Subject: [PATCH 05/63] Inference implementation using import Gluon (model should be saved in json and params using exports) --- src/inference/inference_mxnet.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/src/inference/inference_mxnet.py b/src/inference/inference_mxnet.py index 1e35f7b76..4379f3153 100644 --- a/src/inference/inference_mxnet.py +++ b/src/inference/inference_mxnet.py @@ -7,6 +7,7 @@ import mxnet import mxnet.gluon.model_zoo.vision as model_zoo +import warnings import postprocessing_data as pp from io_adapter import IOAdapter @@ -28,7 +29,6 @@ def cli_argument_parser(): parser.add_argument('-n', '--model_name', help='Model name to download using Gluon package.', type=str, - required=True, dest='model_name') parser.add_argument('-i', '--input', help='Path to data', @@ -128,11 +128,16 @@ def get_device_to_infer(device): raise ValueError('The device is not supported') -def load_network(model_json, model_params, context): - raise ValueError('Function is not implemented') +def load_network_gluon(model_json, model_params, context, input_name): + log.info(f'Deserializing network from file ({model_json}, {model_params})') + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + deserialized_net = mxnet.gluon.nn.SymbolBlock.imports( + model_json, [input_name], model_params, ctx=context) + return deserialized_net -def load_network_gluon(model_name, context, input_name, input_shape): +def load_network_gluon_model_zoo(model_name, context, input_name, input_shape): log.info(f'Loading network \"{model_name}\"') net = model_zoo.get_model(model_name, pretrained=True, ctx=context) @@ -233,10 +238,11 @@ def main(): if (args.model_name is not None) and \ (args.model_json is None) and \ (args.model_params is None): - net = load_network_gluon(args.model_name, context, args.input_name, - args.input_shape) + net = load_network_gluon_model_zoo(args.model_name, context, args.input_name, + args.input_shape) elif (args.model_json is not None) and (args.model_params is not None): - net = load_network(args.model_json, args.model_params, context) + net = load_network_gluon(args.model_json, args.model_params, context, + args.input_name) else: raise ValueError('Incorrect arguments.') From 545103352bfbda1b0c19b7b9b88d18fb2b32b776 Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Sun, 5 Feb 2023 15:59:49 +0300 Subject: [PATCH 06/63] Validation table for MXNet (Gluon Model zoo) --- ...validation_results_mxnet_gluon_modelzoo.md | 151 ++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100644 results/validation/validation_results_mxnet_gluon_modelzoo.md diff --git a/results/validation/validation_results_mxnet_gluon_modelzoo.md b/results/validation/validation_results_mxnet_gluon_modelzoo.md new file mode 100644 index 000000000..ff4c51a23 --- /dev/null +++ b/results/validation/validation_results_mxnet_gluon_modelzoo.md @@ -0,0 +1,151 @@ +# Validation results for the models inferring using MXNet (Gluon API) + +## Image classification + +### Test image #1 + +Data source: [ImageNet][imagenet] + +Image resolution: 709 x 510 + + +
+ +
+ + Model | Python (implementation) | +-------------|---------------------------| +alexnet |0.4499783 Granny Smith
0.0933101 dumbbell
0.0876728 ocarina, sweet potato
0.0628702 hair slide
0.0484683 bottlecap| +densenet121 |0.9523346 Granny Smith
0.0132273 orange
0.0125171 lemon
0.0027910 banana
0.0020333 piggy bank, penny bank| +densenet161 || +densenet169 || +densenet201 || +inceptionv3 || +mobilenet0.25|0.3314433 piggy bank, penny bank
0.1333785 maraca
0.1262991 croquet ball
0.0684097 dumbbell
0.0539143 hair slide| +mobilenet0.5 || +mobilenet0.75|| +mobilenet1.0 || +mobilenetv2_0.25|| +mobilenetv2_0.5 || +mobilenetv2_0.75|| +mobilenetv2_1.0 || +resnet101_v1 || +resnet101_v2 || +resnet152_v1 || +resnet152_v2 || +resnet18_v1 |0.7145673 Granny Smith
0.0433349 piggy bank, penny bank
0.0343972 saltshaker, salt shaker
0.0215941 fig
0.0212160 banana| +resnet18_v2 || +resnet34_v1 || +resnet34_v2 || +resnet50_v1 || +resnet50_v2 || +squeezenet1.0 || +squeezenet1.1 || +vgg11 || +vgg11_bn || +vgg13 || +vgg13_bn || +vgg16 || +vgg16_bn || +vgg19 || +vgg19_bn || + +### Test image #2 + +Data source: [ImageNet][imagenet] + +Image resolution: 500 x 500 + + +
+ +
+ + Model | Python (implementation) | +---------------------|---------------------------| +alexnet |0.9947656 junco, snowbird
0.0043087 chickadee
0.0002780 water ouzel, dipper
0.0002770 bulbul
0.0001244 brambling, Fringilla montifringilla| +densenet121 |0.9841659 junco, snowbird
0.0072199 chickadee
0.0034963 brambling, Fringilla montifringilla
0.0016226 water ouzel, dipper
0.0012858 indigo bunting, indigo finch, indigo bird, Passerina cyanea| +densenet161 || +densenet169 || +densenet201 || +inceptionv3 || +mobilenet0.25|0.9301481 junco, snowbird
0.0466449 chickadee
0.0146190 brambling, Fringilla montifringilla
0.0027491 bulbul
0.0024721 jay| +mobilenet0.5 || +mobilenet0.75|| +mobilenet1.0 || +mobilenetv2_0.25|| +mobilenetv2_0.5 || +mobilenetv2_0.75|| +mobilenetv2_1.0 || +resnet101_v1 || +resnet101_v2 || +resnet152_v1 || +resnet152_v2 || +resnet18_v1 |0.9597615 junco, snowbird
0.0103962 chickadee
0.0075481 goldfinch, Carduelis carduelis
0.0054580 house finch, linnet, Carpodacus mexicanus
0.0053979 water ouzel, dipper| +resnet18_v2 || +resnet34_v1 || +resnet34_v2 || +resnet50_v1 || +resnet50_v2 || +squeezenet1.0 || +squeezenet1.1 || +vgg11 || +vgg11_bn || +vgg13 || +vgg13_bn || +vgg16 || +vgg16_bn || +vgg19 || +vgg19_bn || + +### Test image #3 + +Data source: [ImageNet][imagenet] + +Image resolution: 333 x 500 + + +
+ +
+ + Model | Python (implementation) | +---------------------|---------------------------| +alexnet |0.3216891 container ship, containership, container vessel
0.1360617 drilling platform, offshore rig
0.1140692 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.1057475 beacon, lighthouse, beacon light, pharos
0.0471224 liner, ocean liner| +densenet121 |0.3022416 liner, ocean liner
0.1322481 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.1194608 container ship, containership, container vessel
0.0795039 drilling platform, offshore rig
0.0723068 dock, dockage, docking facility| +densenet161 || +densenet169 || +densenet201 || +inceptionv3 || +mobilenet0.25|0.2958905 container ship, containership, container vessel
0.2101003 drilling platform, offshore rig
0.1384616 submarine, pigboat, sub, U-boat
0.0863535 liner, ocean liner
0.0732720 beacon, lighthouse, beacon light, pharos| +mobilenet0.5 || +mobilenet0.75|| +mobilenet1.0 || +mobilenetv2_0.25|| +mobilenetv2_0.5 || +mobilenetv2_0.75|| +mobilenetv2_1.0 || +resnet101_v1 || +resnet101_v2 || +resnet152_v1 || +resnet152_v2 || +resnet18_v1 |0.3416696 container ship, containership, container vessel
0.1224417 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.1104408 liner, ocean liner
0.0661492 lifeboat
0.0649565 pirate, pirate ship| +resnet18_v2 || +resnet34_v1 || +resnet34_v2 || +resnet50_v1 || +resnet50_v2 || +squeezenet1.0 || +squeezenet1.1 || +vgg11 || +vgg11_bn || +vgg13 || +vgg13_bn || +vgg16 || +vgg16_bn || +vgg19 || +vgg19_bn || + + + +[imagenet]: http://www.image-net.org From a902651729179f4a55441beccee8fd758cfada04 Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Mon, 6 Feb 2023 16:18:57 +0300 Subject: [PATCH 07/63] Update list of supported MXNet models + readme for inference script --- ...validation_results_mxnet_gluon_modelzoo.md | 181 +++++++++--------- src/inference/README.md | 40 ++++ 2 files changed, 134 insertions(+), 87 deletions(-) diff --git a/results/validation/validation_results_mxnet_gluon_modelzoo.md b/results/validation/validation_results_mxnet_gluon_modelzoo.md index ff4c51a23..e8d01f2f4 100644 --- a/results/validation/validation_results_mxnet_gluon_modelzoo.md +++ b/results/validation/validation_results_mxnet_gluon_modelzoo.md @@ -2,11 +2,17 @@ ## Image classification +Complete information about the supported classification +models is available [here][gluon_modelzoo_classification]. + ### Test image #1 Data source: [ImageNet][imagenet] Image resolution: 709 x 510 + +Mean: [0.485, 0.456, 0.406] +Standard deviation: [0.229, 0.224, 0.225] 
@@ -17,38 +23,38 @@ Image resolution: 709 x 510 -------------|---------------------------| alexnet |0.4499783 Granny Smith
0.0933101 dumbbell
0.0876728 ocarina, sweet potato
0.0628702 hair slide
0.0484683 bottlecap| densenet121 |0.9523346 Granny Smith
0.0132273 orange
0.0125171 lemon
0.0027910 banana
0.0020333 piggy bank, penny bank| -densenet161 || -densenet169 || -densenet201 || +densenet161 |0.9372969 Granny Smith
0.0082274 dumbbell
0.0056475 piggy bank, penny bank
0.0055374 ping-pong ball
0.0041915 pitcher, ewer| +densenet169 |0.9811633 Granny Smith
0.0033828 piggy bank, penny bank
0.0021365 orange
0.0019196 lemon
0.0017232 pomegranate| +densenet201 |0.9119796 Granny Smith
0.0533456 piggy bank, penny bank
0.0056831 lemon
0.0017810 pool table, billiard table, snooker table
0.0015689 tennis ball| inceptionv3 || mobilenet0.25|0.3314433 piggy bank, penny bank
0.1333785 maraca
0.1262991 croquet ball
0.0684097 dumbbell
0.0539143 hair slide| -mobilenet0.5 || -mobilenet0.75|| -mobilenet1.0 || -mobilenetv2_0.25|| -mobilenetv2_0.5 || -mobilenetv2_0.75|| -mobilenetv2_1.0 || -resnet101_v1 || -resnet101_v2 || -resnet152_v1 || -resnet152_v2 || +mobilenet0.5 |0.7425038 piggy bank, penny bank
0.0554336 saltshaker, salt shaker
0.0353432 Granny Smith
0.0217170 pencil sharpener
0.0169298 rubber eraser, rubber, pencil eraser| +mobilenet0.75|0.2680034 piggy bank, penny bank
0.2459760 Granny Smith
0.0562553 dumbbell
0.0306232 necklace
0.0305400 pitcher, ewer| +mobilenet1.0 |0.6145398 Granny Smith
0.0852871 piggy bank, penny bank
0.0347396 maraca
0.0332023 necklace
0.0145589 whistle| +mobilenetv2_0.25|0.1190805 bell pepper
0.1169647 Granny Smith
0.1004419 candle, taper, wax light
0.0989920 dumbbell
0.0918921 hair slide| +mobilenetv2_0.5 |0.2223127 Granny Smith
0.2116509 piggy bank, penny bank
0.1656679 saltshaker, salt shaker
0.0511496 teapot
0.0504719 necklace| +mobilenetv2_0.75|0.6253413 Granny Smith
0.0585538 piggy bank, penny bank
0.0260610 necklace
0.0252774 bell pepper
0.0218010 hair slide| +mobilenetv2_1.0 |0.7025163 piggy bank, penny bank
0.1278343 Granny Smith
0.0542883 hair slide
0.0226695 necklace
0.0095436 saltshaker, salt shaker| resnet18_v1 |0.7145673 Granny Smith
0.0433349 piggy bank, penny bank
0.0343972 saltshaker, salt shaker
0.0215941 fig
0.0212160 banana| -resnet18_v2 || -resnet34_v1 || -resnet34_v2 || -resnet50_v1 || -resnet50_v2 || -squeezenet1.0 || -squeezenet1.1 || -vgg11 || -vgg11_bn || -vgg13 || -vgg13_bn || -vgg16 || -vgg16_bn || -vgg19 || -vgg19_bn || +resnet18_v2 |0.2885317 Granny Smith
0.1816196 piggy bank, penny bank
0.0722676 saltshaker, salt shaker
0.0635363 rubber eraser, rubber, pencil eraser
0.0440724 soap dispenser| +resnet34_v1 |0.5898089 piggy bank, penny bank
0.3150511 Granny Smith
0.0128028 saltshaker, salt shaker
0.0093140 candle, taper, wax light
0.0089791 pencil sharpener| +resnet34_v2 |0.5082700 Granny Smith
0.3873705 piggy bank, penny bank
0.0163602 pencil sharpener
0.0137499 saltshaker, salt shaker
0.0071418 dumbbell| +resnet50_v1 |0.7435529 Granny Smith
0.0182989 orange
0.0153789 lemon
0.0104132 vase
0.0097732 pop bottle, soda bottle| +resnet50_v2 |0.9931256 Granny Smith
0.0017001 piggy bank, penny bank
0.0007180 saltshaker, salt shaker
0.0006648 dumbbell
0.0002998 tennis ball| +resnet101_v1 |0.8556019 Granny Smith
0.0572227 piggy bank, penny bank
0.0485176 saltshaker, salt shaker
0.0053789 hair slide
0.0048319 dumbbell| +resnet101_v2 |0.8972577 Granny Smith
0.0401716 candle, taper, wax light
0.0074955 nail
0.0072345 screw
0.0056374 hair slide| +resnet152_v1 |0.9127905 Granny Smith
0.0197971 croquet ball
0.0119885 piggy bank, penny bank
0.0078466 saltshaker, salt shaker
0.0043091 analog clock| +resnet152_v2 |0.9972480 Granny Smith
0.0010769 piggy bank, penny bank
0.0002827 orange
0.0002033 pitcher, ewer
0.0001509 lemon| +squeezenet1.0 |0.3275063 piggy bank, penny bank
0.1791327 dumbbell
0.1542634 Granny Smith
0.0912991 water bottle
0.0385819 rubber eraser, rubber, pencil eraser| +squeezenet1.1 |0.5895361 piggy bank, penny bank
0.0677938 Granny Smith
0.0610649 necklace
0.0610449 lemon
0.0490913 bucket, pail| +vgg11 |0.3721453 piggy bank, penny bank
0.2952037 Granny Smith
0.1076759 tennis ball
0.0314685 soap dispenser
0.0285692 dumbbell| +vgg11_bn |0.5464050 Granny Smith
0.2313121 dumbbell
0.0658234 piggy bank, penny bank
0.0269569 tennis ball
0.0218533 teapot| +vgg13 |0.4068239 Granny Smith
0.2272185 dumbbell
0.0475026 necklace
0.0303710 maraca
0.0250665 teapot| +vgg13_bn |0.9389396 Granny Smith
0.0383620 tennis ball
0.0069443 lemon
0.0039320 orange
0.0013574 banana| +vgg16 |0.6872565 piggy bank, penny bank
0.0687328 Granny Smith
0.0588234 teapot
0.0392139 tennis ball
0.0210059 pitcher, ewer| +vgg16_bn |0.9958423 Granny Smith
0.0010665 tennis ball
0.0008365 piggy bank, penny bank
0.0004831 teapot
0.0004004 dumbbell| +vgg19 |0.6034814 Granny Smith
0.1175480 piggy bank, penny bank
0.0277911 dumbbell
0.0249205 whistle
0.0218847 teapot| +vgg19_bn |0.9881877 Granny Smith
0.0025413 piggy bank, penny bank
0.0011374 teapot
0.0009895 saltshaker, salt shaker
0.0006477 cup| ### Test image #2 @@ -65,38 +71,38 @@ Image resolution: 500 x 500 ---------------------|---------------------------| alexnet |0.9947656 junco, snowbird
0.0043087 chickadee
0.0002780 water ouzel, dipper
0.0002770 bulbul
0.0001244 brambling, Fringilla montifringilla| densenet121 |0.9841659 junco, snowbird
0.0072199 chickadee
0.0034963 brambling, Fringilla montifringilla
0.0016226 water ouzel, dipper
0.0012858 indigo bunting, indigo finch, indigo bird, Passerina cyanea| -densenet161 || -densenet169 || -densenet201 || +densenet161 |0.9932107 junco, snowbird
0.0015922 chickadee
0.0012295 brambling, Fringilla montifringilla
0.0011838 indigo bunting, indigo finch, indigo bird, Passerina cyanea
0.0008891 goldfinch, Carduelis carduelis| +densenet169 |0.9640683 junco, snowbird
0.0201314 brambling, Fringilla montifringilla
0.0044098 chickadee
0.0032345 goldfinch, Carduelis carduelis
0.0026739 water ouzel, dipper| +densenet201 |0.9515268 junco, snowbird
0.0178252 water ouzel, dipper
0.0109119 brambling, Fringilla montifringilla
0.0077980 house finch, linnet, Carpodacus mexicanus
0.0044695 chickadee| inceptionv3 || mobilenet0.25|0.9301481 junco, snowbird
0.0466449 chickadee
0.0146190 brambling, Fringilla montifringilla
0.0027491 bulbul
0.0024721 jay| -mobilenet0.5 || -mobilenet0.75|| -mobilenet1.0 || -mobilenetv2_0.25|| -mobilenetv2_0.5 || -mobilenetv2_0.75|| -mobilenetv2_1.0 || -resnet101_v1 || -resnet101_v2 || -resnet152_v1 || -resnet152_v2 || +mobilenet0.5 |0.9290579 junco, snowbird
0.0325394 chickadee
0.0142417 water ouzel, dipper
0.0070296 brambling, Fringilla montifringilla
0.0054897 house finch, linnet, Carpodacus mexicanus| +mobilenet0.75|0.9554648 junco, snowbird
0.0176056 house finch, linnet, Carpodacus mexicanus
0.0163125 brambling, Fringilla montifringilla
0.0040996 chickadee
0.0029940 goldfinch, Carduelis carduelis| +mobilenet1.0 |0.9746482 junco, snowbird
0.0124388 chickadee
0.0072107 brambling, Fringilla montifringilla
0.0013413 goldfinch, Carduelis carduelis
0.0011567 house finch, linnet, Carpodacus mexicanus| +mobilenetv2_0.25|0.9198207 junco, snowbird
0.0456813 chickadee
0.0230144 house finch, linnet, Carpodacus mexicanus
0.0067220 brambling, Fringilla montifringilla
0.0012408 bulbul| +mobilenetv2_0.5 |0.9930903 junco, snowbird
0.0035467 chickadee
0.0017623 brambling, Fringilla montifringilla
0.0013532 house finch, linnet, Carpodacus mexicanus
0.0000977 goldfinch, Carduelis carduelis| +mobilenetv2_0.75|0.9800993 junco, snowbird
0.0092876 chickadee
0.0044733 brambling, Fringilla montifringilla
0.0028389 house finch, linnet, Carpodacus mexicanus
0.0017407 water ouzel, dipper| +mobilenetv2_1.0 |0.9759791 junco, snowbird
0.0164687 chickadee
0.0048956 brambling, Fringilla montifringilla
0.0009601 house finch, linnet, Carpodacus mexicanus
0.0008552 water ouzel, dipper| resnet18_v1 |0.9597615 junco, snowbird
0.0103962 chickadee
0.0075481 goldfinch, Carduelis carduelis
0.0054580 house finch, linnet, Carpodacus mexicanus
0.0053979 water ouzel, dipper| -resnet18_v2 || -resnet34_v1 || -resnet34_v2 || -resnet50_v1 || -resnet50_v2 || -squeezenet1.0 || -squeezenet1.1 || -vgg11 || -vgg11_bn || -vgg13 || -vgg13_bn || -vgg16 || -vgg16_bn || -vgg19 || -vgg19_bn || +resnet18_v2 |0.9460657 junco, snowbird
0.0180776 brambling, Fringilla montifringilla
0.0139834 chickadee
0.0075304 goldfinch, Carduelis carduelis
0.0037932 water ouzel, dipper| +resnet34_v1 |0.9352033 junco, snowbird
0.0226503 water ouzel, dipper
0.0129960 brambling, Fringilla montifringilla
0.0050841 chickadee
0.0037434 goldfinch, Carduelis carduelis| +resnet34_v2 |0.6477917 junco, snowbird
0.0750232 water ouzel, dipper
0.0672592 brambling, Fringilla montifringilla
0.0443261 chickadee
0.0321975 goldfinch, Carduelis carduelis| +resnet50_v1 |0.9883676 junco, snowbird
0.0037137 brambling, Fringilla montifringilla
0.0027347 water ouzel, dipper
0.0025455 chickadee
0.0007354 goldfinch, Carduelis carduelis| +resnet50_v2 |0.9820375 junco, snowbird
0.0070083 water ouzel, dipper
0.0032555 chickadee
0.0021856 indigo bunting, indigo finch, indigo bird, Passerina cyanea
0.0017496 brambling, Fringilla montifringilla| +resnet101_v1 |0.9215411 junco, snowbird
0.0161462 brambling, Fringilla montifringilla
0.0113414 water ouzel, dipper
0.0096167 chickadee
0.0087583 house finch, linnet, Carpodacus mexicanus| +resnet101_v2 |0.9053509 junco, snowbird
0.0451527 water ouzel, dipper
0.0106367 chickadee
0.0089986 brambling, Fringilla montifringilla
0.0037001 goldfinch, Carduelis carduelis| +resnet152_v1 |0.9372504 junco, snowbird
0.0259636 water ouzel, dipper
0.0113407 chickadee
0.0088195 brambling, Fringilla montifringilla
0.0031848 house finch, linnet, Carpodacus mexicanus| +resnet152_v2 |0.9695247 junco, snowbird
0.0054735 brambling, Fringilla montifringilla
0.0041750 water ouzel, dipper
0.0027560 goldfinch, Carduelis carduelis
0.0024612 indigo bunting, indigo finch, indigo bird, Passerina cyanea| +squeezenet1.0 |0.9904419 junco, snowbird
0.0045285 chickadee
0.0040343 brambling, Fringilla montifringilla
0.0003414 water ouzel, dipper
0.0002521 house finch, linnet, Carpodacus mexicanus| +squeezenet1.1 |0.9614601 junco, snowbird
0.0250982 chickadee
0.0040701 brambling, Fringilla montifringilla
0.0035157 goldfinch, Carduelis carduelis
0.0030858 ruffed grouse, partridge, Bonasa umbellus| +vgg11 |0.9998955 junco, snowbird
0.0000967 chickadee
0.0000043 brambling, Fringilla montifringilla
0.0000023 water ouzel, dipper
0.0000006 bulbul| +vgg11_bn |0.9994942 junco, snowbird
0.0002460 brambling, Fringilla montifringilla
0.0002328 chickadee
0.0000130 water ouzel, dipper
0.0000100 goldfinch, Carduelis carduelis| +vgg13 |0.9359032 junco, snowbird
0.0610290 chickadee
0.0012531 brambling, Fringilla montifringilla
0.0012155 water ouzel, dipper
0.0002740 bulbul| +vgg13_bn |0.9927477 junco, snowbird
0.0041162 chickadee
0.0028725 brambling, Fringilla montifringilla
0.0000676 goldfinch, Carduelis carduelis
0.0000641 house finch, linnet, Carpodacus mexicanus| +vgg16 |0.8946120 junco, snowbird
0.0953836 chickadee
0.0077338 brambling, Fringilla montifringilla
0.0018954 water ouzel, dipper
0.0001777 bulbul| +vgg16_bn |0.9928787 junco, snowbird
0.0052144 chickadee
0.0011600 brambling, Fringilla montifringilla
0.0005868 water ouzel, dipper
0.0000735 house finch, linnet, Carpodacus mexicanus| +vgg19 |0.9538233 junco, snowbird
0.0420003 chickadee
0.0040804 water ouzel, dipper
0.0000727 brambling, Fringilla montifringilla
0.0000097 bulbul| +vgg19_bn |0.9974002 junco, snowbird
0.0010910 brambling, Fringilla montifringilla
0.0008814 chickadee
0.0004659 water ouzel, dipper
0.0001015 goldfinch, Carduelis carduelis| ### Test image #3 @@ -113,39 +119,40 @@ Image resolution: 333 x 500 ---------------------|---------------------------| alexnet |0.3216891 container ship, containership, container vessel
0.1360617 drilling platform, offshore rig
0.1140692 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.1057475 beacon, lighthouse, beacon light, pharos
0.0471224 liner, ocean liner| densenet121 |0.3022416 liner, ocean liner
0.1322481 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.1194608 container ship, containership, container vessel
0.0795039 drilling platform, offshore rig
0.0723068 dock, dockage, docking facility| -densenet161 || -densenet169 || -densenet201 || +densenet161 |0.4418390 lifeboat
0.1824288 liner, ocean liner
0.0596465 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0325273 submarine, pigboat, sub, U-boat
0.0298845 dock, dockage, docking facility| +densenet169 |0.2955862 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.2342375 drilling platform, offshore rig
0.0940931 liner, ocean liner
0.0876006 container ship, containership, container vessel
0.0717741 dock, dockage, docking facility| +densenet201 |0.5008167 fireboat
0.0950200 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0701643 lifeboat
0.0622607 liner, ocean liner
0.0582344 container ship, containership, container vessel| inceptionv3 || mobilenet0.25|0.2958905 container ship, containership, container vessel
0.2101003 drilling platform, offshore rig
0.1384616 submarine, pigboat, sub, U-boat
0.0863535 liner, ocean liner
0.0732720 beacon, lighthouse, beacon light, pharos| -mobilenet0.5 || -mobilenet0.75|| -mobilenet1.0 || -mobilenetv2_0.25|| -mobilenetv2_0.5 || -mobilenetv2_0.75|| -mobilenetv2_1.0 || -resnet101_v1 || -resnet101_v2 || -resnet152_v1 || -resnet152_v2 || +mobilenet0.5 |0.6135703 liner, ocean liner
0.1116956 container ship, containership, container vessel
0.0976279 submarine, pigboat, sub, U-boat
0.0407747 drilling platform, offshore rig
0.0312130 breakwater, groin, groyne, mole, bulwark, seawall, jetty| +mobilenet0.75|0.1367185 pirate, pirate ship
0.1318326 container ship, containership, container vessel
0.1117856 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0865669 lifeboat
0.0837456 liner, ocean liner| +mobilenet1.0 |0.4250087 pirate, pirate ship
0.1943999 container ship, containership, container vessel
0.0818260 drilling platform, offshore rig
0.0519248 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0360357 dock, dockage, docking facility| +mobilenetv2_0.25|0.4253865 liner, ocean liner
0.2140183 container ship, containership, container vessel
0.0672245 drilling platform, offshore rig
0.0430525 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0406143 aircraft carrier, carrier, flattop, attack aircraft carrier| +mobilenetv2_0.5 |0.1714408 liner, ocean liner
0.1560861 beacon, lighthouse, beacon light, pharos
0.1120990 water bottle
0.0924766 container ship, containership, container vessel
0.0835731 breakwater, groin, groyne, mole, bulwark, seawall, jetty| +mobilenetv2_0.75|0.4084314 container ship, containership, container vessel
0.1701521 liner, ocean liner
0.1168837 drilling platform, offshore rig
0.1098730 beacon, lighthouse, beacon light, pharos
0.0329633 lifeboat| +mobilenetv2_1.0 |0.2700136 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.1883821 liner, ocean liner
0.1121855 drilling platform, offshore rig
0.0574179 aircraft carrier, carrier, flattop, attack aircraft carrier
0.0394433 pirate, pirate ship| resnet18_v1 |0.3416696 container ship, containership, container vessel
0.1224417 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.1104408 liner, ocean liner
0.0661492 lifeboat
0.0649565 pirate, pirate ship| -resnet18_v2 || -resnet34_v1 || -resnet34_v2 || -resnet50_v1 || -resnet50_v2 || -squeezenet1.0 || -squeezenet1.1 || -vgg11 || -vgg11_bn || -vgg13 || -vgg13_bn || -vgg16 || -vgg16_bn || -vgg19 || -vgg19_bn || +resnet18_v2 |0.2872612 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.1359790 beacon, lighthouse, beacon light, pharos
0.1248603 container ship, containership, container vessel
0.1217949 dock, dockage, docking facility
0.0381380 fireboat| +resnet34_v1 |0.5147033 liner, ocean liner
0.0504066 submarine, pigboat, sub, U-boat
0.0452916 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0438951 sandbar, sand bar
0.0422815 fireboat| +resnet34_v2 |0.4267507 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.2477048 container ship, containership, container vessel
0.0763716 liner, ocean liner
0.0696335 dock, dockage, docking facility
0.0447003 beacon, lighthouse, beacon light, pharos| +resnet50_v1 |0.1132128 dock, dockage, docking facility
0.0919173 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0890666 catamaran
0.0592264 seashore, coast, seacoast, sea-coast
0.0560082 container ship, containership, container vessel| +resnet50_v2 |0.4814199 container ship, containership, container vessel
0.1134713 liner, ocean liner
0.0718215 drilling platform, offshore rig
0.0571725 dock, dockage, docking facility
0.0450073 lifeboat| +resnet101_v1 |0.2196452 beacon, lighthouse, beacon light, pharos
0.1348773 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0985295 aircraft carrier, carrier, flattop, attack aircraft carrier
0.0834723 fireboat
0.0616940 drilling platform, offshore rig| +resnet101_v2 |0.2760636 pirate, pirate ship
0.1350918 wreck
0.0808768 liner, ocean liner
0.0636322 drilling platform, offshore rig
0.0605573 breakwater, groin, groyne, mole, bulwark, seawall, jetty| +resnet152_v1 |0.1550222 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.1267105 liner, ocean liner
0.1191449 lifeboat
0.1074122 beacon, lighthouse, beacon light, pharos
0.0826740 pier| +resnet152_v2 |0.1715954 drilling platform, offshore rig
0.1319755 beacon, lighthouse, beacon light, pharos
0.0740683 dock, dockage, docking facility
0.0558942 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0398145 container ship, containership, container vessel| +squeezenet1.0 |0.8105506 liner, ocean liner
0.0785145 drilling platform, offshore rig
0.0295156 container ship, containership, container vessel
0.0153660 dock, dockage, docking facility
0.0115069 submarine, pigboat, sub, U-boat| +squeezenet1.1 |0.4413084 liner, ocean liner
0.1931022 container ship, containership, container vessel
0.1459108 pirate, pirate ship
0.0937747 fireboat
0.0198683 drilling platform, offshore rig| +vgg11 |0.3343849 container ship, containership, container vessel
0.3068860 liner, ocean liner
0.0492899 submarine, pigboat, sub, U-boat
0.0455568 fireboat
0.0391509 lifeboat| +vgg11_bn |0.7272964 container ship, containership, container vessel
0.1716903 liner, ocean liner
0.0226533 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0206520 dock, dockage, docking facility
0.0114507 lifeboat| +vgg13 |0.3224934 container ship, containership, container vessel
0.2891446 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.1808191 liner, ocean liner
0.0591594 beacon, lighthouse, beacon light, pharos
0.0270379 dock, dockage, docking facility| +vgg13_bn |0.3478981 container ship, containership, container vessel
0.2664557 fireboat
0.0766566 lifeboat
0.0664669 liner, ocean liner
0.0515883 submarine, pigboat, sub, U-boat| +vgg16 |0.2839452 liner, ocean liner
0.2079168 fireboat
0.1477822 container ship, containership, container vessel
0.0909363 lifeboat
0.0704186 dock, dockage, docking facility| +vgg16_bn |0.3687426 container ship, containership, container vessel
0.3540941 liner, ocean liner
0.1349645 fireboat
0.0337696 speedboat
0.0263291 submarine, pigboat, sub, U-boat| +vgg19 |0.4434196 liner, ocean liner
0.1207930 container ship, containership, container vessel
0.0979091 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0852779 drilling platform, offshore rig
0.0730739 dock, dockage, docking facility| +vgg19_bn |0.5504534 fireboat
0.1722971 liner, ocean liner
0.0720537 container ship, containership, container vessel
0.0406803 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0388017 drilling platform, offshore rig| [imagenet]: http://www.image-net.org +[gluon_modelzoo_classification]: https://cv.gluon.ai/model_zoo/classification.html diff --git a/src/inference/README.md b/src/inference/README.md index 343bb9ed6..6197d44cb 100644 --- a/src/inference/README.md +++ b/src/inference/README.md @@ -8,6 +8,7 @@ 1. Intel® Optimization for Caffe. 1. Intel® Optimization for TensorFlow. 1. TensorFlow Lite. +1. MXNet. ## Вывод глубоких моделей с использованием Inference Engine @@ -433,3 +434,42 @@ python3 inference_tensorflowlite.py \ -m /.tflite \ -i / ``` + +## Вывод глубоких моделей с использованием MXNet (Gluon API) + +#### Аргументы командной строки + +Название скрипта: + +```bash +inference_mxnet.py +``` + +Обязательные аргументы: + +- `-m / --model` +- `-w / --weights` +- `-mn / --model_name` +- `-i / --input` + +Опциональные аргументы: + +- - `-in / --input_name` +- `-is / --input_shape` +- `--mean` +- `--std` +- `--norm` +- `--channel_swap` +- `-ni / --number_iter` +- `--raw_output` +- `-d / --device` + +#### Примеры запуска + +```bash +python inference_mxnet.py --model_name --input --input_name --input_shape --mean --std --batch_size +``` + +```bash +python inference_mxnet.py --model .json --weights .params --input_name --input_shape --input --labels .json --batch_size +``` From 532f3c727beed55bfcce8a6e170b5322f9bad8bb Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Mon, 6 Feb 2023 16:20:35 +0300 Subject: [PATCH 08/63] Fix line break --- results/validation/validation_results_mxnet_gluon_modelzoo.md | 1 + 1 file changed, 1 insertion(+) diff --git a/results/validation/validation_results_mxnet_gluon_modelzoo.md b/results/validation/validation_results_mxnet_gluon_modelzoo.md index e8d01f2f4..25c2cf069 100644 --- a/results/validation/validation_results_mxnet_gluon_modelzoo.md +++ b/results/validation/validation_results_mxnet_gluon_modelzoo.md @@ -12,6 +12,7 @@ Data source: [ImageNet][imagenet] Image resolution: 709 x 510 Mean: [0.485, 0.456, 0.406] + Standard deviation: [0.229, 0.224, 0.225]  From ef9da5710e400a5275278b7e91aad9e18032f622 Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Tue, 7 Feb 2023 15:20:47 +0300 Subject: [PATCH 09/63] Fix arguments + update inference readme --- src/inference/README.md | 91 ++++++++++++++++++++++++++------ src/inference/inference_mxnet.py | 4 +- 2 files changed, 78 insertions(+), 17 deletions(-) diff --git a/src/inference/README.md b/src/inference/README.md index 6197d44cb..232bfe31f 100644 --- a/src/inference/README.md +++ b/src/inference/README.md @@ -447,29 +447,90 @@ inference_mxnet.py Обязательные аргументы: -- `-m / --model` -- `-w / --weights` -- `-mn / --model_name` -- `-i / --input` +- `-m / --model` - путь до описания архитектуры модели + в формате `.json`, если модель загружается из файла. + Модель должна быть обучена с использованием Gluon API + и экспортирована с помощью метода `export`. +- `-w / --weights` - путь до весов обученной модели, + которые хранятся в файле расширением `.params`, + если модель загружается из файла. Модель должна быть + обучена с использованием Gluon API и экспортирована + с помощью метода `export`. +- `-mn / --model_name` - название модели, если модель + загружается из [Gluon Model Zoo][gluon_modelzoo]. + При таком варианте запуска модель загружается из сети Интернет. +- `-i / --input` - путь до изображения или директории + с изображениями (расширения файлов `.jpg`, `.png`, + `.bmp` и т.д.). +- `-is / --input_shape` - размеры входного тензора сети в формате + BxCxWxH, B - размер пачки, C - количество каналов изображений, + W - ширина изображений, H - высота изображений. Опциональные аргументы: -- - `-in / --input_name` -- `-is / --input_shape` -- `--mean` -- `--std` -- `--norm` -- `--channel_swap` -- `-ni / --number_iter` -- `--raw_output` -- `-d / --device` +- `-t / --task` - название задачи. Текущая реализация поддерживает + решение задачи классификации. По умолчанию принимает значение + `classification`. +- `-in / --input_name` - название входа модели. По умолчанию модель + имеет один вход с названием `data`. Текущая реализация вывода + предусматривает наличие только одного входа. +- `--norm` - флаг необходимости нормировки изображений. + Выполняется с использованием функции ` mxnet.image.color_normalize`. + Среднее и среднеквадратическое отклонение, которые принимаются + на вход указываются в следующих двух аргументах. +- `--mean` - среднее значение интенсивности, которое вычитается + из изображений в процессе нормировки. Для классификационных моделей + из [Gluon Model Zoo][gluon_modelzoo], которые обучены на наборе + данных ImageNet, значение равно `0.485 0.456 0.406`. По умолчанию + данный параметр принимает значение `0 0 0`. +- `--std` - среднеквадратическое отклонение интенсивности, на которое + делится значение интенсивности каждого пикселя входного изображения + в процессе нормировки. Для классификационных моделей + из [Gluon Model Zoo][gluon_modelzoo], которые обучены на наборе + данных ImageNet, значение равно `0.229 0.224 0.225`. По умолчанию + данный параметр принимает значение `1 1 1`. +- `--channel_swap` - порядок перестановки цветовых каналов изображения. + Загрузка изображений осуществляется в формате BGR (порядок + соответствует `(0, 1, 2)`), а большинство нейронных сетей принимают + на вход изображения в формате RGB, поэтому по умолчанию порядок + `(2, 1, 0)`. +- `-d / --device` - оборудование, на котором выполняется вывод сети. + Поддерживается вывод на CPU (значение параметра `CPU`) и NVIDIA GPU + (значение параметра `GPU`). По умолчанию принимает значение `CPU`. +- `-l / --labels`- путь до файла в формате JSON с перечнем меток + при решении задачи. По умолчанию принимает значение + `image_net_labels.json`, что соответствует меткам набора данных + ImageNet. +- `-ni / --number_iter` - количество прямых проходов по сети. + По умолчанию выполняется один проход по сети. +- `--raw_output` - работа скрипта без логов. По умолчанию не установлен. #### Примеры запуска +**Запуск вывода для модели, которая загружается из Gluon Model Zoo** + ```bash -python inference_mxnet.py --model_name --input --input_name --input_shape --mean --std --batch_size +python inference_mxnet.py --model_name \ + --input \ + --input_name \ + --input_shape \ + --mean \ + --std \ + --batch_size ``` +**Запуск вывода для модели, которая загружается из файлов** + ```bash -python inference_mxnet.py --model .json --weights .params --input_name --input_shape --input --labels .json --batch_size +python inference_mxnet.py --model .json \ + --weights .params \ + --input_name \ + --input_shape \ + --input \ + --labels .json \ + --batch_size ``` + + + +[gluon_modelzoo]: https://cv.gluon.ai/model_zoo/index.html diff --git a/src/inference/inference_mxnet.py b/src/inference/inference_mxnet.py index 4379f3153..dafd8a5a9 100644 --- a/src/inference/inference_mxnet.py +++ b/src/inference/inference_mxnet.py @@ -26,7 +26,7 @@ def cli_argument_parser(): help='Path to an .params file with a trained weights.', type=str, dest='model_params') - parser.add_argument('-n', '--model_name', + parser.add_argument('-mn', '--model_name', help='Model name to download using Gluon package.', type=str, dest='model_name') @@ -46,7 +46,7 @@ def cli_argument_parser(): W is an input tensor width, \ H is an input tensor height, \ C is an input tensor number of channels', - default=None, + required=True, type=int, nargs=4, dest='input_shape') From 1bdff6f50610ed9478248503f2c42177e774b9af Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Tue, 7 Feb 2023 15:48:59 +0300 Subject: [PATCH 10/63] Labels for ImageNet and Cifar-10 required by MXNet --- src/inference/labels/cifar10_labels.json | 12 + src/inference/labels/image_net_labels.json | 1002 ++++++++++++++++++++ 2 files changed, 1014 insertions(+) create mode 100644 src/inference/labels/cifar10_labels.json create mode 100644 src/inference/labels/image_net_labels.json diff --git a/src/inference/labels/cifar10_labels.json b/src/inference/labels/cifar10_labels.json new file mode 100644 index 000000000..f4598d64f --- /dev/null +++ b/src/inference/labels/cifar10_labels.json @@ -0,0 +1,12 @@ +[ +"airplane", +"automobile", +"bird", +"cat", +"deer", +"dog", +"frog", +"horse", +"ship", +"truck" +] \ No newline at end of file diff --git a/src/inference/labels/image_net_labels.json b/src/inference/labels/image_net_labels.json new file mode 100644 index 000000000..e2cea3b31 --- /dev/null +++ b/src/inference/labels/image_net_labels.json @@ -0,0 +1,1002 @@ +[ +"tench, Tinca tinca", +"goldfish, Carassius auratus", +"great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias", +"tiger shark, Galeocerdo cuvieri", +"hammerhead, hammerhead shark", +"electric ray, crampfish, numbfish, torpedo", +"dtingray", +"cock", +"hen", +"ostrich, Struthio camelus", +"brambling, Fringilla montifringilla", +"goldfinch, Carduelis carduelis", +"house finch, linnet, Carpodacus mexicanus", +"junco, snowbird", +"indigo bunting, indigo finch, indigo bird, Passerina cyanea", +"robin, American robin, Turdus migratorius", +"bulbul", +"jay", +"magpie", +"chickadee", +"water ouzel, dipper", +"kite", +"bald eagle, American eagle, Haliaeetus leucocephalus", +"vulture", +"great grey owl, great gray owl, Strix nebulosa", +"European fire salamander, Salamandra salamandra", +"common newt, Triturus vulgaris", +"eft", +"spotted salamander, Ambystoma maculatum", +"axolotl, mud puppy, Ambystoma mexicanum", +"bullfrog, Rana catesbeiana", +"tree frog, tree-frog", +"tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui", +"loggerhead, loggerhead turtle, Caretta caretta", +"leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea", +"mud turtle", +"terrapin", +"box turtle, box tortoise", +"banded gecko", +"common iguana, iguana, Iguana iguana", +"American chameleon, anole, Anolis carolinensis", +"whiptail, whiptail lizard", +"agama", +"frilled lizard, Chlamydosaurus kingi", +"alligator lizard", +"Gila monster, Heloderma suspectum", +"green lizard, Lacerta viridis", +"African chameleon, Chamaeleo chamaeleon", +"Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis", +"African crocodile, Nile crocodile, Crocodylus niloticus", +"American alligator, Alligator mississipiensis", +"triceratops", +"thunder snake, worm snake, Carphophis amoenus", +"ringneck snake, ring-necked snake, ring snake", +"hognose snake, puff adder, sand viper", +"green snake, grass snake", +"king snake, kingsnake", +"garter snake, grass snake", +"water snake", +"vine snake", +"night snake, Hypsiglena torquata", +"boa constrictor, Constrictor constrictor", +"rock python, rock snake, Python sebae", +"Indian cobra, Naja naja", +"green mamba", +"sea snake", +"horned viper, cerastes, sand viper, horned asp, Cerastes cornutus", +"diamondback, diamondback rattlesnake, Crotalus adamanteus", +"sidewinder, horned rattlesnake, Crotalus cerastes", +"trilobite", +"harvestman, daddy longlegs, Phalangium opilio", +"scorpion", +"black and gold garden spider, Argiope aurantia", +"barn spider, Araneus cavaticus", +"garden spider, Aranea diademata", +"black widow, Latrodectus mactans", +"tarantula", +"wolf spider, hunting spider", +"tick", +"centipede", +"black grouse", +"ptarmigan", +"ruffed grouse, partridge, Bonasa umbellus", +"prairie chicken, prairie grouse, prairie fowl", +"peacock", +"quail", +"partridge", +"African grey, African gray, Psittacus erithacus", +"macaw", +"sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita", +"lorikeet", +"coucal", +"bee eater", +"hornbill", +"hummingbird", +"jacamar", +"toucan", +"drake", +"red-breasted merganser, Mergus serrator", +"goose", +"black swan, Cygnus atratus", +"tusker", +"echidna, spiny anteater, anteater", +"platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus", +"wallaby, brush kangaroo", +"koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus", +"wombat", +"jellyfish", +"sea anemone, anemone", +"brain coral", +"flatworm, platyhelminth", +"nematode, nematode worm, roundworm", +"conch", +"snail", +"slug", +"sea slug, nudibranch", +"chiton, coat-of-mail shell, sea cradle, polyplacophore", +"chambered nautilus, pearly nautilus, nautilus", +"Dungeness crab, Cancer magister", +"rock crab, Cancer irroratus", +"fiddler crab", +"king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica", +"American lobster, Northern lobster, Maine lobster, Homarus americanus", +"spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish", +"crayfish, crawfish, crawdad, crawdaddy", +"hermit crab", +"isopod", +"white stork, Ciconia ciconia", +"black stork, Ciconia nigra", +"spoonbill", +"flamingo", +"little blue heron, Egretta caerulea", +"American egret, great white heron, Egretta albus", +"bittern", +"crane", +"limpkin, Aramus pictus", +"European gallinule, Porphyrio porphyrio", +"American coot, marsh hen, mud hen, water hen, Fulica americana", +"bustard", +"ruddy turnstone, Arenaria interpres", +"red-backed sandpiper, dunlin, Erolia alpina", +"redshank, Tringa totanus", +"dowitcher", +"oystercatcher, oyster catcher", +"pelican", +"king penguin, Aptenodytes patagonica", +"albatross, mollymawk", +"grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus", +"killer whale, killer, orca, grampus, sea wolf, Orcinus orca", +"dugong, Dugong dugon", +"sea lion", +"Chihuahua", +"Japanese spaniel", +"Maltese dog, Maltese terrier, Maltese", +"Pekinese, Pekingese, Peke", +"Shih-Tzu", +"Blenheim spaniel", +"papillon", +"toy terrier", +"Rhodesian ridgeback", +"Afghan hound, Afghan", +"basset, basset hound", +"beagle", +"bloodhound, sleuthhound", +"bluetick", +"black-and-tan coonhound", +"Walker hound, Walker foxhound", +"English foxhound", +"redbone", +"borzoi, Russian wolfhound", +"Irish wolfhound", +"Italian greyhound", +"whippet", +"Ibizan hound, Ibizan Podenco", +"Norwegian elkhound, elkhound", +"otterhound, otter hound", +"Saluki, gazelle hound", +"Scottish deerhound, deerhound", +"Weimaraner", +"Staffordshire bullterrier, Staffordshire bull terrier", +"American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier", +"Bedlington terrier", +"Border terrier", +"Kerry blue terrier", +"Irish terrier", +"Norfolk terrier", +"Norwich terrier", +"Yorkshire terrier", +"wire-haired fox terrier", +"Lakeland terrier", +"Sealyham terrier, Sealyham", +"Airedale, Airedale terrier", +"cairn, cairn terrier", +"Australian terrier", +"Dandie Dinmont, Dandie Dinmont terrier", +"Boston bull, Boston terrier", +"miniature schnauzer", +"giant schnauzer", +"standard schnauzer", +"Scotch terrier, Scottish terrier, Scottie", +"Tibetan terrier, chrysanthemum dog", +"silky terrier, Sydney silky", +"soft-coated wheaten terrier", +"West Highland white terrier", +"Lhasa, Lhasa apso", +"flat-coated retriever", +"curly-coated retriever", +"golden retriever", +"Labrador retriever", +"Chesapeake Bay retriever", +"German short-haired pointer", +"vizsla, Hungarian pointer", +"English setter", +"Irish setter, red setter", +"Gordon setter", +"Brittany spaniel", +"clumber, clumber spaniel", +"English springer, English springer spaniel", +"Welsh springer spaniel", +"cocker spaniel, English cocker spaniel, cocker", +"Sussex spaniel", +"Irish water spaniel", +"kuvasz", +"schipperke", +"groenendael", +"malinois", +"briard", +"kelpie", +"komondor", +"Old English sheepdog, bobtail", +"Shetland sheepdog, Shetland sheep dog, Shetland", +"collie", +"Border collie", +"Bouvier des Flandres, Bouviers des Flandres", +"Rottweiler", +"German shepherd, German shepherd dog, German police dog, alsatian", +"Doberman, Doberman pinscher", +"miniature pinscher", +"Greater Swiss Mountain dog", +"Bernese mountain dog", +"Appenzeller", +"EntleBucher", +"boxer", +"bull mastiff", +"Tibetan mastiff", +"French bulldog", +"Great Dane", +"Saint Bernard, St Bernard", +"Eskimo dog, husky", +"malamute, malemute, Alaskan malamute", +"Siberian husky", +"dalmatian, coach dog, carriage dog", +"affenpinscher, monkey pinscher, monkey dog", +"basenji", +"pug, pug-dog", +"Leonberg", +"Newfoundland, Newfoundland dog", +"Great Pyrenees", +"Samoyed, Samoyede", +"Pomeranian", +"chow, chow chow", +"keeshond", +"Brabancon griffon", +"Pembroke, Pembroke Welsh corgi", +"Cardigan, Cardigan Welsh corgi", +"toy poodle", +"miniature poodle", +"standard poodle", +"Mexican hairless", +"timber wolf, grey wolf, gray wolf, Canis lupus", +"white wolf, Arctic wolf, Canis lupus tundrarum", +"red wolf, maned wolf, Canis rufus, Canis niger", +"coyote, prairie wolf, brush wolf, Canis latrans", +"dingo, warrigal, warragal, Canis dingo", +"dhole, Cuon alpinus", +"African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus", +"hyena, hyaena", +"red fox, Vulpes vulpes", +"kit fox, Vulpes macrotis", +"Arctic fox, white fox, Alopex lagopus", +"grey fox, gray fox, Urocyon cinereoargenteus", +"tabby, tabby cat", +"tiger cat", +"Persian cat", +"Siamese cat, Siamese", +"Egyptian cat", +"cougar, puma, catamount, mountain lion, painter, panther, Felis concolor", +"lynx, catamount", +"leopard, Panthera pardus", +"snow leopard, ounce, Panthera uncia", +"jaguar, panther, Panthera onca, Felis onca", +"lion, king of beasts, Panthera leo", +"tiger, Panthera tigris", +"cheetah, chetah, Acinonyx jubatus", +"brown bear, bruin, Ursus arctos", +"American black bear, black bear, Ursus americanus, Euarctos americanus", +"ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus", +"sloth bear, Melursus ursinus, Ursus ursinus", +"mongoose", +"meerkat, mierkat", +"tiger beetle", +"ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle", +"ground beetle, carabid beetle", +"long-horned beetle, longicorn, longicorn beetle", +"leaf beetle, chrysomelid", +"dung beetle", +"rhinoceros beetle", +"weevil", +"fly", +"bee", +"ant, emmet, pismire", +"grasshopper, hopper", +"cricket", +"walking stick, walkingstick, stick insect", +"cockroach, roach", +"mantis, mantid", +"cicada, cicala", +"leafhopper", +"lacewing, lacewing fly", +"dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk", +"damselfly", +"admiral", +"ringlet, ringlet butterfly", +"monarch, monarch butterfly, milkweed butterfly, Danaus plexippus", +"cabbage butterfly", +"sulphur butterfly, sulfur butterfly", +"lycaenid, lycaenid butterfly", +"starfish, sea star", +"sea urchin", +"sea cucumber, holothurian", +"wood rabbit, cottontail, cottontail rabbit", +"hare", +"Angora, Angora rabbit", +"hamster", +"porcupine, hedgehog", +"fox squirrel, eastern fox squirrel, Sciurus niger", +"marmot", +"beaver", +"guinea pig, Cavia cobaya", +"sorrel", +"zebra", +"hog, pig, grunter, squealer, Sus scrofa", +"wild boar, boar, Sus scrofa", +"warthog", +"hippopotamus, hippo, river horse, Hippopotamus amphibius", +"ox", +"water buffalo, water ox, Asiatic buffalo, Bubalus bubalis", +"bison", +"ram, tup", +"bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis", +"ibex, Capra ibex", +"hartebeest", +"impala, Aepyceros melampus", +"gazelle", +"Arabian camel, dromedary, Camelus dromedarius", +"llama", +"weasel", +"mink", +"polecat, fitch, foulmart, foumart, Mustela putorius", +"black-footed ferret, ferret, Mustela nigripes", +"otter", +"skunk, polecat, wood pussy", +"badger", +"armadillo", +"three-toed sloth, ai, Bradypus tridactylus", +"orangutan, orang, orangutang, Pongo pygmaeus", +"gorilla, Gorilla gorilla", +"chimpanzee, chimp, Pan troglodytes", +"gibbon, Hylobates lar", +"siamang, Hylobates syndactylus, Symphalangus syndactylus", +"guenon, guenon monkey", +"patas, hussar monkey, Erythrocebus patas", +"baboon", +"macaque", +"langur", +"colobus, colobus monkey", +"proboscis monkey, Nasalis larvatus", +"marmoset", +"capuchin, ringtail, Cebus capucinus", +"howler monkey, howler", +"titi, titi monkey", +"spider monkey, Ateles geoffroyi", +"squirrel monkey, Saimiri sciureus", +"Madagascar cat, ring-tailed lemur, Lemur catta", +"indri, indris, Indri indri, Indri brevicaudatus", +"Indian elephant, Elephas maximus", +"African elephant, Loxodonta africana", +"lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens", +"giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca", +"barracouta, snoek", +"eel", +"coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch", +"rock beauty, Holocanthus tricolor", +"anemone fish", +"sturgeon", +"gar, garfish, garpike, billfish, Lepisosteus osseus", +"lionfish", +"puffer, pufferfish, blowfish, globefish", +"abacus", +"abaya", +"academic gown, academic robe, judge's robe", +"accordion, piano accordion, squeeze box", +"acoustic guitar", +"aircraft carrier, carrier, flattop, attack aircraft carrier", +"airliner", +"airship, dirigible", +"altar", +"ambulance", +"amphibian, amphibious vehicle", +"analog clock", +"apiary, bee house", +"apron", +"ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin", +"assault rifle, assault gun", +"backpack, back pack, knapsack, packsack, rucksack, haversack", +"bakery, bakeshop, bakehouse", +"balance beam, beam", +"balloon", +"ballpoint, ballpoint pen, ballpen, Biro", +"Band Aid", +"banjo", +"bannister, banister, balustrade, balusters, handrail", +"barbell", +"barber chair", +"barbershop", +"barn", +"barometer", +"barrel, cask", +"barrow, garden cart, lawn cart, wheelbarrow", +"baseball", +"basketball", +"bassinet", +"bassoon", +"bathing cap, swimming cap", +"bath towel", +"bathtub, bathing tub, bath, tub", +"beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon", +"beacon, lighthouse, beacon light, pharos", +"beaker", +"bearskin, busby, shako", +"beer bottle", +"beer glass", +"bell cote, bell cot", +"bib", +"bicycle-built-for-two, tandem bicycle, tandem", +"bikini, two-piece", +"binder, ring-binder", +"binoculars, field glasses, opera glasses", +"birdhouse", +"boathouse", +"bobsled, bobsleigh, bob", +"bolo tie, bolo, bola tie, bola", +"bonnet, poke bonnet", +"bookcase", +"bookshop, bookstore, bookstall", +"bottlecap", +"bow", +"bow tie, bow-tie, bowtie", +"brass, memorial tablet, plaque", +"brassiere, bra, bandeau", +"breakwater, groin, groyne, mole, bulwark, seawall, jetty", +"breastplate, aegis, egis", +"broom", +"bucket, pail", +"buckle", +"bulletproof vest", +"bullet train, bullet", +"butcher shop, meat market", +"cab, hack, taxi, taxicab", +"caldron, cauldron", +"candle, taper, wax light", +"cannon", +"canoe", +"can opener, tin opener", +"cardigan", +"car mirror", +"carousel, carrousel, merry-go-round, roundabout, whirligig", +"carpenter's kit, tool kit", +"carton", +"car wheel", +"cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM", +"cassette", +"cassette player", +"castle", +"catamaran", +"CD player", +"cello, violoncello", +"cellular telephone, cellular phone, cellphone, cell, mobile phone", +"chain", +"chainlink fence", +"chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour", +"chain saw, chainsaw", +"chest", +"chiffonier, commode", +"chime, bell, gong", +"china cabinet, china closet", +"Christmas stocking", +"church, church building", +"cinema, movie theater, movie theatre, movie house, picture palace", +"cleaver, meat cleaver, chopper", +"cliff dwelling", +"cloak", +"clog, geta, patten, sabot", +"cocktail shaker", +"coffee mug", +"coffeepot", +"coil, spiral, volute, whorl, helix", +"combination lock", +"computer keyboard, keypad", +"confectionery, confectionary, candy store", +"container ship, containership, container vessel", +"convertible", +"corkscrew, bottle screw", +"cornet, horn, trumpet, trump", +"cowboy boot", +"cowboy hat, ten-gallon hat", +"cradle", +"crane", +"crash helmet", +"crate", +"crib, cot", +"Crock Pot", +"croquet ball", +"crutch", +"cuirass", +"dam, dike, dyke", +"desk", +"desktop computer", +"dial telephone, dial phone", +"diaper, nappy, napkin", +"digital clock", +"digital watch", +"dining table, board", +"dishrag, dishcloth", +"dishwasher, dish washer, dishwashing machine", +"disk brake, disc brake", +"dock, dockage, docking facility", +"dogsled, dog sled, dog sleigh", +"dome", +"doormat, welcome mat", +"drilling platform, offshore rig", +"drum, membranophone, tympan", +"drumstick", +"dumbbell", +"Dutch oven", +"electric fan, blower", +"electric guitar", +"electric locomotive", +"entertainment center", +"envelope", +"espresso maker", +"face powder", +"feather boa, boa", +"file, file cabinet, filing cabinet", +"fireboat", +"fire engine, fire truck", +"fire screen, fireguard", +"flagpole, flagstaff", +"flute, transverse flute", +"folding chair", +"football helmet", +"forklift", +"fountain", +"fountain pen", +"four-poster", +"freight car", +"French horn, horn", +"frying pan, frypan, skillet", +"fur coat", +"garbage truck, dustcart", +"gasmask, respirator, gas helmet", +"gas pump, gasoline pump, petrol pump, island dispenser", +"goblet", +"go-kart", +"golf ball", +"golfcart, golf cart", +"gondola", +"gong, tam-tam", +"gown", +"grand piano, grand", +"greenhouse, nursery, glasshouse", +"grille, radiator grille", +"grocery store, grocery, food market, market", +"guillotine", +"hair slide", +"hair spray", +"half track", +"hammer", +"hamper", +"hand blower, blow dryer, blow drier, hair dryer, hair drier", +"hand-held computer, hand-held microcomputer", +"handkerchief, hankie, hanky, hankey", +"hard disc, hard disk, fixed disk", +"harmonica, mouth organ, harp, mouth harp", +"harp", +"harvester, reaper", +"hatchet", +"holster", +"home theater, home theatre", +"honeycomb", +"hook, claw", +"hoopskirt, crinoline", +"horizontal bar, high bar", +"horse cart, horse-cart", +"hourglass", +"iPod", +"iron, smoothing iron", +"jack-o'-lantern", +"jean, blue jean, denim", +"jeep, landrover", +"jersey, T-shirt, tee shirt", +"jigsaw puzzle", +"jinrikisha, ricksha, rickshaw", +"joystick", +"kimono", +"knee pad", +"knot", +"lab coat, laboratory coat", +"ladle", +"lampshade, lamp shade", +"laptop, laptop computer", +"lawn mower, mower", +"lens cap, lens cover", +"letter opener, paper knife, paperknife", +"library", +"lifeboat", +"lighter, light, igniter, ignitor", +"limousine, limo", +"liner, ocean liner", +"lipstick, lip rouge", +"Loafer", +"lotion", +"loudspeaker, speaker, speaker unit, loudspeaker system, speaker system", +"loupe, jeweler's loupe", +"lumbermill, sawmill", +"magnetic compass", +"mailbag, postbag", +"mailbox, letter box", +"maillot", +"maillot, tank suit", +"manhole cover", +"maraca", +"marimba, xylophone", +"mask", +"matchstick", +"maypole", +"maze, labyrinth", +"measuring cup", +"medicine chest, medicine cabinet", +"megalith, megalithic structure", +"microphone, mike", +"microwave, microwave oven", +"military uniform", +"milk can", +"minibus", +"miniskirt, mini", +"minivan", +"missile", +"mitten", +"mixing bowl", +"mobile home, manufactured home", +"Model T", +"modem", +"monastery", +"monitor", +"moped", +"mortar", +"mortarboard", +"mosque", +"mosquito net", +"motor scooter, scooter", +"mountain bike, all-terrain bike, off-roader", +"mountain tent", +"mouse, computer mouse", +"mousetrap", +"moving van", +"muzzle", +"nail", +"neck brace", +"necklace", +"nipple", +"notebook, notebook computer", +"obelisk", +"oboe, hautboy, hautbois", +"ocarina, sweet potato", +"odometer, hodometer, mileometer, milometer", +"oil filter", +"organ, pipe organ", +"oscilloscope, scope, cathode-ray oscilloscope, CRO", +"overskirt", +"oxcart", +"oxygen mask", +"packet", +"paddle, boat paddle", +"paddlewheel, paddle wheel", +"padlock", +"paintbrush", +"pajama, pyjama, pj's, jammies", +"palace", +"panpipe, pandean pipe, syrinx", +"paper towel", +"parachute, chute", +"parallel bars, bars", +"park bench", +"parking meter", +"passenger car, coach, carriage", +"patio, terrace", +"pay-phone, pay-station", +"pedestal, plinth, footstall", +"pencil box, pencil case", +"pencil sharpener", +"perfume, essence", +"Petri dish", +"photocopier", +"pick, plectrum, plectron", +"pickelhaube", +"picket fence, paling", +"pickup, pickup truck", +"pier", +"piggy bank, penny bank", +"pill bottle", +"pillow", +"ping-pong ball", +"pinwheel", +"pirate, pirate ship", +"pitcher, ewer", +"plane, carpenter's plane, woodworking plane", +"planetarium", +"plastic bag", +"plate rack", +"plow, plough", +"plunger, plumber's helper", +"Polaroid camera, Polaroid Land camera", +"pole", +"police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria", +"poncho", +"pool table, billiard table, snooker table", +"pop bottle, soda bottle", +"pot, flowerpot", +"potter's wheel", +"power drill", +"prayer rug, prayer mat", +"printer", +"prison, prison house", +"projectile, missile", +"projector", +"puck, hockey puck", +"punching bag, punch bag, punching ball, punchball", +"purse", +"quill, quill pen", +"quilt, comforter, comfort, puff", +"racer, race car, racing car", +"racket, racquet", +"radiator", +"radio, wireless", +"radio telescope, radio reflector", +"rain barrel", +"recreational vehicle, RV, R.V.", +"reel", +"reflex camera", +"refrigerator, icebox", +"remote control, remote", +"restaurant, eating house, eating place, eatery", +"revolver, six-gun, six-shooter", +"rifle", +"rocking chair, rocker", +"rotisserie", +"rubber eraser, rubber, pencil eraser", +"rugby ball", +"rule, ruler", +"running shoe", +"safe", +"safety pin", +"saltshaker, salt shaker", +"sandal", +"sarong", +"sax, saxophone", +"scabbard", +"scale, weighing machine", +"school bus", +"schooner", +"scoreboard", +"screen, CRT screen", +"screw", +"screwdriver", +"seat belt, seatbelt", +"sewing machine", +"shield, buckler", +"shoe shop, shoe-shop, shoe store", +"shoji", +"shopping basket", +"shopping cart", +"shovel", +"shower cap", +"shower curtain", +"ski", +"ski mask", +"sleeping bag", +"slide rule, slipstick", +"sliding door", +"slot, one-armed bandit", +"snorkel", +"snowmobile", +"snowplow, snowplough", +"soap dispenser", +"soccer ball", +"sock", +"solar dish, solar collector, solar furnace", +"sombrero", +"soup bowl", +"space bar", +"space heater", +"space shuttle", +"spatula", +"speedboat", +"spider web, spider's web", +"spindle", +"sports car, sport car", +"spotlight, spot", +"stage", +"steam locomotive", +"steel arch bridge", +"steel drum", +"stethoscope", +"stole", +"stone wall", +"stopwatch, stop watch", +"stove", +"strainer", +"streetcar, tram, tramcar, trolley, trolley car", +"stretcher", +"studio couch, day bed", +"stupa, tope", +"submarine, pigboat, sub, U-boat", +"suit, suit of clothes", +"sundial", +"sunglass", +"sunglasses, dark glasses, shades", +"sunscreen, sunblock, sun blocker", +"suspension bridge", +"swab, swob, mop", +"sweatshirt", +"swimming trunks, bathing trunks", +"swing", +"switch, electric switch, electrical switch", +"syringe", +"table lamp", +"tank, army tank, armored combat vehicle, armoured combat vehicle", +"tape player", +"teapot", +"teddy, teddy bear", +"television, television system", +"tennis ball", +"thatch, thatched roof", +"theater curtain, theatre curtain", +"thimble", +"thresher, thrasher, threshing machine", +"throne", +"tile roof", +"toaster", +"tobacco shop, tobacconist shop, tobacconist", +"toilet seat", +"torch", +"totem pole", +"tow truck, tow car, wrecker", +"toyshop", +"tractor", +"trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi", +"tray", +"trench coat", +"tricycle, trike, velocipede", +"trimaran", +"tripod", +"triumphal arch", +"trolleybus, trolley coach, trackless trolley", +"trombone", +"tub, vat", +"turnstile", +"typewriter keyboard", +"umbrella", +"unicycle, monocycle", +"upright, upright piano", +"vacuum, vacuum cleaner", +"vase", +"vault", +"velvet", +"vending machine", +"vestment", +"viaduct", +"violin, fiddle", +"volleyball", +"waffle iron", +"wall clock", +"wallet, billfold, notecase, pocketbook", +"wardrobe, closet, press", +"warplane, military plane", +"washbasin, handbasin, washbowl, lavabo, wash-hand basin", +"washer, automatic washer, washing machine", +"water bottle", +"water jug", +"water tower", +"whiskey jug", +"whistle", +"wig", +"window screen", +"window shade", +"Windsor tie", +"wine bottle", +"wing", +"wok", +"wooden spoon", +"wool, woolen, woollen", +"worm fence, snake fence, snake-rail fence, Virginia fence", +"wreck", +"yawl", +"yurt", +"web site, website, internet site, site", +"comic book", +"crossword puzzle, crossword", +"street sign", +"traffic light, traffic signal, stoplight", +"book jacket, dust cover, dust jacket, dust wrapper", +"menu", +"plate", +"guacamole", +"consomme", +"hot pot, hotpot", +"trifle", +"ice cream, icecream", +"ice lolly, lolly, lollipop, popsicle", +"French loaf", +"bagel, beigel", +"pretzel", +"cheeseburger", +"hotdog, hot dog, red hot", +"mashed potato", +"head cabbage", +"broccoli", +"cauliflower", +"zucchini, courgette", +"spaghetti squash", +"acorn squash", +"butternut squash", +"cucumber, cuke", +"artichoke, globe artichoke", +"bell pepper", +"cardoon", +"mushroom", +"Granny Smith", +"strawberry", +"orange", +"lemon", +"fig", +"pineapple, ananas", +"banana", +"jackfruit, jak, jack", +"custard apple", +"pomegranate", +"hay", +"carbonara", +"chocolate sauce, chocolate syrup", +"dough", +"meat loaf, meatloaf", +"pizza, pizza pie", +"potpie", +"burrito", +"red wine", +"espresso", +"cup", +"eggnog", +"alp", +"bubble", +"cliff, drop, drop-off", +"coral reef", +"geyser", +"lakeside, lakeshore", +"promontory, headland, head, foreland", +"sandbar, sand bar", +"seashore, coast, seacoast, sea-coast", +"valley, vale", +"volcano", +"ballplayer, baseball player", +"groom, bridegroom", +"scuba diver", +"rapeseed", +"daisy", +"yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum", +"corn", +"acorn", +"hip, rose hip, rosehip", +"buckeye, horse chestnut, conker", +"coral fungus", +"agaric", +"gyromitra", +"stinkhorn, carrion fungus", +"earthstar", +"hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa", +"bolete", +"ear, spike, capitulum", +"toilet tissue, toilet paper, bathroom tissue" +] \ No newline at end of file From 559e8f4a95820d9e798d53b6c37bd695d9c54137 Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Fri, 10 Feb 2023 16:06:06 +0300 Subject: [PATCH 11/63] Prepare benchmark to infer MXNet models (requires to be tested) --- src/benchmark/README.md | 115 +++++++++++------- src/benchmark/config_parser_factory.py | 3 + .../frameworks/framework_wrapper_registry.py | 2 + src/benchmark/frameworks/known_frameworks.py | 1 + src/benchmark/frameworks/mxnet/__init__.py | 0 .../mxnet/mxnet_parameters_parser.py | 60 +++++++++ .../frameworks/mxnet/mxnet_process.py | 72 +++++++++++ src/benchmark/frameworks/mxnet/mxnet_test.py | 29 +++++ .../frameworks/mxnet/mxnet_wrapper.py | 16 +++ .../benchmark_configuration_file_template.xml | 29 +++++ 10 files changed, 280 insertions(+), 47 deletions(-) create mode 100644 src/benchmark/frameworks/mxnet/__init__.py create mode 100644 src/benchmark/frameworks/mxnet/mxnet_parameters_parser.py create mode 100644 src/benchmark/frameworks/mxnet/mxnet_process.py create mode 100644 src/benchmark/frameworks/mxnet/mxnet_test.py create mode 100644 src/benchmark/frameworks/mxnet/mxnet_wrapper.py diff --git a/src/benchmark/README.md b/src/benchmark/README.md index ca3d084f9..12dae92c8 100644 --- a/src/benchmark/README.md +++ b/src/benchmark/README.md @@ -9,9 +9,12 @@ вывод с использованием следующих фреймворков: - Inference Engine в составе - [Intel® Distribution of OpenVINO™ Toolkit][openvino-toolkit]; + [Intel® Distribution of OpenVINO™ Toolkit][openvino-toolkit]. - [Intel® Optimization for Caffe][intel-caffe]. - [Intel® Optimization for TensorFlow][intel-tensorflow]. +- [TensorFlow Lite][tensorflow-lite]. +- [ONNX Runtime][onnx-runtime]. +- [MXNet][mxnet]. ### Алгоритм работы скрипта @@ -102,14 +105,14 @@ Inference Engine предоставляет 2 программных интер **Примечание:** в публикуемой html-таблице содержатся только показатели FPS. -### Показатели производительности вывода для Intel® Optimization for Caffe и Intel® Optimization for TensorFlow +### Показатели производительности вывода для Intel® Optimization for Caffe, Intel® Optimization for TensorFlow, TensorFlow Lite и MXNet -При оценке производительности вывода для Intel® Optimization for Caffe -и Intel® Optimization for TensorFlow осуществляется последовательный -и независимый запуск запросов. Запуск очередного запроса выполняется -после завершения предыдущего. Для каждого запроса осуществляется замер -времени его выполнения. Для множества полученных времен определяется -стандартное среднеквадратичное отклонение, и отбрасываются времена, +При оценке производительности вывода для Intel® Optimization for Caffe, +Intel® Optimization for TensorFlow, TensorFlow Lite и MXNet осуществляется +последовательный и независимый запуск запросов. Запуск очередного запроса +выполняется после завершения предыдущего. Для каждого запроса осуществляется +замервремени его выполнения. Для множества полученных времен определяется +стандартное среднеквадратичное отклонение, и отбрасываются времена, выходящие за пределы трех стандартных отклонений от среднего времени вывода. Результирующий набор времен используется для вычисления показателя латентности. Остальные показатели вычисляются для всего множества времен. @@ -121,6 +124,13 @@ Inference Engine предоставляет 2 программных интер отношение размера обрабатываемой “пачки” изображений к среднему времени одного прохода. +### Показатели производительности вывода при использовании benchmark_app + +На данный момент для измерения производительности вывода библиотек +можно использовать инструмент `benchmark_app`, входящий в состав +[Intel® Distribution of OpenVINO™ Toolkit][openvino-toolkit]. +Алгоритм вычисления показателей производительности доступен в документации. + ## Использование скрипта Общий вид командной строки: @@ -159,72 +169,83 @@ python3 inference_benchmark.py \ ### Сборка (linux) -1. Клонирование репозитория. Рекомендуется использовать стабильную версию из списка https://github.com/openvinotoolkit/openvino/releases -```bash -git clone https://github.com/openvinotoolkit/openvino.git -git checkout -cd openvino -git submodule update --init --recursive -``` -2. Построить openvino, следуя официальной инструкции https://github.com/openvinotoolkit/openvino/wiki/BuildingForLinux +1. Клонирование репозитория. Рекомендуется использовать стабильную версию из списка + https://github.com/openvinotoolkit/openvino/releases. -3. В случае использования стабильной версии, установить python wheels из pypi хранилища. -```bash + ```bash + git clone https://github.com/openvinotoolkit/openvino.git + git checkout + cd openvino + git submodule update --init --recursive + ``` -pip install --upgrade pip +1. Построить openvino, следуя официальной инструкции + https://github.com/openvinotoolkit/openvino/wiki/BuildingForLinux. -pip install openvino== +1. В случае использования стабильной версии установить python wheels + из PyPI-хранилища. -pip install openvino_dev + ```bash + pip install --upgrade pip + pip install openvino== + pip install openvino_dev + pip install openvino_dev[mxnet,caffe,caffe2,onnx,pytorch,tensorflow2]== + ``` -pip install openvino_dev[mxnet,caffe,caffe2,onnx,pytorch,tensorflow2]== -``` +1. Запустите setupvars.sh: -4. Запустите setupvars.sh: -```bash -source INSTALL_DIR/setupvars.sh -``` + ```bash + source INSTALL_DIR/setupvars.sh + ``` -5. В директории INSTALL_DIR/samples/cpp запустите ./build_samples.sh. +1. В директории `INSTALL_DIR/samples/cpp` запустите `./build_samples.sh`. ### Использование -1. В конфигурационном файле (секция `FrameworkDependent`) укажите `Mode`: `ovbenchmark_cpp_latency` или `ovbenchmark_cpp_throughput` +1. В конфигурационном файле (секция `FrameworkDependent`) + укажите `Mode`: `ovbenchmark_cpp_latency` или `ovbenchmark_cpp_throughput`. -2. Найдите исполняемый файл benchmark_app по адресу: -``` -/home//inference_engine_cpp_samples_build/intel64/Release/benchmark_app -``` -3. Используйте его в качестве параметра для inference_benchmark.py: -```bash -python3 inference_benchmark.py -c -r result.csv -b /home//inference_engine_cpp_samples_build/intel64/Release/benchmark_app -``` +1. Найдите исполняемый файл `benchmark_app` по адресу, приведенному ниже. + + ``` + /home//inference_engine_cpp_samples_build/intel64/Release/benchmark_app + ``` + +1. Используйте его в качестве параметра для `inference_benchmark.py`: + + ```bash + python3 inference_benchmark.py -c -r result.csv -b /home//inference_engine_cpp_samples_build/intel64/Release/benchmark_app + ``` ## Использование OpenVINO Benchmark Python tool в качестве инструмента для замеров ### Установка -1. В случае использования стабильной версии, установить python wheels из pypi хранилища. -```bash -pip install --upgrade pip +В случае использования стабильной версии, установить python wheels из PyPI-хранилища. +```bash +pip install --upgrade pip pip install openvino== - pip install openvino_dev - pip install openvino_dev[mxnet,caffe,caffe2,onnx,pytorch,tensorflow2]== ``` ### Использование -1. В конфигурационном файле (секция `FrameworkDependent`) укажите `Mode`: `ovbenchmark_python_latency` или `ovbenchmark_python_throughput` +1. В конфигурационном файле (секция `FrameworkDependent`) + укажите `Mode`: `ovbenchmark_python_latency` или `ovbenchmark_python_throughput`. + +1. Запустите скрипт `nference_benchmark.py`. + + ```bash + python3 inference_benchmark.py -c -r result.csv + ``` -2. Запустите скрипт -```bash -python3 inference_benchmark.py -c -r result.csv -``` [openvino-toolkit]: https://software.intel.com/en-us/openvino-toolkit [intel-caffe]: https://github.com/intel/caffe [intel-tensorflow]: https://www.intel.com/content/www/us/en/developer/articles/guide/optimization-for-tensorflow-installation-guide.html +[tensorflow-lite]: https://www.tensorflow.org/lite +[onnx-runtime]: https://onnxruntime.ai +[mxnet]: https://mxnet.apache.org diff --git a/src/benchmark/config_parser_factory.py b/src/benchmark/config_parser_factory.py index c0f0aa11c..afbf42672 100644 --- a/src/benchmark/config_parser_factory.py +++ b/src/benchmark/config_parser_factory.py @@ -4,6 +4,7 @@ from frameworks.openvino.openvino_parameters_parser import OpenVINOParametersParser from frameworks.tensorflow.tensorflow_parameters_parser import TensorFlowParametersParser from frameworks.tensorflow_lite.tensorflow_lite_parameters_parser import TensorFlowLiteParametersParser +from frameworks.mxnet.mxnet_parameters_parser import MXNetParametersParser def get_parameters_parser(framework): @@ -17,4 +18,6 @@ def get_parameters_parser(framework): return OnnxRuntimeParametersParser() if framework == KnownFrameworks.tensorflow_lite: return TensorFlowLiteParametersParser() + if framework == KnownFrameworks.mxnet: + return MXNetParametersParser() raise NotImplementedError(f'Unknown framework {framework}') diff --git a/src/benchmark/frameworks/framework_wrapper_registry.py b/src/benchmark/frameworks/framework_wrapper_registry.py index d251c7de0..268a7b0e2 100644 --- a/src/benchmark/frameworks/framework_wrapper_registry.py +++ b/src/benchmark/frameworks/framework_wrapper_registry.py @@ -6,6 +6,7 @@ from .tensorflow.tensorflow_wrapper import TensorFlowWrapper from .onnx_runtime.onnx_runtime_wrapper import OnnxRuntimeWrapper from .tensorflow_lite.tensorflow_lite_wrapper import TensorFlowLiteWrapper +from .mxnet.mxnet_wrapper import MXNetWrapper class FrameworkWrapperRegistry(metaclass=Singleton): @@ -32,3 +33,4 @@ def _get_wrappers(self): self._framework_wrappers[OpenVINOWrapper.framework_name] = OpenVINOWrapper() self._framework_wrappers[OnnxRuntimeWrapper.framework_name] = OnnxRuntimeWrapper() self._framework_wrappers[TensorFlowLiteWrapper.framework_name] = TensorFlowLiteWrapper() + self._framework_wrappers[MXNetWrapper.framework_name] = MXNetWrapper() diff --git a/src/benchmark/frameworks/known_frameworks.py b/src/benchmark/frameworks/known_frameworks.py index 2829d0b86..6b44f5621 100644 --- a/src/benchmark/frameworks/known_frameworks.py +++ b/src/benchmark/frameworks/known_frameworks.py @@ -4,3 +4,4 @@ class KnownFrameworks: caffe = 'Caffe' onnx_runtime = 'ONNX Runtime' tensorflow_lite = 'TensorFlowLite' + mxnet = 'MXNet' diff --git a/src/benchmark/frameworks/mxnet/__init__.py b/src/benchmark/frameworks/mxnet/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/benchmark/frameworks/mxnet/mxnet_parameters_parser.py b/src/benchmark/frameworks/mxnet/mxnet_parameters_parser.py new file mode 100644 index 000000000..ad6dbbc3d --- /dev/null +++ b/src/benchmark/frameworks/mxnet/mxnet_parameters_parser.py @@ -0,0 +1,60 @@ +from ..config_parser.dependent_parameters_parser import DependentParametersParser +from ..config_parser.framework_parameters_parser import FrameworkParameters + + +class MXNetParametersParser(DependentParametersParser): + def parse_parameters(self, curr_test): + CONFIG_FRAMEWORK_DEPENDENT_TAG = 'FrameworkDependent' + CONFIG_FRAMEWORK_DEPENDENT_INPUT_NAME_TAG = 'InputName' + CONFIG_FRAMEWORK_DEPENDENT_INPUT_SHAPE_TAG = 'InputShape' + CONFIG_FRAMEWORK_DEPENDENT_NORMALIZE_TAG = 'Normalize' + CONFIG_FRAMEWORK_DEPENDENT_MEAN_TAG = 'Mean' + CONFIG_FRAMEWORK_DEPENDENT_STD_TAG = 'Std' + CONFIG_FRAMEWORK_DEPENDENT_CHANNEL_SWAP_TAG = 'ChannelSwap' + + dep_parameters_tag = curr_test.getElementsByTagName(CONFIG_FRAMEWORK_DEPENDENT_TAG)[0] + + _input_name = dep_parameters_tag.getElementsByTagName( + CONFIG_FRAMEWORK_DEPENDENT_INPUT_NAME_TAG)[0].firstChild + _input_shape = dep_parameters_tag.getElementsByTagName( + CONFIG_FRAMEWORK_DEPENDENT_INPUT_SHAPE_TAG)[0].firstChild + _normalize = dep_parameters_tag.getElementsByTagName( + CONFIG_FRAMEWORK_DEPENDENT_NORMALIZE_TAG)[0].firstChild + _mean = dep_parameters_tag.getElementsByTagName( + CONFIG_FRAMEWORK_DEPENDENT_MEAN_TAG)[0].firstChild + _std = dep_parameters_tag.getElementsByTagName( + CONFIG_FRAMEWORK_DEPENDENT_STD_TAG)[0].firstChild + _channel_swap = dep_parameters_tag.getElementsByTagName( + CONFIG_FRAMEWORK_DEPENDENT_CHANNEL_SWAP_TAG)[0].firstChild + + return MXNetParameters( + input_name=_input_name.data if _input_name else None, + input_shape=_input_shape.data if _input_shape else None, + normalize=_normalize.data if _normalize else None, + mean=_mean.data if _mean else None, + std=_std.data if _std else None, + channel_swap=_channel_swap.data if _channel_swap else None, + ) + + +class MXNetParameters(FrameworkParameters): + def __init__(self, input_name, input_shape, normalize, mean, std, channel_swap): + self.input_name = None + self.input_shape = None + self.normalize = None + self.mean = None + self.std = None + self.channel_swap = None + + if self._parameter_not_is_none(input_name): + self.input_name = input_name + if self._parameter_not_is_none(input_shape): + self.input_shape = input_shape + if self._parameter_not_is_none(normalize): + self.normalize = normalize + if self._parameter_not_is_none(mean): + self.mean = mean + if self._parameter_not_is_none(std): + self.std = std + if self._parameter_not_is_none(channel_swap): + self.channel_swap = channel_swap diff --git a/src/benchmark/frameworks/mxnet/mxnet_process.py b/src/benchmark/frameworks/mxnet/mxnet_process.py new file mode 100644 index 000000000..aeb131db4 --- /dev/null +++ b/src/benchmark/frameworks/mxnet/mxnet_process.py @@ -0,0 +1,72 @@ +from pathlib import Path + +from ..processes import ProcessHandler + + +class MXNetProcess(ProcessHandler): + def __init__(self, test, executor, log): + super().__init__(test, executor, log) + + @staticmethod + def create_process(test, executor, log): + return MXNetProcess(test, executor, log) + + def get_performance_metrics(self): + if self._status != 0 or len(self._output) == 0: + return None, None, None + + result = self._output[-1].strip().split(',') + average_time = float(result[0]) + fps = float(result[1]) + latency = float(result[2]) + + return average_time, fps, latency + + def _fill_command_line(self): + path_to_mxnet_script = Path.joinpath(self.inference_script_root, 'inference_mxnet.py') + python = ProcessHandler.get_cmd_python_version() + + name = self._test.model.name + model_json = self._test.model.model + model_params = self._test.model.weight + dataset = self._test.dataset.path + input_shape = self._test.dep_parameters.input_shape + batch_size = self._test.indep_parameters.batch_size + iteration = self._test.indep_parameters.iteration + if (name is not None) and (model_json is None) and (model_params is None): + common_params = f'-mn {name} -i {dataset} -is {input_shape} -b {batch_size} -ni {iteration}' + elif (name is None) and (model_json is not None) and (model_params is not None): + common_params = f'-m {model_json} -w {model_params} -i {dataset} -is {input_shape} -b {batch_size} -ni {iteration}' + else: + raise Exception('Incorrect model parameters. Set model name or file names.') + + input_name = self._test.dep_parameters.input_name + common_params = MXNetProcess._add_optional_argument_to_cmd_line( + common_params, '--input_name', input_name) + + normalize = self._test.dep_parameters.normalize + common_params = MXNetProcess._add_optional_argument_to_cmd_line( + common_params, '--norm', normalize) + + mean = self._test.dep_parameters.mean + common_params = MXNetProcess._add_optional_argument_to_cmd_line( + common_params, '--mean', mean) + + std = self._test.dep_parameters.std + common_params = MXNetProcess._add_optional_argument_to_cmd_line( + common_params, '--std', std) + + channel_swap = self._test.dep_parameters.channel_swap + common_params = MXNetProcess._add_optional_argument_to_cmd_line( + common_params, '--channel_swap', channel_swap) + + device = self._test.indep_parameters.device + common_params = MXNetProcess._add_optional_argument_to_cmd_line( + common_params, device) + + common_params = MXNetProcess._add_argument_to_cmd_line( + common_params, '--raw_output', 'true') + + command_line = f'{python} {path_to_mxnet_script} {common_params}' + + return command_line diff --git a/src/benchmark/frameworks/mxnet/mxnet_test.py b/src/benchmark/frameworks/mxnet/mxnet_test.py new file mode 100644 index 000000000..c695dacfe --- /dev/null +++ b/src/benchmark/frameworks/mxnet/mxnet_test.py @@ -0,0 +1,29 @@ +from ..config_parser.test_reporter import Test + + +class MXNetTest(Test): + def __init__(self, model, dataset, indep_parameters, dep_parameters): + super().__init__(model, dataset, indep_parameters, dep_parameters) + + def get_report(self): + other_param = ', '.join([f'Device: {self.indep_parameters.device}', + f'Iteration count: {self.indep_parameters.iteration}', + f'Input name: {self.dep_parameters.input_name}', + f'Input shape: {self.dep_parameters.input_shape}', + f'Normalization flag: {self.dep_parameters.normalize}', + f'Mean: {self.dep_parameters.mean}', + f'Standard deviation: {self.dep_parameters.std}', + f'Channel swap: {self.dep_parameters.channel_swap}']) + + report_res = { + 'task': self.model.task, + 'model': self.model.name, + 'dataset': self.dataset.name, + 'source_framework': self.model.source_framework, + 'inference_framework': self.indep_parameters.inference_framework, + 'precision': self.model.precision, + 'batch_size': self.indep_parameters.batch_size, + 'framework_params': other_param, + } + + return report_res diff --git a/src/benchmark/frameworks/mxnet/mxnet_wrapper.py b/src/benchmark/frameworks/mxnet/mxnet_wrapper.py new file mode 100644 index 000000000..bb100efe6 --- /dev/null +++ b/src/benchmark/frameworks/mxnet/mxnet_wrapper.py @@ -0,0 +1,16 @@ +from .mxnet_process import MXNetProcess +from .mxnet_test import MXNetTest +from ..framework_wrapper import FrameworkWrapper +from ..known_frameworks import KnownFrameworks + + +class MXNetWrapper(FrameworkWrapper): + framework_name = KnownFrameworks.mxnet + + @staticmethod + def create_process(test, executor, log, cpp_benchmarks_dir=None): + return MXNetProcess.create_process(test, executor, log) + + @staticmethod + def create_test(model, dataset, indep_parameters, dep_parameters): + return MXNetTest(model, dataset, indep_parameters, dep_parameters) diff --git a/src/configs/benchmark_configuration_file_template.xml b/src/configs/benchmark_configuration_file_template.xml index 1f969e522..5e1ae6611 100644 --- a/src/configs/benchmark_configuration_file_template.xml +++ b/src/configs/benchmark_configuration_file_template.xml @@ -123,4 +123,33 @@ + + + + + + + + + + + + + + + MXNet + + CPU + + + + + + + + + + + + \ No newline at end of file From d827f72493caefd3bb0f95701634f3135be6d1ac Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Fri, 10 Feb 2023 16:10:32 +0300 Subject: [PATCH 12/63] Becnhmark readme fixes --- src/benchmark/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/benchmark/README.md b/src/benchmark/README.md index 12dae92c8..712d4e366 100644 --- a/src/benchmark/README.md +++ b/src/benchmark/README.md @@ -169,7 +169,7 @@ python3 inference_benchmark.py \ ### Сборка (linux) -1. Клонирование репозитория. Рекомендуется использовать стабильную версию из списка +1. Клонировать репозиторий. Рекомендуется использовать стабильную версию из списка https://github.com/openvinotoolkit/openvino/releases. ```bash @@ -179,7 +179,7 @@ python3 inference_benchmark.py \ git submodule update --init --recursive ``` -1. Построить openvino, следуя официальной инструкции +1. Собрать OpenVINO, следуя официальной инструкции https://github.com/openvinotoolkit/openvino/wiki/BuildingForLinux. 1. В случае использования стабильной версии установить python wheels @@ -192,13 +192,13 @@ python3 inference_benchmark.py \ pip install openvino_dev[mxnet,caffe,caffe2,onnx,pytorch,tensorflow2]== ``` -1. Запустите setupvars.sh: +1. Запустить `setupvars.sh`: ```bash source INSTALL_DIR/setupvars.sh ``` -1. В директории `INSTALL_DIR/samples/cpp` запустите `./build_samples.sh`. +1. В директории `INSTALL_DIR/samples/cpp` запустить `./build_samples.sh`. ### Использование From 1a0eca156934a0ee326de8f4e52079ddb8632685 Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Fri, 17 Feb 2023 08:03:01 +0300 Subject: [PATCH 13/63] Bug fixes (checked for alexnet) --- src/benchmark/config_processor.py | 62 ++++++++++++++----- .../frameworks/mxnet/mxnet_process.py | 6 +- src/benchmark/frameworks/mxnet/mxnet_test.py | 23 ++++--- src/benchmark/frameworks/processes.py | 4 +- src/benchmark/inference_benchmark.py | 23 ++++--- src/inference/inference_mxnet.py | 9 +-- 6 files changed, 88 insertions(+), 39 deletions(-) diff --git a/src/benchmark/config_processor.py b/src/benchmark/config_processor.py index e158398e6..ce1b72f21 100644 --- a/src/benchmark/config_processor.py +++ b/src/benchmark/config_processor.py @@ -8,7 +8,7 @@ def process_config(config, log): - test_parser = TestConfigParser() + test_parser = TestConfigParser(log) test_list = [] tests = test_parser.get_tests_list(config) @@ -28,39 +28,73 @@ def process_config(config, log): class TestConfigParser: + def __init__(self, log): + self._log = log + def get_tests_list(self, config): return minidom.parse(config).getElementsByTagName('Test') def parse_model(self, curr_test): model_tag = curr_test.getElementsByTagName('Model')[0] + task = model_tag.getElementsByTagName('Task')[0].firstChild.data + model_name = model_tag.getElementsByTagName('Name')[0].firstChild.data + precision = model_tag.getElementsByTagName('Precision')[0].firstChild.data + source_framework = model_tag.getElementsByTagName('SourceFramework')[0].firstChild.data + + model_path = "" + try: + model_path = model_tag.getElementsByTagName('ModelPath')[0].firstChild.data + except Exception as ex: + self._log.warning(f'Parsing model path failed. Exception was generated: {str(ex)}.') + weights_path = "" + try: + weights_path = model_tag.getElementsByTagName('WeightsPath')[0].firstChild.data + except Exception as ex: + self._log.warning(f'Parsing weights path failed. Exception was generated: {str(ex)}.') + + self._log.info(f'Model:\n\tName - {model_name}\n\tTask - {task}\n\t' + f'Precision - {precision}\n\tSource framework - {source_framework}\n\t' + f'Model path - {model_path}\n\tWeights path - {weights_path}') return Model( - task=model_tag.getElementsByTagName('Task')[0].firstChild.data, - name=model_tag.getElementsByTagName('Name')[0].firstChild.data, - precision=model_tag.getElementsByTagName('Precision')[0].firstChild.data, - source_framework=model_tag.getElementsByTagName('SourceFramework')[0].firstChild.data, - model_path=model_tag.getElementsByTagName('ModelPath')[0].firstChild.data, - weights_path=model_tag.getElementsByTagName('WeightsPath')[0].firstChild.data, + task=task, name=model_name, precision=precision, + source_framework=source_framework, model_path=model_path, + weights_path=weights_path, ) def parse_dataset(self, curr_test): dataset_tag = curr_test.getElementsByTagName('Dataset')[0] + name = dataset_tag.getElementsByTagName('Name')[0].firstChild.data + path = dataset_tag.getElementsByTagName('Path')[0].firstChild.data + + self._log.info(f'Dataset:\n\tName - {name}\n\tPath - {path}') return Dataset( - name=dataset_tag.getElementsByTagName('Name')[0].firstChild.data, - path=dataset_tag.getElementsByTagName('Path')[0].firstChild.data, + name=name, path=path, ) def parse_independent_parameters(self, curr_test): indep_parameters_tag = curr_test.getElementsByTagName('FrameworkIndependent')[0] _batch_size = indep_parameters_tag.getElementsByTagName('BatchSize')[0].firstChild + inference_framework = indep_parameters_tag.getElementsByTagName('InferenceFramework')[0].firstChild.data + batch_size = _batch_size.data if _batch_size else None + device = indep_parameters_tag.getElementsByTagName('Device')[0].firstChild.data + iteration_count = indep_parameters_tag.getElementsByTagName('IterationCount')[0].firstChild.data + test_time_limit = indep_parameters_tag.getElementsByTagName('TestTimeLimit')[0].firstChild.data + + self._log.info(f'Framework independent parameters:\n\t' + f'Inference framework - {inference_framework}\n\t' + f'Batch size - {batch_size}\n\t' + f'Device - {device}\n\t' + f'Number of iterations - {iteration_count}\n\t' + f'Time limit of test execution - {test_time_limit}') return FrameworkIndependentParameters( - inference_framework=indep_parameters_tag.getElementsByTagName('InferenceFramework')[0].firstChild.data, - batch_size=_batch_size.data if _batch_size else None, - device=indep_parameters_tag.getElementsByTagName('Device')[0].firstChild.data, - iterarion_count=indep_parameters_tag.getElementsByTagName('IterationCount')[0].firstChild.data, - test_time_limit=indep_parameters_tag.getElementsByTagName('TestTimeLimit')[0].firstChild.data, + inference_framework=inference_framework, + batch_size=batch_size, + device=device, + iterarion_count=iteration_count, + test_time_limit=test_time_limit, ) def parse_dependent_parameters(self, curr_test, framework): diff --git a/src/benchmark/frameworks/mxnet/mxnet_process.py b/src/benchmark/frameworks/mxnet/mxnet_process.py index aeb131db4..f812b9a31 100644 --- a/src/benchmark/frameworks/mxnet/mxnet_process.py +++ b/src/benchmark/frameworks/mxnet/mxnet_process.py @@ -33,7 +33,9 @@ def _fill_command_line(self): input_shape = self._test.dep_parameters.input_shape batch_size = self._test.indep_parameters.batch_size iteration = self._test.indep_parameters.iteration - if (name is not None) and (model_json is None) and (model_params is None): + if (name is not None) and \ + (model_json is None or model_json == '') and \ + (model_params is None or model_params == ''): common_params = f'-mn {name} -i {dataset} -is {input_shape} -b {batch_size} -ni {iteration}' elif (name is None) and (model_json is not None) and (model_params is not None): common_params = f'-m {model_json} -w {model_params} -i {dataset} -is {input_shape} -b {batch_size} -ni {iteration}' @@ -62,7 +64,7 @@ def _fill_command_line(self): device = self._test.indep_parameters.device common_params = MXNetProcess._add_optional_argument_to_cmd_line( - common_params, device) + common_params, '--device', device) common_params = MXNetProcess._add_argument_to_cmd_line( common_params, '--raw_output', 'true') diff --git a/src/benchmark/frameworks/mxnet/mxnet_test.py b/src/benchmark/frameworks/mxnet/mxnet_test.py index c695dacfe..47795b9d8 100644 --- a/src/benchmark/frameworks/mxnet/mxnet_test.py +++ b/src/benchmark/frameworks/mxnet/mxnet_test.py @@ -1,3 +1,5 @@ +from collections import OrderedDict + from ..config_parser.test_reporter import Test @@ -6,15 +8,17 @@ def __init__(self, model, dataset, indep_parameters, dep_parameters): super().__init__(model, dataset, indep_parameters, dep_parameters) def get_report(self): - other_param = ', '.join([f'Device: {self.indep_parameters.device}', - f'Iteration count: {self.indep_parameters.iteration}', - f'Input name: {self.dep_parameters.input_name}', - f'Input shape: {self.dep_parameters.input_shape}', - f'Normalization flag: {self.dep_parameters.normalize}', - f'Mean: {self.dep_parameters.mean}', - f'Standard deviation: {self.dep_parameters.std}', - f'Channel swap: {self.dep_parameters.channel_swap}']) - + parameters = OrderedDict() + parameters.update({'Device': self.indep_parameters.device}) + parameters.update({'Iteration count': self.indep_parameters.iteration}) + parameters.update({'Input name': self.dep_parameters.input_name}) + parameters.update({'Input shape': self.dep_parameters.input_shape}) + parameters.update({'Normalization flag': self.dep_parameters.normalize}) + parameters.update({'Mean': self.dep_parameters.mean}) + parameters.update({'Standard deviation': self.dep_parameters.std}) + parameters.update({'Channel swap': self.dep_parameters.channel_swap}) + other_param = self._get_optional_parameters_string(parameters) + report_res = { 'task': self.model.task, 'model': self.model.name, @@ -23,6 +27,7 @@ def get_report(self): 'inference_framework': self.indep_parameters.inference_framework, 'precision': self.model.precision, 'batch_size': self.indep_parameters.batch_size, + 'mode': 'Sync', 'framework_params': other_param, } diff --git a/src/benchmark/frameworks/processes.py b/src/benchmark/frameworks/processes.py index ecb8d0a1b..7f24d1098 100644 --- a/src/benchmark/frameworks/processes.py +++ b/src/benchmark/frameworks/processes.py @@ -35,8 +35,8 @@ def execute(self): command_line = self._fill_command_line() if command_line == '': self.__log.error('Command line is empty') - self.__log.info(f'Start inference test on model : {self._test.model.name}') - self.__log.info(f'Command line is : {command_line}') + self.__log.info(f'Start inference test on model: {self._test.model.name}') + self.__log.info(f'Command line is: {command_line}') self._executor.set_target_framework(self._test.indep_parameters.inference_framework) self._status, self._output = self._executor.execute_process(command_line, self._test.indep_parameters.test_time_limit) diff --git a/src/benchmark/inference_benchmark.py b/src/benchmark/inference_benchmark.py index 33bb6e403..02d65ebe1 100644 --- a/src/benchmark/inference_benchmark.py +++ b/src/benchmark/inference_benchmark.py @@ -55,8 +55,8 @@ def cli_argument_parser(): return args -def inference_benchmark(executor_type, test_list, output_handler, log, cpp_benchmarks_dir=None, - openvino_cpp_benchmark_dir=None): +def inference_benchmark(executor_type, test_list, output_handler, log, + cpp_benchmarks_dir=None, openvino_cpp_benchmark_dir=None): status = Status.EXIT_SUCCESS try: @@ -72,8 +72,11 @@ def inference_benchmark(executor_type, test_list, output_handler, log, cpp_bench benchmarks_path = openvino_cpp_benchmark_dir try: - test_process = FrameworkWrapperRegistry()[framework_name].create_process(test, process_executor, - log, benchmarks_path) + log.info('Creating separate process for the test') + test_process = FrameworkWrapperRegistry()[framework_name].create_process( + test, process_executor, log, benchmarks_path) + + log.info('Executing process') test_process.execute() test_status = test_process.get_status() @@ -85,7 +88,7 @@ def inference_benchmark(executor_type, test_list, output_handler, log, cpp_bench log.error(f'Inference failed with exception: {ex}', exc_info=True) test_process = None - log.info('Saving test result in file\n') + log.info('Saving test result in file') output_handler.add_row_to_table(process_executor, test, test_process) return status @@ -96,6 +99,8 @@ def inference_benchmark(executor_type, test_list, output_handler, log, cpp_bench sys.excepthook = exception_hook args = cli_argument_parser() + + log.info(f'Parsing configuration file {args.config_path}') test_list = process_config(args.config_path, log) log.info(f'Create result table with name: {args.result_file}') @@ -103,9 +108,11 @@ def inference_benchmark(executor_type, test_list, output_handler, log, cpp_bench output_handler = OutputHandler(args.result_file, args.csv_delimiter) output_handler.create_table() - log.info(f'Start {len(test_list)} inference tests\n') + log.info(f'Start {len(test_list)} inference tests') - inference_status = inference_benchmark(args.executor_type, test_list, output_handler, log, - args.cpp_benchmarks_dir, args.openvino_cpp_benchmark_dir) + inference_status = inference_benchmark(args.executor_type, test_list, + output_handler, log, + args.cpp_benchmarks_dir, + args.openvino_cpp_benchmark_dir) log.info('Inference tests completed' if not inference_status.value else 'Inference tests failed') sys.exit(inference_status.value) diff --git a/src/inference/inference_mxnet.py b/src/inference/inference_mxnet.py index dafd8a5a9..4a259b26e 100644 --- a/src/inference/inference_mxnet.py +++ b/src/inference/inference_mxnet.py @@ -137,7 +137,7 @@ def load_network_gluon(model_json, model_params, context, input_name): return deserialized_net -def load_network_gluon_model_zoo(model_name, context, input_name, input_shape): +def load_network_gluon_model_zoo(model_name, context): log.info(f'Loading network \"{model_name}\"') net = model_zoo.get_model(model_name, pretrained=True, ctx=context) @@ -238,15 +238,16 @@ def main(): if (args.model_name is not None) and \ (args.model_json is None) and \ (args.model_params is None): - net = load_network_gluon_model_zoo(args.model_name, context, args.input_name, - args.input_shape) + net = load_network_gluon_model_zoo(args.model_name, context) elif (args.model_json is not None) and (args.model_params is not None): net = load_network_gluon(args.model_json, args.model_params, context, args.input_name) else: raise ValueError('Incorrect arguments.') - log.info('Preparing input data') + log.info(f'Shape for input layer {args.input_name}: {args.input_shape}') + + log.info(f'Preparing input data {args.input}') io.prepare_input(net, args.input) log.info(f'Starting inference ({args.number_iter} iterations) on {args.device}') From dcab1db41bc34aa46deff7f369c132efe30c02ac Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Fri, 17 Feb 2023 08:12:14 +0300 Subject: [PATCH 14/63] Coding style bug fixes --- src/benchmark/config_processor.py | 4 ++-- src/benchmark/frameworks/mxnet/mxnet_test.py | 2 +- src/inference/inference_mxnet.py | 12 ++++++------ 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/benchmark/config_processor.py b/src/benchmark/config_processor.py index ce1b72f21..e48fecfa7 100644 --- a/src/benchmark/config_processor.py +++ b/src/benchmark/config_processor.py @@ -42,12 +42,12 @@ def parse_model(self, curr_test): precision = model_tag.getElementsByTagName('Precision')[0].firstChild.data source_framework = model_tag.getElementsByTagName('SourceFramework')[0].firstChild.data - model_path = "" + model_path = '' try: model_path = model_tag.getElementsByTagName('ModelPath')[0].firstChild.data except Exception as ex: self._log.warning(f'Parsing model path failed. Exception was generated: {str(ex)}.') - weights_path = "" + weights_path = '' try: weights_path = model_tag.getElementsByTagName('WeightsPath')[0].firstChild.data except Exception as ex: diff --git a/src/benchmark/frameworks/mxnet/mxnet_test.py b/src/benchmark/frameworks/mxnet/mxnet_test.py index 47795b9d8..53ff48db2 100644 --- a/src/benchmark/frameworks/mxnet/mxnet_test.py +++ b/src/benchmark/frameworks/mxnet/mxnet_test.py @@ -18,7 +18,7 @@ def get_report(self): parameters.update({'Standard deviation': self.dep_parameters.std}) parameters.update({'Channel swap': self.dep_parameters.channel_swap}) other_param = self._get_optional_parameters_string(parameters) - + report_res = { 'task': self.model.task, 'model': self.model.name, diff --git a/src/inference/inference_mxnet.py b/src/inference/inference_mxnet.py index 4a259b26e..8027986f7 100644 --- a/src/inference/inference_mxnet.py +++ b/src/inference/inference_mxnet.py @@ -3,11 +3,11 @@ import sys from time import time import json +import warnings import numpy as np import mxnet import mxnet.gluon.model_zoo.vision as model_zoo -import warnings import postprocessing_data as pp from io_adapter import IOAdapter @@ -42,10 +42,10 @@ def cli_argument_parser(): type=str, dest='input_name') parser.add_argument('-is', '--input_shape', - help='Input shape BxWxHxC, B is a batch size,\ - W is an input tensor width, \ - H is an input tensor height, \ - C is an input tensor number of channels', + help='Input shape BxWxHxC, B is a batch size,' + 'W is an input tensor width,' + 'H is an input tensor height,' + 'C is an input tensor number of channels', required=True, type=int, nargs=4, @@ -131,7 +131,7 @@ def get_device_to_infer(device): def load_network_gluon(model_json, model_params, context, input_name): log.info(f'Deserializing network from file ({model_json}, {model_params})') with warnings.catch_warnings(): - warnings.simplefilter("ignore") + warnings.simplefilter('ignore') deserialized_net = mxnet.gluon.nn.SymbolBlock.imports( model_json, [input_name], model_params, ctx=context) return deserialized_net From f2155bd33bfd72ee866de37a21ce9ad14c87af6d Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Sun, 19 Feb 2023 07:25:17 +0300 Subject: [PATCH 15/63] Fix logging format --- src/benchmark/inference_benchmark.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/benchmark/inference_benchmark.py b/src/benchmark/inference_benchmark.py index 02d65ebe1..2506e7897 100644 --- a/src/benchmark/inference_benchmark.py +++ b/src/benchmark/inference_benchmark.py @@ -9,7 +9,7 @@ from output import OutputHandler sys.path.append(str(Path(__file__).resolve().parents[1].joinpath('utils'))) -from logger_conf import configure_logger, exception_hook # noqa: E402 +from logger_conf import exception_hook # noqa: E402 from constants import Status # noqa: E402 @@ -95,7 +95,11 @@ def inference_benchmark(executor_type, test_list, output_handler, log, if __name__ == '__main__': - configure_logger() + log.basicConfig( + format='[ %(levelname)s ] %(message)s', + level=log.INFO, + stream=sys.stdout, + ) sys.excepthook = exception_hook args = cli_argument_parser() From bf72f8869611eed6b3eccde33b264bcea783d407 Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Tue, 21 Feb 2023 15:51:36 +0300 Subject: [PATCH 16/63] Validation results for InceptionV3 --- .../validation/validation_results_mxnet_gluon_modelzoo.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/results/validation/validation_results_mxnet_gluon_modelzoo.md b/results/validation/validation_results_mxnet_gluon_modelzoo.md index 25c2cf069..8d83e9c86 100644 --- a/results/validation/validation_results_mxnet_gluon_modelzoo.md +++ b/results/validation/validation_results_mxnet_gluon_modelzoo.md @@ -27,7 +27,7 @@ densenet121 |0.9523346 Granny Smith
0.0132273 orange
0.0125171 lemon
0 densenet161 |0.9372969 Granny Smith
0.0082274 dumbbell
0.0056475 piggy bank, penny bank
0.0055374 ping-pong ball
0.0041915 pitcher, ewer| densenet169 |0.9811633 Granny Smith
0.0033828 piggy bank, penny bank
0.0021365 orange
0.0019196 lemon
0.0017232 pomegranate| densenet201 |0.9119796 Granny Smith
0.0533456 piggy bank, penny bank
0.0056831 lemon
0.0017810 pool table, billiard table, snooker table
0.0015689 tennis ball| -inceptionv3 || +inceptionv3 |0.9884678 Granny Smith
0.0006726 bikini, two-piece
0.0005334 piggy bank, penny bank
0.0003360 pomegranate
0.0001373 pool table, billiard table, snooker table| mobilenet0.25|0.3314433 piggy bank, penny bank
0.1333785 maraca
0.1262991 croquet ball
0.0684097 dumbbell
0.0539143 hair slide| mobilenet0.5 |0.7425038 piggy bank, penny bank
0.0554336 saltshaker, salt shaker
0.0353432 Granny Smith
0.0217170 pencil sharpener
0.0169298 rubber eraser, rubber, pencil eraser| mobilenet0.75|0.2680034 piggy bank, penny bank
0.2459760 Granny Smith
0.0562553 dumbbell
0.0306232 necklace
0.0305400 pitcher, ewer| @@ -75,7 +75,7 @@ densenet121 |0.9841659 junco, snowbird
0.0072199 chickadee
0.0034963 bram densenet161 |0.9932107 junco, snowbird
0.0015922 chickadee
0.0012295 brambling, Fringilla montifringilla
0.0011838 indigo bunting, indigo finch, indigo bird, Passerina cyanea
0.0008891 goldfinch, Carduelis carduelis| densenet169 |0.9640683 junco, snowbird
0.0201314 brambling, Fringilla montifringilla
0.0044098 chickadee
0.0032345 goldfinch, Carduelis carduelis
0.0026739 water ouzel, dipper| densenet201 |0.9515268 junco, snowbird
0.0178252 water ouzel, dipper
0.0109119 brambling, Fringilla montifringilla
0.0077980 house finch, linnet, Carpodacus mexicanus
0.0044695 chickadee| -inceptionv3 || +inceptionv3 |0.9401748 junco, snowbird
0.0005783 water ouzel, dipper
0.0005624 iron, smoothing iron
0.0003841 indigo bunting, indigo finch, indigo bird, Passerina cyanea
0.0003383 cleaver, meat cleaver, chopper| mobilenet0.25|0.9301481 junco, snowbird
0.0466449 chickadee
0.0146190 brambling, Fringilla montifringilla
0.0027491 bulbul
0.0024721 jay| mobilenet0.5 |0.9290579 junco, snowbird
0.0325394 chickadee
0.0142417 water ouzel, dipper
0.0070296 brambling, Fringilla montifringilla
0.0054897 house finch, linnet, Carpodacus mexicanus| mobilenet0.75|0.9554648 junco, snowbird
0.0176056 house finch, linnet, Carpodacus mexicanus
0.0163125 brambling, Fringilla montifringilla
0.0040996 chickadee
0.0029940 goldfinch, Carduelis carduelis| @@ -123,7 +123,7 @@ densenet121 |0.3022416 liner, ocean liner
0.1322481 breakwater, groin, groyn densenet161 |0.4418390 lifeboat
0.1824288 liner, ocean liner
0.0596465 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0325273 submarine, pigboat, sub, U-boat
0.0298845 dock, dockage, docking facility| densenet169 |0.2955862 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.2342375 drilling platform, offshore rig
0.0940931 liner, ocean liner
0.0876006 container ship, containership, container vessel
0.0717741 dock, dockage, docking facility| densenet201 |0.5008167 fireboat
0.0950200 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0701643 lifeboat
0.0622607 liner, ocean liner
0.0582344 container ship, containership, container vessel| -inceptionv3 || +inceptionv3 |0.3135454 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.2686124 beacon, lighthouse, beacon light, pharos
0.1458646 submarine, pigboat, sub, U-boat
0.0374827 lifeboat
0.0280400 liner, ocean liner| mobilenet0.25|0.2958905 container ship, containership, container vessel
0.2101003 drilling platform, offshore rig
0.1384616 submarine, pigboat, sub, U-boat
0.0863535 liner, ocean liner
0.0732720 beacon, lighthouse, beacon light, pharos| mobilenet0.5 |0.6135703 liner, ocean liner
0.1116956 container ship, containership, container vessel
0.0976279 submarine, pigboat, sub, U-boat
0.0407747 drilling platform, offshore rig
0.0312130 breakwater, groin, groyne, mole, bulwark, seawall, jetty| mobilenet0.75|0.1367185 pirate, pirate ship
0.1318326 container ship, containership, container vessel
0.1117856 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0865669 lifeboat
0.0837456 liner, ocean liner| From 817bacd6f3c6777826abd93a8d21b6035eca7efb Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Tue, 21 Feb 2023 16:18:24 +0300 Subject: [PATCH 17/63] Fix coding style --- src/benchmark/frameworks/mxnet/mxnet_process.py | 6 +++--- src/inference/inference_mxnet.py | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/benchmark/frameworks/mxnet/mxnet_process.py b/src/benchmark/frameworks/mxnet/mxnet_process.py index f812b9a31..95d64dd1e 100644 --- a/src/benchmark/frameworks/mxnet/mxnet_process.py +++ b/src/benchmark/frameworks/mxnet/mxnet_process.py @@ -33,9 +33,9 @@ def _fill_command_line(self): input_shape = self._test.dep_parameters.input_shape batch_size = self._test.indep_parameters.batch_size iteration = self._test.indep_parameters.iteration - if (name is not None) and \ - (model_json is None or model_json == '') and \ - (model_params is None or model_params == ''): + if ((name is not None) and + (model_json is None or model_json == '') and + (model_params is None or model_params == '')): common_params = f'-mn {name} -i {dataset} -is {input_shape} -b {batch_size} -ni {iteration}' elif (name is None) and (model_json is not None) and (model_params is not None): common_params = f'-m {model_json} -w {model_params} -i {dataset} -is {input_shape} -b {batch_size} -ni {iteration}' diff --git a/src/inference/inference_mxnet.py b/src/inference/inference_mxnet.py index 8027986f7..6b978093e 100644 --- a/src/inference/inference_mxnet.py +++ b/src/inference/inference_mxnet.py @@ -178,7 +178,7 @@ def print_topk_predictions(predictions, k, file_labels): idx = int(index) probability = predictions[prediction_idx][idx] category = categories[idx] - log.info('\t{:.7f} {}'.format(probability.asscalar(), category)) + log.info('\t{0:.7f} {1}'.format(probability.asscalar(), category)) def inference_mxnet(net, num_iterations, get_slice, input_name, @@ -235,9 +235,9 @@ def main(): context = get_device_to_infer(args.device) - if (args.model_name is not None) and \ - (args.model_json is None) and \ - (args.model_params is None): + if ((args.model_name is not None) and + (args.model_json is None) and + (args.model_params is None)): net = load_network_gluon_model_zoo(args.model_name, context) elif (args.model_json is not None) and (args.model_params is not None): net = load_network_gluon(args.model_json, args.model_params, context, From 1e4ae8440c42fa321dd164c96c60967205073461 Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Tue, 21 Feb 2023 16:26:39 +0300 Subject: [PATCH 18/63] Fix coding style --- src/benchmark/frameworks/mxnet/mxnet_process.py | 12 +++++++----- src/inference/inference_mxnet.py | 6 +++--- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/benchmark/frameworks/mxnet/mxnet_process.py b/src/benchmark/frameworks/mxnet/mxnet_process.py index 95d64dd1e..9510a402d 100644 --- a/src/benchmark/frameworks/mxnet/mxnet_process.py +++ b/src/benchmark/frameworks/mxnet/mxnet_process.py @@ -33,12 +33,14 @@ def _fill_command_line(self): input_shape = self._test.dep_parameters.input_shape batch_size = self._test.indep_parameters.batch_size iteration = self._test.indep_parameters.iteration - if ((name is not None) and - (model_json is None or model_json == '') and - (model_params is None or model_params == '')): - common_params = f'-mn {name} -i {dataset} -is {input_shape} -b {batch_size} -ni {iteration}' + if ((name is not None) + and (model_json is None or model_json == '') + and (model_params is None or model_params == '')): + common_params = (f'-mn {name} -i {dataset} -is {input_shape} ' + f'-b {batch_size} -ni {iteration}') elif (name is None) and (model_json is not None) and (model_params is not None): - common_params = f'-m {model_json} -w {model_params} -i {dataset} -is {input_shape} -b {batch_size} -ni {iteration}' + common_params = (f'-m {model_json} -w {model_params} -i {dataset} ' + f'-is {input_shape} -b {batch_size} -ni {iteration}') else: raise Exception('Incorrect model parameters. Set model name or file names.') diff --git a/src/inference/inference_mxnet.py b/src/inference/inference_mxnet.py index 6b978093e..9610b3102 100644 --- a/src/inference/inference_mxnet.py +++ b/src/inference/inference_mxnet.py @@ -235,9 +235,9 @@ def main(): context = get_device_to_infer(args.device) - if ((args.model_name is not None) and - (args.model_json is None) and - (args.model_params is None)): + if ((args.model_name is not None) + and (args.model_json is None) + and (args.model_params is None)): net = load_network_gluon_model_zoo(args.model_name, context) elif (args.model_json is not None) and (args.model_params is not None): net = load_network_gluon(args.model_json, args.model_params, context, From dd0e1d4c52863831e7cf09ead28749aa9d7cad50 Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Tue, 21 Feb 2023 16:39:51 +0300 Subject: [PATCH 19/63] Add smoke test for MXNet inference --- test/smoke_test/run_bench_smoke_test.sh | 2 +- test/smoke_test/smoke_config.xml | 31 ++++++++++++++++++++++++- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/test/smoke_test/run_bench_smoke_test.sh b/test/smoke_test/run_bench_smoke_test.sh index 6e0475b48..065ee798b 100755 --- a/test/smoke_test/run_bench_smoke_test.sh +++ b/test/smoke_test/run_bench_smoke_test.sh @@ -10,6 +10,6 @@ result_file="results_benchmark.csv" python3 ../../src/benchmark/inference_benchmark.py -r $result_file --executor_type host_machine -c ./smoke_config.xml check_exit_code inference_benchmark -check_results_file $result_file 4 +check_results_file $result_file 5 exit $return_value \ No newline at end of file diff --git a/test/smoke_test/smoke_config.xml b/test/smoke_test/smoke_config.xml index 42deb5b91..ed28b6498 100644 --- a/test/smoke_test/smoke_config.xml +++ b/test/smoke_test/smoke_config.xml @@ -121,5 +121,34 @@ - + + + + classification + AlexNet + FP32 + MXNet + + + + + ImageNet + ./black_square.jpg + + + MXNet + 1 + CPU + 10 + 60 + + + data + 3 3 224 224 + True + 0.485 0.456 0.406 + 0.229 0.224 0.225 + + + From cb4e02574cb98b5dc5fcf8d29ab291b00ae30d43 Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Wed, 22 Feb 2023 07:18:28 +0300 Subject: [PATCH 20/63] Fix smoke test for MXNet inference --- .github/workflows/smoke_test.yml | 2 +- test/smoke_test/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/smoke_test.yml b/.github/workflows/smoke_test.yml index e3d0963a6..f10a83c0f 100644 --- a/.github/workflows/smoke_test.yml +++ b/.github/workflows/smoke_test.yml @@ -27,7 +27,7 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install -r requirements.txt - python -m pip install openvino-dev[caffe,tensorflow2]==2022.2.0 + python -m pip install openvino-dev[caffe,mxnet,tensorflow2]==2022.2.0 - name: Run smoke test for inference benchmark run: cd test/smoke_test && ./run_bench_smoke_test.sh diff --git a/test/smoke_test/README.md b/test/smoke_test/README.md index ba9968b56..ebb0de09b 100644 --- a/test/smoke_test/README.md +++ b/test/smoke_test/README.md @@ -7,5 +7,5 @@ ./run_smoke_test.sh ``` This script performs download and convert steps for test models, runs inference_benchmark.py on simple config, and checks result file: - * There are 3 tests + * There are 5 tests * All the tests are passed From 08a965816002e411e7bab19ceea3d4e920a6dc01 Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Wed, 22 Feb 2023 07:23:35 +0300 Subject: [PATCH 21/63] Fix config for MXNet smoke test --- test/smoke_test/smoke_config.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/smoke_test/smoke_config.xml b/test/smoke_test/smoke_config.xml index ed28b6498..9576a10e6 100644 --- a/test/smoke_test/smoke_config.xml +++ b/test/smoke_test/smoke_config.xml @@ -144,7 +144,7 @@ data - 3 3 224 224 + 1 3 224 224 True 0.485 0.456 0.406 0.229 0.224 0.225 From ffce6ccbd1f5e1fd474c174cb58bb46c7cbaaedb Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Wed, 22 Feb 2023 07:39:17 +0300 Subject: [PATCH 22/63] Fix readme (add MXNet) --- README.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 301741118..93afc87d9 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ DLI supports inference using the following frameworks: - [Intel® Optimization for TensorFlow][intel-tensorflow]. - [TensorFlow Lite][tensorflow-lite]. - [ONNX Runtime][onnx-runtime]. +- [MXNet][mxnet]. More information about DLI is available [here][dli-ru-web-page] (in Russian) @@ -85,6 +86,9 @@ Novgorod State University Publishing House, 2021. – 423 p. - [`validation_results_tensorflow.md`](results/validation/validation_results.md) is a table that confirms correctness of inference implementation based on Intel® Optimization for TensorFlow for several public models. + - [`validation_results_mxnet_gluon_modelzoo.md`](results/validation/validation_results_mxnet_gluon_modelzoo.md) + is a table that confirms correctness of inference implementation + based on MXNet. - [`models_checklist.md`](results/models_checklist.md) contains a list of supported deep models (in accordance with the Open Model Zoo). @@ -114,7 +118,7 @@ Novgorod State University Publishing House, 2021. – 423 p. thus provide consistent performance results. - `quantization` contains scripts to quantize model to INT8-precision using Post-Training Optimization Tool (POT) - of Intel® Distribution of OpenVINO™ toolkit. + of Intel® Distribution of OpenVINO™ toolkit. - `remote_control` contains scripts to execute benchmark remotely. - `utils` is a package of auxiliary utilities. @@ -157,8 +161,9 @@ Report questions, issues and suggestions, using: [openvino-toolkit]: https://software.intel.com/en-us/openvino-toolkit [intel-caffe]: https://github.com/intel/caffe [intel-tensorflow]: https://www.intel.com/content/www/us/en/developer/articles/guide/optimization-for-tensorflow-installation-guide.html -[tensorflow-lite]: https://www.tensorflow.org/lite/ -[onnx-runtime]: https://onnxruntime.ai/ +[tensorflow-lite]: https://www.tensorflow.org/lite +[onnx-runtime]: https://onnxruntime.ai +[mxnet]: https://mxnet.apache.org [dli-ru-web-page]: http://hpc-education.unn.ru/dli-ru [dli-web-page]: http://hpc-education.unn.ru/dli [open-model-zoo]: https://github.com/opencv/open_model_zoo From 85eb12d25f2d9c35f445d30f7917a0f40ae73e79 Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Wed, 22 Feb 2023 15:49:22 +0300 Subject: [PATCH 23/63] Fix space --- src/benchmark/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/benchmark/README.md b/src/benchmark/README.md index 712d4e366..f1635725b 100644 --- a/src/benchmark/README.md +++ b/src/benchmark/README.md @@ -111,7 +111,7 @@ Inference Engine предоставляет 2 программных интер Intel® Optimization for TensorFlow, TensorFlow Lite и MXNet осуществляется последовательный и независимый запуск запросов. Запуск очередного запроса выполняется после завершения предыдущего. Для каждого запроса осуществляется -замервремени его выполнения. Для множества полученных времен определяется +замер времени его выполнения. Для множества полученных времен определяется стандартное среднеквадратичное отклонение, и отбрасываются времена, выходящие за пределы трех стандартных отклонений от среднего времени вывода. Результирующий набор времен используется для вычисления показателя латентности. From 3a064586c9f3b4ad0e42918def928066ccd3e55b Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Thu, 23 Feb 2023 16:50:07 +0300 Subject: [PATCH 24/63] Replace mxnet.gluon by gluoncv --- src/inference/inference_mxnet.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/inference/inference_mxnet.py b/src/inference/inference_mxnet.py index 9610b3102..0f024e9c7 100644 --- a/src/inference/inference_mxnet.py +++ b/src/inference/inference_mxnet.py @@ -7,7 +7,7 @@ import numpy as np import mxnet -import mxnet.gluon.model_zoo.vision as model_zoo +import gluoncv import postprocessing_data as pp from io_adapter import IOAdapter @@ -139,7 +139,7 @@ def load_network_gluon(model_json, model_params, context, input_name): def load_network_gluon_model_zoo(model_name, context): log.info(f'Loading network \"{model_name}\"') - net = model_zoo.get_model(model_name, pretrained=True, ctx=context) + net = gluoncv.model_zoo.get_model(model_name, pretrained=True, ctx=context) log.info(f'Info about the network:\n{net}') From 6b355db44ab1e7cc472b8eb76194d885f2d4421b Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Thu, 23 Feb 2023 17:23:01 +0300 Subject: [PATCH 25/63] Fix smoke test --- .github/workflows/smoke_test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/smoke_test.yml b/.github/workflows/smoke_test.yml index f10a83c0f..c7213c252 100644 --- a/.github/workflows/smoke_test.yml +++ b/.github/workflows/smoke_test.yml @@ -28,6 +28,7 @@ jobs: python -m pip install --upgrade pip python -m pip install -r requirements.txt python -m pip install openvino-dev[caffe,mxnet,tensorflow2]==2022.2.0 + python -m pip install gluoncv - name: Run smoke test for inference benchmark run: cd test/smoke_test && ./run_bench_smoke_test.sh From e7fc95c9b5e5f11b37d4c13ccb3989de569f7935 Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Fri, 24 Feb 2023 15:37:26 +0300 Subject: [PATCH 26/63] Update valudation table --- ...validation_results_mxnet_gluon_modelzoo.md | 286 +++++++++++------- 1 file changed, 185 insertions(+), 101 deletions(-) diff --git a/results/validation/validation_results_mxnet_gluon_modelzoo.md b/results/validation/validation_results_mxnet_gluon_modelzoo.md index 8d83e9c86..c4a87a1d7 100644 --- a/results/validation/validation_results_mxnet_gluon_modelzoo.md +++ b/results/validation/validation_results_mxnet_gluon_modelzoo.md @@ -22,40 +22,68 @@ Standard deviation: [0.229, 0.224, 0.225] Model | Python (implementation) | -------------|---------------------------| -alexnet |0.4499783 Granny Smith
0.0933101 dumbbell
0.0876728 ocarina, sweet potato
0.0628702 hair slide
0.0484683 bottlecap| -densenet121 |0.9523346 Granny Smith
0.0132273 orange
0.0125171 lemon
0.0027910 banana
0.0020333 piggy bank, penny bank| +alexnet |0.4499776 Granny Smith
0.0933101 dumbbell
0.0876726 ocarina, sweet potato
0.0628703 hair slide
0.0484683 bottlecap| +densenet121 |0.9523344 Granny Smith
0.0132273 orange
0.0125171 lemon
0.0027910 banana
0.0020333 piggy bank, penny bank| densenet161 |0.9372969 Granny Smith
0.0082274 dumbbell
0.0056475 piggy bank, penny bank
0.0055374 ping-pong ball
0.0041915 pitcher, ewer| -densenet169 |0.9811633 Granny Smith
0.0033828 piggy bank, penny bank
0.0021365 orange
0.0019196 lemon
0.0017232 pomegranate| -densenet201 |0.9119796 Granny Smith
0.0533456 piggy bank, penny bank
0.0056831 lemon
0.0017810 pool table, billiard table, snooker table
0.0015689 tennis ball| -inceptionv3 |0.9884678 Granny Smith
0.0006726 bikini, two-piece
0.0005334 piggy bank, penny bank
0.0003360 pomegranate
0.0001373 pool table, billiard table, snooker table| -mobilenet0.25|0.3314433 piggy bank, penny bank
0.1333785 maraca
0.1262991 croquet ball
0.0684097 dumbbell
0.0539143 hair slide| -mobilenet0.5 |0.7425038 piggy bank, penny bank
0.0554336 saltshaker, salt shaker
0.0353432 Granny Smith
0.0217170 pencil sharpener
0.0169298 rubber eraser, rubber, pencil eraser| -mobilenet0.75|0.2680034 piggy bank, penny bank
0.2459760 Granny Smith
0.0562553 dumbbell
0.0306232 necklace
0.0305400 pitcher, ewer| -mobilenet1.0 |0.6145398 Granny Smith
0.0852871 piggy bank, penny bank
0.0347396 maraca
0.0332023 necklace
0.0145589 whistle| -mobilenetv2_0.25|0.1190805 bell pepper
0.1169647 Granny Smith
0.1004419 candle, taper, wax light
0.0989920 dumbbell
0.0918921 hair slide| -mobilenetv2_0.5 |0.2223127 Granny Smith
0.2116509 piggy bank, penny bank
0.1656679 saltshaker, salt shaker
0.0511496 teapot
0.0504719 necklace| -mobilenetv2_0.75|0.6253413 Granny Smith
0.0585538 piggy bank, penny bank
0.0260610 necklace
0.0252774 bell pepper
0.0218010 hair slide| -mobilenetv2_1.0 |0.7025163 piggy bank, penny bank
0.1278343 Granny Smith
0.0542883 hair slide
0.0226695 necklace
0.0095436 saltshaker, salt shaker| -resnet18_v1 |0.7145673 Granny Smith
0.0433349 piggy bank, penny bank
0.0343972 saltshaker, salt shaker
0.0215941 fig
0.0212160 banana| -resnet18_v2 |0.2885317 Granny Smith
0.1816196 piggy bank, penny bank
0.0722676 saltshaker, salt shaker
0.0635363 rubber eraser, rubber, pencil eraser
0.0440724 soap dispenser| -resnet34_v1 |0.5898089 piggy bank, penny bank
0.3150511 Granny Smith
0.0128028 saltshaker, salt shaker
0.0093140 candle, taper, wax light
0.0089791 pencil sharpener| -resnet34_v2 |0.5082700 Granny Smith
0.3873705 piggy bank, penny bank
0.0163602 pencil sharpener
0.0137499 saltshaker, salt shaker
0.0071418 dumbbell| -resnet50_v1 |0.7435529 Granny Smith
0.0182989 orange
0.0153789 lemon
0.0104132 vase
0.0097732 pop bottle, soda bottle| -resnet50_v2 |0.9931256 Granny Smith
0.0017001 piggy bank, penny bank
0.0007180 saltshaker, salt shaker
0.0006648 dumbbell
0.0002998 tennis ball| -resnet101_v1 |0.8556019 Granny Smith
0.0572227 piggy bank, penny bank
0.0485176 saltshaker, salt shaker
0.0053789 hair slide
0.0048319 dumbbell| -resnet101_v2 |0.8972577 Granny Smith
0.0401716 candle, taper, wax light
0.0074955 nail
0.0072345 screw
0.0056374 hair slide| -resnet152_v1 |0.9127905 Granny Smith
0.0197971 croquet ball
0.0119885 piggy bank, penny bank
0.0078466 saltshaker, salt shaker
0.0043091 analog clock| -resnet152_v2 |0.9972480 Granny Smith
0.0010769 piggy bank, penny bank
0.0002827 orange
0.0002033 pitcher, ewer
0.0001509 lemon| -squeezenet1.0 |0.3275063 piggy bank, penny bank
0.1791327 dumbbell
0.1542634 Granny Smith
0.0912991 water bottle
0.0385819 rubber eraser, rubber, pencil eraser| -squeezenet1.1 |0.5895361 piggy bank, penny bank
0.0677938 Granny Smith
0.0610649 necklace
0.0610449 lemon
0.0490913 bucket, pail| -vgg11 |0.3721453 piggy bank, penny bank
0.2952037 Granny Smith
0.1076759 tennis ball
0.0314685 soap dispenser
0.0285692 dumbbell| -vgg11_bn |0.5464050 Granny Smith
0.2313121 dumbbell
0.0658234 piggy bank, penny bank
0.0269569 tennis ball
0.0218533 teapot| -vgg13 |0.4068239 Granny Smith
0.2272185 dumbbell
0.0475026 necklace
0.0303710 maraca
0.0250665 teapot| -vgg13_bn |0.9389396 Granny Smith
0.0383620 tennis ball
0.0069443 lemon
0.0039320 orange
0.0013574 banana| -vgg16 |0.6872565 piggy bank, penny bank
0.0687328 Granny Smith
0.0588234 teapot
0.0392139 tennis ball
0.0210059 pitcher, ewer| -vgg16_bn |0.9958423 Granny Smith
0.0010665 tennis ball
0.0008365 piggy bank, penny bank
0.0004831 teapot
0.0004004 dumbbell| -vgg19 |0.6034814 Granny Smith
0.1175480 piggy bank, penny bank
0.0277911 dumbbell
0.0249205 whistle
0.0218847 teapot| -vgg19_bn |0.9881877 Granny Smith
0.0025413 piggy bank, penny bank
0.0011374 teapot
0.0009895 saltshaker, salt shaker
0.0006477 cup| +densenet169 |0.9811631 Granny Smith
0.0033828 piggy bank, penny bank
0.0021366 orange
0.0019196 lemon
0.0017232 pomegranate| +densenet201 |0.9119797 Granny Smith
0.0533454 piggy bank, penny bank
0.0056832 lemon
0.0017810 pool table, billiard table, snooker table
0.0015689 tennis ball| +googlenet |0.2217809 Granny Smith
0.2117919 piggy bank, penny bank
0.0270375 dumbbell
0.0116782 saltshaker, salt shaker
0.0108081 candle, taper, wax light| +inceptionv3 |0.9185177 Granny Smith
0.0018308 candle, taper, wax light
0.0012390 crane
0.0009742 orange
0.0007275 syringe| +mobilenet0.25|0.1413059 Granny Smith
0.0491460 necklace
0.0399561 bell pepper
0.0370546 hair slide
0.0301661 piggy bank, penny bank| +mobilenet0.5 |0.0988460 teapot
0.0556895 piggy bank, penny bank
0.0552285 saltshaker, salt shaker
0.0383402 pitcher, ewer
0.0319066 necklace| +mobilenet0.75|0.1711094 Granny Smith
0.1602635 piggy bank, penny bank
0.0857521 teapot
0.0533995 pitcher, ewer
0.0354786 soap dispenser| +mobilenet1.0 |0.4475225 Granny Smith
0.0954533 piggy bank, penny bank
0.0523449 saltshaker, salt shaker
0.0358669 pencil sharpener
0.0232730 dumbbell| +mobilenetv2_0.25|0.1189503 saltshaker, salt shaker
0.0919519 bell pepper
0.0862974 Granny Smith
0.0828691 piggy bank, penny bank
0.0527703 hair slide| +mobilenetv2_0.5 |0.1700138 hair slide
0.0890763 dumbbell
0.0777359 piggy bank, penny bank
0.0521625 saltshaker, salt shaker
0.0489330 necklace| +mobilenetv2_0.75|0.3621189 saltshaker, salt shaker
0.2135993 Granny Smith
0.1286786 piggy bank, penny bank
0.0391913 lemon
0.0134064 ocarina, sweet potato| +mobilenetv2_1.0 |0.6325442 Granny Smith
0.0556754 piggy bank, penny bank
0.0443766 lemon
0.0086360 teapot
0.0071484 vase| +mobilenetV3_large |0.7482075 Granny Smith
0.1232551 piggy bank, penny bank
0.0285252 saltshaker, salt shaker
0.0054824 lemon
0.0047922 croquet ball| +mobilenetV3_small |0.5579336 Granny Smith
0.0599188 bell pepper
0.0381997 piggy bank, penny bank
0.0303460 saltshaker, salt shaker
0.0262051 lemon| +resnet18_v1 |0.7145669 Granny Smith
0.0433350 piggy bank, penny bank
0.0343973 saltshaker, salt shaker
0.0215942 fig
0.0212160 banana| +resnet18_v2 |0.2885310 Granny Smith
0.1816195 piggy bank, penny bank
0.0722676 saltshaker, salt shaker
0.0635361 rubber eraser, rubber, pencil eraser
0.0440725 soap dispenser| +resnet18_v1b |0.9093145 Granny Smith
0.0187165 saltshaker, salt shaker
0.0117636 piggy bank, penny bank
0.0104429 pomegranate
0.0058578 perfume, essence| +resnet34_v1 |0.5898067 piggy bank, penny bank
0.3150526 Granny Smith
0.0128028 saltshaker, salt shaker
0.0093141 candle, taper, wax light
0.0089791 pencil sharpener| +resnet34_v2 |0.5082690 Granny Smith
0.3873709 piggy bank, penny bank
0.0163602 pencil sharpener
0.0137501 saltshaker, salt shaker
0.0071418 dumbbell| +resnet34_v1b |0.8592082 Granny Smith
0.0595315 piggy bank, penny bank
0.0257422 pitcher, ewer
0.0107621 saltshaker, salt shaker
0.0081604 water jug| +resnet50_v1 |0.7377543 Granny Smith
0.0241721 piggy bank, penny bank
0.0123405 lemon
0.0061283 candle, taper, wax light
0.0051573 orange| +resnet50_v2 |0.9931253 Granny Smith
0.0017001 piggy bank, penny bank
0.0007180 saltshaker, salt shaker
0.0006648 dumbbell
0.0002998 tennis ball| +resnet50_v1b |0.5989549 Granny Smith
0.1451391 piggy bank, penny bank
0.0253168 pitcher, ewer
0.0129785 candle, taper, wax light
0.0104263 saltshaker, salt shaker| +resnet50_v1b_gn || +resnet50_v1c |0.8618550 Granny Smith
0.0188060 candle, taper, wax light
0.0055579 orange
0.0039412 pitcher, ewer
0.0034311 lemon| +resnet50_v1d |0.7543250 Granny Smith
0.0171997 dumbbell
0.0116827 candle, taper, wax light
0.0074653 spindle
0.0059659 piggy bank, penny bank| +resnet101_v1 |0.8556004 Granny Smith
0.0572233 piggy bank, penny bank
0.0485183 saltshaker, salt shaker
0.0053788 hair slide
0.0048319 dumbbell| +resnet101_v2 |0.8972552 Granny Smith
0.0401729 candle, taper, wax light
0.0074956 nail
0.0072347 screw
0.0056376 hair slide| +resnet101_v1b |0.7857985 Granny Smith
0.0282332 piggy bank, penny bank
0.0049071 pool table, billiard table, snooker table
0.0044100 nail
0.0031941 orange| +resnet101_v1c |0.8046857 Granny Smith
0.0042279 lemon
0.0039940 orange
0.0035649 piggy bank, penny bank
0.0026069 pool table, billiard table, snooker table| +resnet101_v1d |0.8904037 Granny Smith
0.0013014 orange
0.0011862 candle, taper, wax light
0.0011250 lemon
0.0007383 syringe| +resnet152_v1 |0.9902509 Granny Smith
0.0005738 lemon
0.0005411 orange
0.0004003 banana
0.0001558 paper towel| +resnet152_v2 |0.9972478 Granny Smith
0.0010769 piggy bank, penny bank
0.0002827 orange
0.0002033 pitcher, ewer
0.0001509 lemon| +resnet152_v1b |0.9882135 Granny Smith
0.0015362 orange
0.0005067 lemon
0.0004334 candle, taper, wax light
0.0003944 banana| +resnet152_v1c |0.9887015 Granny Smith
0.0004545 lemon
0.0002470 orange
0.0001788 piggy bank, penny bank
0.0001443 banana| +resnet152_v1d |0.9998449 Granny Smith
0.0000251 candle, taper, wax light
0.0000094 piggy bank, penny bank
0.0000013 orange
0.0000013 nail| +resnext50_32x4d |0.5039393 Granny Smith
0.1542056 piggy bank, penny bank
0.0462400 candle, taper, wax light
0.0045215 pitcher, ewer
0.0038302 necklace| +resnext101_32x4d |0.9283170 Granny Smith
0.0027030 candle, taper, wax light
0.0021046 spindle
0.0011345 lemon
0.0008427 orange| +se_resnext50_32x4d |0.9321796 Granny Smith
0.0013750 candle, taper, wax light
0.0006428 piggy bank, penny bank
0.0004036 tennis ball
0.0004002 lemon| +se_resnext101_32x4d |0.8162102 Granny Smith
0.0103485 saltshaker, salt shaker
0.0090229 lemon
0.0089084 orange
0.0052811 candle, taper, wax light| +se_resnext101_64x4d |0.8560185 Granny Smith
0.0008338 orange
0.0006294 banana
0.0005233 syringe
0.0005156 piggy bank, penny bank| +resnest14 |0.5492676 piggy bank, penny bank
0.1353471 Granny Smith
0.0317922 candle, taper, wax light
0.0150942 teapot
0.0075024 water jug| +resnest26 |0.6255676 Granny Smith
0.0436678 piggy bank, penny bank
0.0079170 orange
0.0076749 pomegranate
0.0044506 lemon| +resnest50 |0.8672453 Granny Smith
0.0019027 pomegranate
0.0018386 orange
0.0018165 lemon
0.0016820 piggy bank, penny bank| +resnest101 |0.7884610 Granny Smith
0.0017616 orange
0.0016064 Arabian camel, dromedary, Camelus dromedarius
0.0011836 candle, taper, wax light
0.0010934 wardrobe, closet, press| +resnest200 |0.8712394 Granny Smith
0.0014679 orange
0.0013268 candle, taper, wax light
0.0011122 lemon
0.0006881 Bouvier des Flandres, Bouviers des Flandres| +resnest269 |0.8810812 Granny Smith
0.0014211 orange
0.0007651 lemon
0.0006411 neck brace
0.0005766 banana| +squeezenet1.0 |0.3275021 piggy bank, penny bank
0.1791335 dumbbell
0.1542641 Granny Smith
0.0912996 water bottle
0.0385820 rubber eraser, rubber, pencil eraser| +squeezenet1.1 |0.5895351 piggy bank, penny bank
0.0677937 Granny Smith
0.0610657 necklace
0.0610450 lemon
0.0490915 bucket, pail| +vgg11 |0.3721463 piggy bank, penny bank
0.2952025 Granny Smith
0.1076758 tennis ball
0.0314686 soap dispenser
0.0285692 dumbbell| +vgg11_bn |0.5464048 Granny Smith
0.2313122 dumbbell
0.0658233 piggy bank, penny bank
0.0269569 tennis ball
0.0218533 teapot| +vgg13 |0.4068233 Granny Smith
0.2272184 dumbbell
0.0475027 necklace
0.0303711 maraca
0.0250665 teapot| +vgg13_bn |0.9389399 Granny Smith
0.0383621 tennis ball
0.0069443 lemon
0.0039320 orange
0.0013574 banana| +vgg16 |0.6872561 piggy bank, penny bank
0.0687330 Granny Smith
0.0588234 teapot
0.0392138 tennis ball
0.0210059 pitcher, ewer| +vgg16_bn |0.9958419 Granny Smith
0.0010665 tennis ball
0.0008365 piggy bank, penny bank
0.0004831 teapot
0.0004005 dumbbell| +vgg19 |0.6034821 Granny Smith
0.1175481 piggy bank, penny bank
0.0277910 dumbbell
0.0249204 whistle
0.0218847 teapot| +vgg19_bn |0.9881874 Granny Smith
0.0025413 piggy bank, penny bank
0.0011374 teapot
0.0009895 saltshaker, salt shaker
0.0006477 cup| +xception |0.9998661 Granny Smith
0.0001228 lemon
0.0000048 piggy bank, penny bank
0.0000015 banana
0.0000010 tennis ball| +senet_154 |0.6984364 Granny Smith
0.0078498 syringe
0.0034595 tennis ball
0.0028654 banana
0.0025570 hare| ### Test image #2 @@ -69,41 +97,69 @@ Image resolution: 500 x 500
Model | Python (implementation) | ----------------------|---------------------------| -alexnet |0.9947656 junco, snowbird
0.0043087 chickadee
0.0002780 water ouzel, dipper
0.0002770 bulbul
0.0001244 brambling, Fringilla montifringilla| -densenet121 |0.9841659 junco, snowbird
0.0072199 chickadee
0.0034963 brambling, Fringilla montifringilla
0.0016226 water ouzel, dipper
0.0012858 indigo bunting, indigo finch, indigo bird, Passerina cyanea| -densenet161 |0.9932107 junco, snowbird
0.0015922 chickadee
0.0012295 brambling, Fringilla montifringilla
0.0011838 indigo bunting, indigo finch, indigo bird, Passerina cyanea
0.0008891 goldfinch, Carduelis carduelis| -densenet169 |0.9640683 junco, snowbird
0.0201314 brambling, Fringilla montifringilla
0.0044098 chickadee
0.0032345 goldfinch, Carduelis carduelis
0.0026739 water ouzel, dipper| -densenet201 |0.9515268 junco, snowbird
0.0178252 water ouzel, dipper
0.0109119 brambling, Fringilla montifringilla
0.0077980 house finch, linnet, Carpodacus mexicanus
0.0044695 chickadee| -inceptionv3 |0.9401748 junco, snowbird
0.0005783 water ouzel, dipper
0.0005624 iron, smoothing iron
0.0003841 indigo bunting, indigo finch, indigo bird, Passerina cyanea
0.0003383 cleaver, meat cleaver, chopper| -mobilenet0.25|0.9301481 junco, snowbird
0.0466449 chickadee
0.0146190 brambling, Fringilla montifringilla
0.0027491 bulbul
0.0024721 jay| -mobilenet0.5 |0.9290579 junco, snowbird
0.0325394 chickadee
0.0142417 water ouzel, dipper
0.0070296 brambling, Fringilla montifringilla
0.0054897 house finch, linnet, Carpodacus mexicanus| -mobilenet0.75|0.9554648 junco, snowbird
0.0176056 house finch, linnet, Carpodacus mexicanus
0.0163125 brambling, Fringilla montifringilla
0.0040996 chickadee
0.0029940 goldfinch, Carduelis carduelis| -mobilenet1.0 |0.9746482 junco, snowbird
0.0124388 chickadee
0.0072107 brambling, Fringilla montifringilla
0.0013413 goldfinch, Carduelis carduelis
0.0011567 house finch, linnet, Carpodacus mexicanus| -mobilenetv2_0.25|0.9198207 junco, snowbird
0.0456813 chickadee
0.0230144 house finch, linnet, Carpodacus mexicanus
0.0067220 brambling, Fringilla montifringilla
0.0012408 bulbul| -mobilenetv2_0.5 |0.9930903 junco, snowbird
0.0035467 chickadee
0.0017623 brambling, Fringilla montifringilla
0.0013532 house finch, linnet, Carpodacus mexicanus
0.0000977 goldfinch, Carduelis carduelis| -mobilenetv2_0.75|0.9800993 junco, snowbird
0.0092876 chickadee
0.0044733 brambling, Fringilla montifringilla
0.0028389 house finch, linnet, Carpodacus mexicanus
0.0017407 water ouzel, dipper| -mobilenetv2_1.0 |0.9759791 junco, snowbird
0.0164687 chickadee
0.0048956 brambling, Fringilla montifringilla
0.0009601 house finch, linnet, Carpodacus mexicanus
0.0008552 water ouzel, dipper| -resnet18_v1 |0.9597615 junco, snowbird
0.0103962 chickadee
0.0075481 goldfinch, Carduelis carduelis
0.0054580 house finch, linnet, Carpodacus mexicanus
0.0053979 water ouzel, dipper| -resnet18_v2 |0.9460657 junco, snowbird
0.0180776 brambling, Fringilla montifringilla
0.0139834 chickadee
0.0075304 goldfinch, Carduelis carduelis
0.0037932 water ouzel, dipper| -resnet34_v1 |0.9352033 junco, snowbird
0.0226503 water ouzel, dipper
0.0129960 brambling, Fringilla montifringilla
0.0050841 chickadee
0.0037434 goldfinch, Carduelis carduelis| -resnet34_v2 |0.6477917 junco, snowbird
0.0750232 water ouzel, dipper
0.0672592 brambling, Fringilla montifringilla
0.0443261 chickadee
0.0321975 goldfinch, Carduelis carduelis| -resnet50_v1 |0.9883676 junco, snowbird
0.0037137 brambling, Fringilla montifringilla
0.0027347 water ouzel, dipper
0.0025455 chickadee
0.0007354 goldfinch, Carduelis carduelis| -resnet50_v2 |0.9820375 junco, snowbird
0.0070083 water ouzel, dipper
0.0032555 chickadee
0.0021856 indigo bunting, indigo finch, indigo bird, Passerina cyanea
0.0017496 brambling, Fringilla montifringilla| -resnet101_v1 |0.9215411 junco, snowbird
0.0161462 brambling, Fringilla montifringilla
0.0113414 water ouzel, dipper
0.0096167 chickadee
0.0087583 house finch, linnet, Carpodacus mexicanus| -resnet101_v2 |0.9053509 junco, snowbird
0.0451527 water ouzel, dipper
0.0106367 chickadee
0.0089986 brambling, Fringilla montifringilla
0.0037001 goldfinch, Carduelis carduelis| -resnet152_v1 |0.9372504 junco, snowbird
0.0259636 water ouzel, dipper
0.0113407 chickadee
0.0088195 brambling, Fringilla montifringilla
0.0031848 house finch, linnet, Carpodacus mexicanus| +-------------|---------------------------| +alexnet |0.9947649 junco, snowbird
0.0043087 chickadee
0.0002780 water ouzel, dipper
0.0002770 bulbul
0.0001244 brambling, Fringilla montifringilla| +densenet121 |0.9841599 junco, snowbird
0.0072199 chickadee
0.0034962 brambling, Fringilla montifringilla
0.0016226 water ouzel, dipper
0.0012858 indigo bunting, indigo finch, indigo bird, Passerina cyanea| +densenet161 |0.9932058 junco, snowbird
0.0015922 chickadee
0.0012295 brambling, Fringilla montifringilla
0.0011838 indigo bunting, indigo finch, indigo bird, Passerina cyanea
0.0008891 goldfinch, Carduelis carduelis| +densenet169 |0.9640697 junco, snowbird
0.0201313 brambling, Fringilla montifringilla
0.0044098 chickadee
0.0032345 goldfinch, Carduelis carduelis
0.0026739 water ouzel, dipper| +densenet201 |0.9515250 junco, snowbird
0.0178252 water ouzel, dipper
0.0109119 brambling, Fringilla montifringilla
0.0077980 house finch, linnet, Carpodacus mexicanus
0.0044695 chickadee| +googlenet |0.8450467 junco, snowbird
0.0073040 brambling, Fringilla montifringilla
0.0059225 chickadee
0.0033832 goldfinch, Carduelis carduelis
0.0031529 indigo bunting, indigo finch, indigo bird, Passerina cyanea| +inceptionv3 |0.8331636 junco, snowbird
0.0094580 water ouzel, dipper
0.0049014 brambling, Fringilla montifringilla
0.0012859 chickadee
0.0010424 robin, American robin, Turdus migratorius| +mobilenet0.25|0.7976559 junco, snowbird
0.0642661 brambling, Fringilla montifringilla
0.0495699 chickadee
0.0412341 house finch, linnet, Carpodacus mexicanus
0.0166411 goldfinch, Carduelis carduelis| +mobilenet0.5 |0.6876235 junco, snowbird
0.0567821 chickadee
0.0440355 house finch, linnet, Carpodacus mexicanus
0.0327330 brambling, Fringilla montifringilla
0.0170564 hummingbird| +mobilenet0.75|0.7232150 junco, snowbird
0.0267700 brambling, Fringilla montifringilla
0.0224716 chickadee
0.0173853 water ouzel, dipper
0.0107710 house finch, linnet, Carpodacus mexicanus| +mobilenet1.0 |0.3200681 junco, snowbird
0.0977401 chickadee
0.0255691 brambling, Fringilla montifringilla
0.0200887 water ouzel, dipper
0.0052986 indigo bunting, indigo finch, indigo bird, Passerina cyanea| +mobilenetv2_0.25|0.5445739 junco, snowbird
0.1674697 chickadee
0.0738745 brambling, Fringilla montifringilla
0.0237202 house finch, linnet, Carpodacus mexicanus
0.0177469 coucal| +mobilenetv2_0.5 |0.9121019 junco, snowbird
0.0402663 chickadee
0.0144705 brambling, Fringilla montifringilla
0.0111383 house finch, linnet, Carpodacus mexicanus
0.0019353 bulbul| +mobilenetv2_0.75|0.4884860 junco, snowbird
0.0884391 chickadee
0.0189599 brambling, Fringilla montifringilla
0.0151568 water ouzel, dipper
0.0111601 hummingbird| +mobilenetv2_1.0 |0.8265611 junco, snowbird
0.0339170 chickadee
0.0146587 brambling, Fringilla montifringilla
0.0095486 water ouzel, dipper
0.0065168 hummingbird| +mobilenetV3_large |0.8610585 junco, snowbird
0.0212062 chickadee
0.0043952 house finch, linnet, Carpodacus mexicanus
0.0040245 brambling, Fringilla montifringilla
0.0034385 goldfinch, Carduelis carduelis| +mobilenetV3_small |0.4654124 junco, snowbird
0.0976697 brambling, Fringilla montifringilla
0.0351128 goldfinch, Carduelis carduelis
0.0220040 house finch, linnet, Carpodacus mexicanus
0.0132010 water ouzel, dipper| +resnet18_v1 |0.9597536 junco, snowbird
0.0103961 chickadee
0.0075481 goldfinch, Carduelis carduelis
0.0054579 house finch, linnet, Carpodacus mexicanus
0.0053979 water ouzel, dipper| +resnet18_v2 |0.9460587 junco, snowbird
0.0180775 brambling, Fringilla montifringilla
0.0139834 chickadee
0.0075303 goldfinch, Carduelis carduelis
0.0037932 water ouzel, dipper| +resnet18_v1b |0.9505776 junco, snowbird
0.0350148 brambling, Fringilla montifringilla
0.0052746 goldfinch, Carduelis carduelis
0.0044680 chickadee
0.0029356 house finch, linnet, Carpodacus mexicanus| +resnet34_v1 |0.9352032 junco, snowbird
0.0226504 water ouzel, dipper
0.0129960 brambling, Fringilla montifringilla
0.0050841 chickadee
0.0037434 goldfinch, Carduelis carduelis| +resnet34_v2 |0.6477929 junco, snowbird
0.0750227 water ouzel, dipper
0.0672589 brambling, Fringilla montifringilla
0.0443260 chickadee
0.0321975 goldfinch, Carduelis carduelis| +resnet34_v1b |0.9445685 junco, snowbird
0.0109547 water ouzel, dipper
0.0106829 goldfinch, Carduelis carduelis
0.0075599 brambling, Fringilla montifringilla
0.0043267 chickadee| +resnet50_v1 |0.8778600 junco, snowbird
0.0045333 water ouzel, dipper
0.0018932 brambling, Fringilla montifringilla
0.0016121 chickadee
0.0005472 magpie| +resnet50_v2 |0.9820374 junco, snowbird
0.0070083 water ouzel, dipper
0.0032555 chickadee
0.0021856 indigo bunting, indigo finch, indigo bird, Passerina cyanea
0.0017496 brambling, Fringilla montifringilla| +resnet50_v1b |0.9325832 junco, snowbird
0.0022530 water ouzel, dipper
0.0019862 chickadee
0.0011287 brambling, Fringilla montifringilla
0.0005958 goldfinch, Carduelis carduelis| +resnet50_v1b_gn || +resnet50_v1c |0.9273983 junco, snowbird
0.0029364 chickadee
0.0021531 water ouzel, dipper
0.0007963 brambling, Fringilla montifringilla
0.0007281 indigo bunting, indigo finch, indigo bird, Passerina cyanea| +resnet50_v1d |0.8592040 junco, snowbird
0.0012552 trilobite
0.0008265 indigo bunting, indigo finch, indigo bird, Passerina cyanea
0.0007142 brambling, Fringilla montifringilla
0.0006234 chickadee| +resnet101_v1 |0.9215414 junco, snowbird
0.0161461 brambling, Fringilla montifringilla
0.0113413 water ouzel, dipper
0.0096166 chickadee
0.0087583 house finch, linnet, Carpodacus mexicanus| +resnet101_v2 |0.9053523 junco, snowbird
0.0451527 water ouzel, dipper
0.0106367 chickadee
0.0089987 brambling, Fringilla montifringilla
0.0037001 goldfinch, Carduelis carduelis| +resnet101_v1b |0.9276263 junco, snowbird
0.0049964 water ouzel, dipper
0.0046615 chickadee
0.0030906 brambling, Fringilla montifringilla
0.0008694 goldfinch, Carduelis carduelis| +resnet101_v1c |0.8793292 junco, snowbird
0.0013544 water ouzel, dipper
0.0010500 indigo bunting, indigo finch, indigo bird, Passerina cyanea
0.0009873 chickadee
0.0009162 brambling, Fringilla montifringilla| +resnet101_v1d |0.8917584 junco, snowbird
0.0015399 water ouzel, dipper
0.0006728 indigo bunting, indigo finch, indigo bird, Passerina cyanea
0.0006276 loupe, jeweler's loupe
0.0005075 brambling, Fringilla montifringilla| +resnet152_v1 |0.9541085 junco, snowbird
0.0007439 indigo bunting, indigo finch, indigo bird, Passerina cyanea
0.0006390 brambling, Fringilla montifringilla
0.0005719 chickadee
0.0004141 water ouzel, dipper| resnet152_v2 |0.9695247 junco, snowbird
0.0054735 brambling, Fringilla montifringilla
0.0041750 water ouzel, dipper
0.0027560 goldfinch, Carduelis carduelis
0.0024612 indigo bunting, indigo finch, indigo bird, Passerina cyanea| -squeezenet1.0 |0.9904419 junco, snowbird
0.0045285 chickadee
0.0040343 brambling, Fringilla montifringilla
0.0003414 water ouzel, dipper
0.0002521 house finch, linnet, Carpodacus mexicanus| -squeezenet1.1 |0.9614601 junco, snowbird
0.0250982 chickadee
0.0040701 brambling, Fringilla montifringilla
0.0035157 goldfinch, Carduelis carduelis
0.0030858 ruffed grouse, partridge, Bonasa umbellus| +resnet152_v1b |0.9181211 junco, snowbird
0.0008047 water ouzel, dipper
0.0006730 chickadee
0.0004612 indigo bunting, indigo finch, indigo bird, Passerina cyanea
0.0004567 brambling, Fringilla montifringilla| +resnet152_v1c |0.9407274 junco, snowbird
0.0019899 water ouzel, dipper
0.0008130 chickadee
0.0008025 magpie
0.0006756 brambling, Fringilla montifringilla| +resnet152_v1d |0.9677259 junco, snowbird
0.0004674 water ouzel, dipper
0.0003674 brambling, Fringilla montifringilla
0.0003074 indigo bunting, indigo finch, indigo bird, Passerina cyanea
0.0001905 redshank, Tringa totanus| +resnext50_32x4d |0.8792271 junco, snowbird
0.0019800 water ouzel, dipper
0.0006261 brambling, Fringilla montifringilla
0.0005920 indigo bunting, indigo finch, indigo bird, Passerina cyanea
0.0005325 American coot, marsh hen, mud hen, water hen, Fulica americana| +resnext101_32x4d |0.9276594 junco, snowbird
0.0029483 water ouzel, dipper
0.0007209 chickadee
0.0006452 brambling, Fringilla montifringilla
0.0004491 indigo bunting, indigo finch, indigo bird, Passerina cyanea| +se_resnext50_32x4d |0.9159099 junco, snowbird
0.0075064 chickadee
0.0031539 water ouzel, dipper
0.0008047 brambling, Fringilla montifringilla
0.0006744 indigo bunting, indigo finch, indigo bird, Passerina cyanea| +se_resnext101_32x4d |0.8849785 junco, snowbird
0.0082190 water ouzel, dipper
0.0021819 brambling, Fringilla montifringilla
0.0016918 chickadee
0.0008970 indigo bunting, indigo finch, indigo bird, Passerina cyanea| +se_resnext101_64x4d |0.9599983 junco, snowbird
0.0001568 container ship, containership, container vessel
0.0001457 water ouzel, dipper
0.0001314 oystercatcher, oyster catcher
0.0001283 dugong, Dugong dugon| +resnest14 |0.3178357 junco, snowbird
0.0798734 chickadee
0.0207251 water ouzel, dipper
0.0119278 brambling, Fringilla montifringilla
0.0053170 goldfinch, Carduelis carduelis| +resnest26 |0.8787090 junco, snowbird
0.0026270 chickadee
0.0017752 brambling, Fringilla montifringilla
0.0011295 water ouzel, dipper
0.0008994 goldfinch, Carduelis carduelis| +resnest50 |0.8861975 junco, snowbird
0.0022739 water ouzel, dipper
0.0017978 indigo bunting, indigo finch, indigo bird, Passerina cyanea
0.0009528 brambling, Fringilla montifringilla
0.0005547 ringlet, ringlet butterfly| +resnest101 |0.8264657 junco, snowbird
0.0013493 water ouzel, dipper
0.0011776 chickadee
0.0010429 diamondback, diamondback rattlesnake, Crotalus adamanteus
0.0010088 tiger shark, Galeocerdo cuvieri| +resnest200 |0.9008964 junco, snowbird
0.0014222 water ouzel, dipper
0.0011092 indigo bunting, indigo finch, indigo bird, Passerina cyanea
0.0008399 brambling, Fringilla montifringilla
0.0007130 chickadee| +resnest269 |0.8601478 junco, snowbird
0.0010935 indigo bunting, indigo finch, indigo bird, Passerina cyanea
0.0008388 brambling, Fringilla montifringilla
0.0007640 water ouzel, dipper
0.0007627 ski mask| +squeezenet1.0 |0.9904411 junco, snowbird
0.0045287 chickadee
0.0040343 brambling, Fringilla montifringilla
0.0003414 water ouzel, dipper
0.0002521 house finch, linnet, Carpodacus mexicanus| +squeezenet1.1 |0.9614584 junco, snowbird
0.0250979 chickadee
0.0040701 brambling, Fringilla montifringilla
0.0035156 goldfinch, Carduelis carduelis
0.0030858 ruffed grouse, partridge, Bonasa umbellus| vgg11 |0.9998955 junco, snowbird
0.0000967 chickadee
0.0000043 brambling, Fringilla montifringilla
0.0000023 water ouzel, dipper
0.0000006 bulbul| -vgg11_bn |0.9994942 junco, snowbird
0.0002460 brambling, Fringilla montifringilla
0.0002328 chickadee
0.0000130 water ouzel, dipper
0.0000100 goldfinch, Carduelis carduelis| -vgg13 |0.9359032 junco, snowbird
0.0610290 chickadee
0.0012531 brambling, Fringilla montifringilla
0.0012155 water ouzel, dipper
0.0002740 bulbul| -vgg13_bn |0.9927477 junco, snowbird
0.0041162 chickadee
0.0028725 brambling, Fringilla montifringilla
0.0000676 goldfinch, Carduelis carduelis
0.0000641 house finch, linnet, Carpodacus mexicanus| -vgg16 |0.8946120 junco, snowbird
0.0953836 chickadee
0.0077338 brambling, Fringilla montifringilla
0.0018954 water ouzel, dipper
0.0001777 bulbul| -vgg16_bn |0.9928787 junco, snowbird
0.0052144 chickadee
0.0011600 brambling, Fringilla montifringilla
0.0005868 water ouzel, dipper
0.0000735 house finch, linnet, Carpodacus mexicanus| -vgg19 |0.9538233 junco, snowbird
0.0420003 chickadee
0.0040804 water ouzel, dipper
0.0000727 brambling, Fringilla montifringilla
0.0000097 bulbul| -vgg19_bn |0.9974002 junco, snowbird
0.0010910 brambling, Fringilla montifringilla
0.0008814 chickadee
0.0004659 water ouzel, dipper
0.0001015 goldfinch, Carduelis carduelis| +vgg11_bn |0.9994940 junco, snowbird
0.0002460 brambling, Fringilla montifringilla
0.0002328 chickadee
0.0000130 water ouzel, dipper
0.0000100 goldfinch, Carduelis carduelis| +vgg13 |0.9359031 junco, snowbird
0.0610291 chickadee
0.0012531 brambling, Fringilla montifringilla
0.0012155 water ouzel, dipper
0.0002740 bulbul| +vgg13_bn |0.9927477 junco, snowbird
0.0041163 chickadee
0.0028725 brambling, Fringilla montifringilla
0.0000676 goldfinch, Carduelis carduelis
0.0000641 house finch, linnet, Carpodacus mexicanus| +vgg16 |0.8946126 junco, snowbird
0.0953828 chickadee
0.0077339 brambling, Fringilla montifringilla
0.0018954 water ouzel, dipper
0.0001777 bulbul| +vgg16_bn |0.9928786 junco, snowbird
0.0052144 chickadee
0.0011600 brambling, Fringilla montifringilla
0.0005868 water ouzel, dipper
0.0000735 house finch, linnet, Carpodacus mexicanus| +vgg19 |0.9538229 junco, snowbird
0.0420008 chickadee
0.0040804 water ouzel, dipper
0.0000727 brambling, Fringilla montifringilla
0.0000097 bulbul| +vgg19_bn |0.9973996 junco, snowbird
0.0010910 brambling, Fringilla montifringilla
0.0008814 chickadee
0.0004659 water ouzel, dipper
0.0001015 goldfinch, Carduelis carduelis| +xception |0.9466470 junco, snowbird
0.0224749 chickadee
0.0208618 brambling, Fringilla montifringilla
0.0095579 water ouzel, dipper
0.0004126 goldfinch, Carduelis carduelis| +senet_154 |0.8654007 junco, snowbird
0.0118391 water ouzel, dipper
0.0041501 brambling, Fringilla montifringilla
0.0038129 chickadee
0.0018405 house finch, linnet, Carpodacus mexicanus| ### Test image #3 @@ -117,41 +173,69 @@ Image resolution: 333 x 500 Model | Python (implementation) | ----------------------|---------------------------| -alexnet |0.3216891 container ship, containership, container vessel
0.1360617 drilling platform, offshore rig
0.1140692 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.1057475 beacon, lighthouse, beacon light, pharos
0.0471224 liner, ocean liner| -densenet121 |0.3022416 liner, ocean liner
0.1322481 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.1194608 container ship, containership, container vessel
0.0795039 drilling platform, offshore rig
0.0723068 dock, dockage, docking facility| -densenet161 |0.4418390 lifeboat
0.1824288 liner, ocean liner
0.0596465 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0325273 submarine, pigboat, sub, U-boat
0.0298845 dock, dockage, docking facility| -densenet169 |0.2955862 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.2342375 drilling platform, offshore rig
0.0940931 liner, ocean liner
0.0876006 container ship, containership, container vessel
0.0717741 dock, dockage, docking facility| -densenet201 |0.5008167 fireboat
0.0950200 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0701643 lifeboat
0.0622607 liner, ocean liner
0.0582344 container ship, containership, container vessel| -inceptionv3 |0.3135454 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.2686124 beacon, lighthouse, beacon light, pharos
0.1458646 submarine, pigboat, sub, U-boat
0.0374827 lifeboat
0.0280400 liner, ocean liner| -mobilenet0.25|0.2958905 container ship, containership, container vessel
0.2101003 drilling platform, offshore rig
0.1384616 submarine, pigboat, sub, U-boat
0.0863535 liner, ocean liner
0.0732720 beacon, lighthouse, beacon light, pharos| -mobilenet0.5 |0.6135703 liner, ocean liner
0.1116956 container ship, containership, container vessel
0.0976279 submarine, pigboat, sub, U-boat
0.0407747 drilling platform, offshore rig
0.0312130 breakwater, groin, groyne, mole, bulwark, seawall, jetty| -mobilenet0.75|0.1367185 pirate, pirate ship
0.1318326 container ship, containership, container vessel
0.1117856 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0865669 lifeboat
0.0837456 liner, ocean liner| -mobilenet1.0 |0.4250087 pirate, pirate ship
0.1943999 container ship, containership, container vessel
0.0818260 drilling platform, offshore rig
0.0519248 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0360357 dock, dockage, docking facility| -mobilenetv2_0.25|0.4253865 liner, ocean liner
0.2140183 container ship, containership, container vessel
0.0672245 drilling platform, offshore rig
0.0430525 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0406143 aircraft carrier, carrier, flattop, attack aircraft carrier| -mobilenetv2_0.5 |0.1714408 liner, ocean liner
0.1560861 beacon, lighthouse, beacon light, pharos
0.1120990 water bottle
0.0924766 container ship, containership, container vessel
0.0835731 breakwater, groin, groyne, mole, bulwark, seawall, jetty| -mobilenetv2_0.75|0.4084314 container ship, containership, container vessel
0.1701521 liner, ocean liner
0.1168837 drilling platform, offshore rig
0.1098730 beacon, lighthouse, beacon light, pharos
0.0329633 lifeboat| -mobilenetv2_1.0 |0.2700136 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.1883821 liner, ocean liner
0.1121855 drilling platform, offshore rig
0.0574179 aircraft carrier, carrier, flattop, attack aircraft carrier
0.0394433 pirate, pirate ship| -resnet18_v1 |0.3416696 container ship, containership, container vessel
0.1224417 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.1104408 liner, ocean liner
0.0661492 lifeboat
0.0649565 pirate, pirate ship| -resnet18_v2 |0.2872612 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.1359790 beacon, lighthouse, beacon light, pharos
0.1248603 container ship, containership, container vessel
0.1217949 dock, dockage, docking facility
0.0381380 fireboat| -resnet34_v1 |0.5147033 liner, ocean liner
0.0504066 submarine, pigboat, sub, U-boat
0.0452916 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0438951 sandbar, sand bar
0.0422815 fireboat| -resnet34_v2 |0.4267507 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.2477048 container ship, containership, container vessel
0.0763716 liner, ocean liner
0.0696335 dock, dockage, docking facility
0.0447003 beacon, lighthouse, beacon light, pharos| -resnet50_v1 |0.1132128 dock, dockage, docking facility
0.0919173 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0890666 catamaran
0.0592264 seashore, coast, seacoast, sea-coast
0.0560082 container ship, containership, container vessel| -resnet50_v2 |0.4814199 container ship, containership, container vessel
0.1134713 liner, ocean liner
0.0718215 drilling platform, offshore rig
0.0571725 dock, dockage, docking facility
0.0450073 lifeboat| -resnet101_v1 |0.2196452 beacon, lighthouse, beacon light, pharos
0.1348773 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0985295 aircraft carrier, carrier, flattop, attack aircraft carrier
0.0834723 fireboat
0.0616940 drilling platform, offshore rig| -resnet101_v2 |0.2760636 pirate, pirate ship
0.1350918 wreck
0.0808768 liner, ocean liner
0.0636322 drilling platform, offshore rig
0.0605573 breakwater, groin, groyne, mole, bulwark, seawall, jetty| -resnet152_v1 |0.1550222 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.1267105 liner, ocean liner
0.1191449 lifeboat
0.1074122 beacon, lighthouse, beacon light, pharos
0.0826740 pier| -resnet152_v2 |0.1715954 drilling platform, offshore rig
0.1319755 beacon, lighthouse, beacon light, pharos
0.0740683 dock, dockage, docking facility
0.0558942 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0398145 container ship, containership, container vessel| -squeezenet1.0 |0.8105506 liner, ocean liner
0.0785145 drilling platform, offshore rig
0.0295156 container ship, containership, container vessel
0.0153660 dock, dockage, docking facility
0.0115069 submarine, pigboat, sub, U-boat| -squeezenet1.1 |0.4413084 liner, ocean liner
0.1931022 container ship, containership, container vessel
0.1459108 pirate, pirate ship
0.0937747 fireboat
0.0198683 drilling platform, offshore rig| -vgg11 |0.3343849 container ship, containership, container vessel
0.3068860 liner, ocean liner
0.0492899 submarine, pigboat, sub, U-boat
0.0455568 fireboat
0.0391509 lifeboat| -vgg11_bn |0.7272964 container ship, containership, container vessel
0.1716903 liner, ocean liner
0.0226533 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0206520 dock, dockage, docking facility
0.0114507 lifeboat| -vgg13 |0.3224934 container ship, containership, container vessel
0.2891446 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.1808191 liner, ocean liner
0.0591594 beacon, lighthouse, beacon light, pharos
0.0270379 dock, dockage, docking facility| -vgg13_bn |0.3478981 container ship, containership, container vessel
0.2664557 fireboat
0.0766566 lifeboat
0.0664669 liner, ocean liner
0.0515883 submarine, pigboat, sub, U-boat| -vgg16 |0.2839452 liner, ocean liner
0.2079168 fireboat
0.1477822 container ship, containership, container vessel
0.0909363 lifeboat
0.0704186 dock, dockage, docking facility| -vgg16_bn |0.3687426 container ship, containership, container vessel
0.3540941 liner, ocean liner
0.1349645 fireboat
0.0337696 speedboat
0.0263291 submarine, pigboat, sub, U-boat| -vgg19 |0.4434196 liner, ocean liner
0.1207930 container ship, containership, container vessel
0.0979091 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0852779 drilling platform, offshore rig
0.0730739 dock, dockage, docking facility| -vgg19_bn |0.5504534 fireboat
0.1722971 liner, ocean liner
0.0720537 container ship, containership, container vessel
0.0406803 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0388017 drilling platform, offshore rig| +-------------|---------------------------| +alexnet |0.3216886 container ship, containership, container vessel
0.1360614 drilling platform, offshore rig
0.1140693 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.1057479 beacon, lighthouse, beacon light, pharos
0.0471224 liner, ocean liner| +densenet121 |0.3022410 liner, ocean liner
0.1322484 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.1194606 container ship, containership, container vessel
0.0795041 drilling platform, offshore rig
0.0723068 dock, dockage, docking facility| +densenet161 |0.4418391 lifeboat
0.1824287 liner, ocean liner
0.0596467 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0325274 submarine, pigboat, sub, U-boat
0.0298845 dock, dockage, docking facility| +densenet169 |0.2955866 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.2342385 drilling platform, offshore rig
0.0940928 liner, ocean liner
0.0876009 container ship, containership, container vessel
0.0717737 dock, dockage, docking facility| +densenet201 |0.5008176 fireboat
0.0950196 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0701646 lifeboat
0.0622607 liner, ocean liner
0.0582344 container ship, containership, container vessel| +googlenet |0.0838070 lifeboat
0.0731668 container ship, containership, container vessel
0.0730509 liner, ocean liner
0.0729539 fireboat
0.0689271 drilling platform, offshore rig| +inceptionv3 |0.1505833 dock, dockage, docking facility
0.1179259 drilling platform, offshore rig
0.1058787 liner, ocean liner
0.0981266 container ship, containership, container vessel
0.0621234 lifeboat| +mobilenet0.25|0.1772091 water bottle
0.1686604 liner, ocean liner
0.0955276 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0937019 submarine, pigboat, sub, U-boat
0.0643868 beacon, lighthouse, beacon light, pharos| +mobilenet0.5 |0.1413260 submarine, pigboat, sub, U-boat
0.1064001 aircraft carrier, carrier, flattop, attack aircraft carrier
0.0897172 container ship, containership, container vessel
0.0690325 liner, ocean liner
0.0603464 breakwater, groin, groyne, mole, bulwark, seawall, jetty| +mobilenet0.75|0.1406201 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0317071 liner, ocean liner
0.0312595 beacon, lighthouse, beacon light, pharos
0.0274736 container ship, containership, container vessel
0.0195056 fireboat| +mobilenet1.0 |0.0580719 liner, ocean liner
0.0576046 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0517998 aircraft carrier, carrier, flattop, attack aircraft carrier
0.0483820 lifeboat
0.0473302 water bottle| +mobilenetv2_0.25|0.2591519 liner, ocean liner
0.0933797 drilling platform, offshore rig
0.0799795 aircraft carrier, carrier, flattop, attack aircraft carrier
0.0531668 fireboat
0.0529574 container ship, containership, container vessel| +mobilenetv2_0.5 |0.1369159 container ship, containership, container vessel
0.1195146 liner, ocean liner
0.0921707 drilling platform, offshore rig
0.0855153 submarine, pigboat, sub, U-boat
0.0492694 beacon, lighthouse, beacon light, pharos| +mobilenetv2_0.75|0.1567852 liner, ocean liner
0.0603636 lifeboat
0.0488465 pirate, pirate ship
0.0427859 container ship, containership, container vessel
0.0276454 drilling platform, offshore rig| +mobilenetv2_1.0 |0.1127411 liner, ocean liner
0.1014166 container ship, containership, container vessel
0.0582132 submarine, pigboat, sub, U-boat
0.0552070 lifeboat
0.0221046 breakwater, groin, groyne, mole, bulwark, seawall, jetty| +mobilenetV3_large |0.1392086 fireboat
0.0813295 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0483475 submarine, pigboat, sub, U-boat
0.0482562 drilling platform, offshore rig
0.0418803 aircraft carrier, carrier, flattop, attack aircraft carrier| +mobilenetV3_small |0.0824172 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0700601 drilling platform, offshore rig
0.0612908 pirate, pirate ship
0.0589956 beacon, lighthouse, beacon light, pharos
0.0458271 submarine, pigboat, sub, U-boat| +resnet18_v1 |0.3416699 container ship, containership, container vessel
0.1224417 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.1104409 liner, ocean liner
0.0661493 lifeboat
0.0649564 pirate, pirate ship| +resnet18_v2 |0.2872601 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.1359790 beacon, lighthouse, beacon light, pharos
0.1248601 container ship, containership, container vessel
0.1217950 dock, dockage, docking facility
0.0381382 fireboat| +resnet18_v1b |0.1451897 liner, ocean liner
0.1121958 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0984692 water bottle
0.0750244 pop bottle, soda bottle
0.0602022 dock, dockage, docking facility| +resnet34_v1 |0.5147043 liner, ocean liner
0.0504066 submarine, pigboat, sub, U-boat
0.0452913 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0438951 sandbar, sand bar
0.0422814 fireboat| +resnet34_v2 |0.4267518 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.2477035 container ship, containership, container vessel
0.0763714 liner, ocean liner
0.0696334 dock, dockage, docking facility
0.0447001 beacon, lighthouse, beacon light, pharos| +resnet34_v1b |0.3476998 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.1918807 liner, ocean liner
0.1346360 drilling platform, offshore rig
0.1013912 container ship, containership, container vessel
0.0379062 aircraft carrier, carrier, flattop, attack aircraft carrier| +resnet50_v1 |0.4411839 liner, ocean liner
0.0861827 container ship, containership, container vessel
0.0609572 speedboat
0.0587049 dock, dockage, docking facility
0.0369093 breakwater, groin, groyne, mole, bulwark, seawall, jetty| +resnet50_v2 |0.4814195 container ship, containership, container vessel
0.1134715 liner, ocean liner
0.0718216 drilling platform, offshore rig
0.0571725 dock, dockage, docking facility
0.0450075 lifeboat| +resnet50_v1b |0.3147515 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.1076186 beacon, lighthouse, beacon light, pharos
0.0938480 liner, ocean liner
0.0601708 container ship, containership, container vessel
0.0513268 catamaran| +resnet50_v1b_gn || +resnet50_v1c |0.1537983 liner, ocean liner
0.1531646 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.1442180 catamaran
0.0900431 container ship, containership, container vessel
0.0479482 dock, dockage, docking facility| +resnet50_v1d |0.2062457 lifeboat
0.0950574 beacon, lighthouse, beacon light, pharos
0.0893152 liner, ocean liner
0.0759133 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0445664 submarine, pigboat, sub, U-boat| +resnet101_v1 |0.2196445 beacon, lighthouse, beacon light, pharos
0.1348773 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0985300 aircraft carrier, carrier, flattop, attack aircraft carrier
0.0834722 fireboat
0.0616940 drilling platform, offshore rig| +resnet101_v2 |0.2760630 pirate, pirate ship
0.1350921 wreck
0.0808766 liner, ocean liner
0.0636320 drilling platform, offshore rig
0.0605572 breakwater, groin, groyne, mole, bulwark, seawall, jetty| +resnet101_v1b |0.0885735 beacon, lighthouse, beacon light, pharos
0.0653725 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0633635 catamaran
0.0506628 seashore, coast, seacoast, sea-coast
0.0471093 pirate, pirate ship| +resnet101_v1c |0.1898097 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.1086092 beacon, lighthouse, beacon light, pharos
0.0697590 liner, ocean liner
0.0553854 catamaran
0.0528718 dock, dockage, docking facility| +resnet101_v1d |0.3125349 beacon, lighthouse, beacon light, pharos
0.0871910 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0688296 drilling platform, offshore rig
0.0468906 liner, ocean liner
0.0428201 container ship, containership, container vessel| +resnet152_v1 |0.5388135 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.2476967 beacon, lighthouse, beacon light, pharos
0.0155439 drilling platform, offshore rig
0.0131202 fireboat
0.0130310 lifeboat| +resnet152_v2 |0.1715948 drilling platform, offshore rig
0.1319761 beacon, lighthouse, beacon light, pharos
0.0740682 dock, dockage, docking facility
0.0558942 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0398144 container ship, containership, container vessel| +resnet152_v1b |0.1880362 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.1045246 dock, dockage, docking facility
0.0919990 liner, ocean liner
0.0881177 beacon, lighthouse, beacon light, pharos
0.0596821 container ship, containership, container vessel| +resnet152_v1c |0.6195551 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0571322 drilling platform, offshore rig
0.0511749 beacon, lighthouse, beacon light, pharos
0.0398432 dock, dockage, docking facility
0.0232956 liner, ocean liner| +resnet152_v1d |0.2989854 beacon, lighthouse, beacon light, pharos
0.0665542 lifeboat
0.0560873 catamaran
0.0472852 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0300369 drilling platform, offshore rig| +resnext50_32x4d |0.2343378 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.1751438 beacon, lighthouse, beacon light, pharos
0.0791734 dock, dockage, docking facility
0.0537418 liner, ocean liner
0.0213372 container ship, containership, container vessel| +resnext101_32x4d |0.4391710 drilling platform, offshore rig
0.0688623 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0503909 beacon, lighthouse, beacon light, pharos
0.0230537 liner, ocean liner
0.0186267 container ship, containership, container vessel| +se_resnext50_32x4d |0.1261469 beacon, lighthouse, beacon light, pharos
0.1138437 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0567448 sandbar, sand bar
0.0548189 dock, dockage, docking facility
0.0368647 drilling platform, offshore rig| +se_resnext101_32x4d |0.4807236 drilling platform, offshore rig
0.0311037 beacon, lighthouse, beacon light, pharos
0.0283869 liner, ocean liner
0.0250085 lifeboat
0.0242792 breakwater, groin, groyne, mole, bulwark, seawall, jetty| +se_resnext101_64x4d |0.4892049 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.1304153 beacon, lighthouse, beacon light, pharos
0.0428700 lifeboat
0.0310130 drilling platform, offshore rig
0.0167582 container ship, containership, container vessel| +resnest14 |0.4676793 lifeboat
0.0637489 drilling platform, offshore rig
0.0437817 liner, ocean liner
0.0277586 container ship, containership, container vessel
0.0164626 breakwater, groin, groyne, mole, bulwark, seawall, jetty| +resnest26 |0.2446134 lifeboat
0.1767577 liner, ocean liner
0.1365290 container ship, containership, container vessel
0.0678219 dock, dockage, docking facility
0.0492914 breakwater, groin, groyne, mole, bulwark, seawall, jetty| +resnest50 |0.5605062 lifeboat
0.1056664 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0542563 drilling platform, offshore rig
0.0259242 liner, ocean liner
0.0175263 dock, dockage, docking facility| +resnest101 |0.6135045 lifeboat
0.1416010 beacon, lighthouse, beacon light, pharos
0.0873754 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0122695 drilling platform, offshore rig
0.0065282 fireboat| +resnest200 |0.7472307 lifeboat
0.0687387 beacon, lighthouse, beacon light, pharos
0.0341304 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0075599 fireboat
0.0058176 container ship, containership, container vessel| +resnest269 |0.7652729 lifeboat
0.0210087 liner, ocean liner
0.0207347 drilling platform, offshore rig
0.0199058 dock, dockage, docking facility
0.0144701 fireboat| +squeezenet1.0 |0.8105509 liner, ocean liner
0.0785139 drilling platform, offshore rig
0.0295157 container ship, containership, container vessel
0.0153661 dock, dockage, docking facility
0.0115070 submarine, pigboat, sub, U-boat| +squeezenet1.1 |0.4413090 liner, ocean liner
0.1931009 container ship, containership, container vessel
0.1459102 pirate, pirate ship
0.0937755 fireboat
0.0198681 drilling platform, offshore rig| +vgg11 |0.3343858 container ship, containership, container vessel
0.3068859 liner, ocean liner
0.0492899 submarine, pigboat, sub, U-boat
0.0455569 fireboat
0.0391509 lifeboat| +vgg11_bn |0.7272950 container ship, containership, container vessel
0.1716907 liner, ocean liner
0.0226532 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0206520 dock, dockage, docking facility
0.0114507 lifeboat| +vgg13 |0.3224940 container ship, containership, container vessel
0.2891446 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.1808189 liner, ocean liner
0.0591592 beacon, lighthouse, beacon light, pharos
0.0270378 dock, dockage, docking facility| +vgg13_bn |0.3478979 container ship, containership, container vessel
0.2664563 fireboat
0.0766567 lifeboat
0.0664667 liner, ocean liner
0.0515883 submarine, pigboat, sub, U-boat| +vgg16 |0.2839452 liner, ocean liner
0.2079167 fireboat
0.1477824 container ship, containership, container vessel
0.0909363 lifeboat
0.0704186 dock, dockage, docking facility| +vgg16_bn |0.3687423 container ship, containership, container vessel
0.3540938 liner, ocean liner
0.1349642 fireboat
0.0337694 speedboat
0.0263290 submarine, pigboat, sub, U-boat| +vgg19 |0.4434193 liner, ocean liner
0.1207933 container ship, containership, container vessel
0.0979090 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0852779 drilling platform, offshore rig
0.0730739 dock, dockage, docking facility| +vgg19_bn |0.5504531 fireboat
0.1722970 liner, ocean liner
0.0720538 container ship, containership, container vessel
0.0406804 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0388017 drilling platform, offshore rig| +xception |0.9181097 water bottle
0.0309580 sandbar, sand bar
0.0148073 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0098892 seashore, coast, seacoast, sea-coast
0.0050070 lifeboat| +senet_154 |0.4430478 beacon, lighthouse, beacon light, pharos
0.2592156 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0319326 lifeboat
0.0148673 seashore, coast, seacoast, sea-coast
0.0140872 submarine, pigboat, sub, U-boat| From 6d1ae6e80854f75c54342257184f33bbcf97684b Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Fri, 24 Feb 2023 15:40:01 +0300 Subject: [PATCH 27/63] Fix valudation table --- results/validation/validation_results_mxnet_gluon_modelzoo.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/results/validation/validation_results_mxnet_gluon_modelzoo.md b/results/validation/validation_results_mxnet_gluon_modelzoo.md index c4a87a1d7..029e941d5 100644 --- a/results/validation/validation_results_mxnet_gluon_modelzoo.md +++ b/results/validation/validation_results_mxnet_gluon_modelzoo.md @@ -48,7 +48,6 @@ resnet34_v1b |0.8592082 Granny Smith
0.0595315 piggy bank, penny bank
0.02 resnet50_v1 |0.7377543 Granny Smith
0.0241721 piggy bank, penny bank
0.0123405 lemon
0.0061283 candle, taper, wax light
0.0051573 orange| resnet50_v2 |0.9931253 Granny Smith
0.0017001 piggy bank, penny bank
0.0007180 saltshaker, salt shaker
0.0006648 dumbbell
0.0002998 tennis ball| resnet50_v1b |0.5989549 Granny Smith
0.1451391 piggy bank, penny bank
0.0253168 pitcher, ewer
0.0129785 candle, taper, wax light
0.0104263 saltshaker, salt shaker| -resnet50_v1b_gn || resnet50_v1c |0.8618550 Granny Smith
0.0188060 candle, taper, wax light
0.0055579 orange
0.0039412 pitcher, ewer
0.0034311 lemon| resnet50_v1d |0.7543250 Granny Smith
0.0171997 dumbbell
0.0116827 candle, taper, wax light
0.0074653 spindle
0.0059659 piggy bank, penny bank| resnet101_v1 |0.8556004 Granny Smith
0.0572233 piggy bank, penny bank
0.0485183 saltshaker, salt shaker
0.0053788 hair slide
0.0048319 dumbbell| @@ -124,7 +123,6 @@ resnet34_v1b |0.9445685 junco, snowbird
0.0109547 water ouzel, dipper
0.01 resnet50_v1 |0.8778600 junco, snowbird
0.0045333 water ouzel, dipper
0.0018932 brambling, Fringilla montifringilla
0.0016121 chickadee
0.0005472 magpie| resnet50_v2 |0.9820374 junco, snowbird
0.0070083 water ouzel, dipper
0.0032555 chickadee
0.0021856 indigo bunting, indigo finch, indigo bird, Passerina cyanea
0.0017496 brambling, Fringilla montifringilla| resnet50_v1b |0.9325832 junco, snowbird
0.0022530 water ouzel, dipper
0.0019862 chickadee
0.0011287 brambling, Fringilla montifringilla
0.0005958 goldfinch, Carduelis carduelis| -resnet50_v1b_gn || resnet50_v1c |0.9273983 junco, snowbird
0.0029364 chickadee
0.0021531 water ouzel, dipper
0.0007963 brambling, Fringilla montifringilla
0.0007281 indigo bunting, indigo finch, indigo bird, Passerina cyanea| resnet50_v1d |0.8592040 junco, snowbird
0.0012552 trilobite
0.0008265 indigo bunting, indigo finch, indigo bird, Passerina cyanea
0.0007142 brambling, Fringilla montifringilla
0.0006234 chickadee| resnet101_v1 |0.9215414 junco, snowbird
0.0161461 brambling, Fringilla montifringilla
0.0113413 water ouzel, dipper
0.0096166 chickadee
0.0087583 house finch, linnet, Carpodacus mexicanus| @@ -200,7 +198,6 @@ resnet34_v1b |0.3476998 breakwater, groin, groyne, mole, bulwark, seawall, jetty resnet50_v1 |0.4411839 liner, ocean liner
0.0861827 container ship, containership, container vessel
0.0609572 speedboat
0.0587049 dock, dockage, docking facility
0.0369093 breakwater, groin, groyne, mole, bulwark, seawall, jetty| resnet50_v2 |0.4814195 container ship, containership, container vessel
0.1134715 liner, ocean liner
0.0718216 drilling platform, offshore rig
0.0571725 dock, dockage, docking facility
0.0450075 lifeboat| resnet50_v1b |0.3147515 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.1076186 beacon, lighthouse, beacon light, pharos
0.0938480 liner, ocean liner
0.0601708 container ship, containership, container vessel
0.0513268 catamaran| -resnet50_v1b_gn || resnet50_v1c |0.1537983 liner, ocean liner
0.1531646 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.1442180 catamaran
0.0900431 container ship, containership, container vessel
0.0479482 dock, dockage, docking facility| resnet50_v1d |0.2062457 lifeboat
0.0950574 beacon, lighthouse, beacon light, pharos
0.0893152 liner, ocean liner
0.0759133 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0445664 submarine, pigboat, sub, U-boat| resnet101_v1 |0.2196445 beacon, lighthouse, beacon light, pharos
0.1348773 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0985300 aircraft carrier, carrier, flattop, attack aircraft carrier
0.0834722 fireboat
0.0616940 drilling platform, offshore rig| From 4191d6ad778372b0851d27af1bc62a11c7840fd0 Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Sat, 25 Feb 2023 15:02:43 +0300 Subject: [PATCH 28/63] Add checklist for supported MXNet models --- results/mxnet_models_checklist.md | 380 ++++++++++++++++++ ...cklist.md => openvino_models_checklist.md} | 0 2 files changed, 380 insertions(+) create mode 100644 results/mxnet_models_checklist.md rename results/{models_checklist.md => openvino_models_checklist.md} (100%) diff --git a/results/mxnet_models_checklist.md b/results/mxnet_models_checklist.md new file mode 100644 index 000000000..2968d9310 --- /dev/null +++ b/results/mxnet_models_checklist.md @@ -0,0 +1,380 @@ +# Model validation and performance analysis status for MXNet + +## Public models + +### Image classification on ImageNet + +Model | Availability in OMZ (2023.02.24)| Availability in the validation table | +-|-|-| +alexnet|+|+| +darknet53|-|-| +densenet121|+|+| +densenet161|+|+| +densenet169|+|+| +densenet201|+|+| +googlenet|+|+| +hrnet_w18_c|-|-| +hrnet_w18_small_v1_c|-|-| +hrnet_w18_small_v2_c|-|-| +hrnet_w30_c|-|-| +hrnet_w32_c|-|-| +hrnet_w40_c|-|-| +hrnet_w44_c|-|-| +hrnet_w48_c|-|-| +hrnet_w64_c|-|-| +inceptionv3|+|+| +mobilenet0.25|+|+| +mobilenet0.5|+|+| +mobilenet0.75|+|+| +mobilenet1.0|+|+| +mobilenet1.0_int8|-|-| +mobilenetv2_0.25|+|+| +mobilenetv2_0.5|+|+| +mobilenetv2_0.75|+|+| +mobilenetv2_1.0|+|+| +mobilenetv3_large|+|+| +mobilenetv3_small|+|+| +residualattentionnet128|-|-| +residualattentionnet164|-|-| +residualattentionnet200|-|-| +residualattentionnet236|-|-| +residualattentionnet452|-|-| +residualattentionnet56|-|-| +residualattentionnet92|-|-| +resnest101|+|+| +resnest14|+|+| +resnest200|+|+| +resnest26|+|+| +resnest269|+|+| +resnest50|+|+| +resnet101_v1|+|+| +resnet101_v1b|+|+| +resnet101_v1b_gn|-|-| +resnet101_v1c|+|+| +resnet101_v1d|+|+| +resnet101_v1d_0.73|-|-| +resnet101_v1d_0.76|-|-| +resnet101_v1e|-|-| +resnet101_v1s|-|-| +resnet101_v2|+|+| +resnet152_v1|+|+| +resnet152_v1b|+|+| +resnet152_v1c|+|+| +resnet152_v1d|+|+| +resnet152_v1e|-|-| +resnet152_v1s|-|-| +resnet152_v2|+|+| +resnet18_v1|+|+| +resnet18_v1b|+|+| +resnet18_v1b_0.89|-|-| +resnet18_v1b_custom|-|-| +resnet18_v2|+|+| +resnet34_v1|+|+| +resnet34_v1b|+|+| +resnet34_v2|+|+| +resnet50_v1|+|+| +resnet50_v1_int8|-|-| +resnet50_v1b|+|+| +resnet50_v1b_custom|-|-| +resnet50_v1b_gn|-|-| +resnet50_v1c|+|+| +resnet50_v1d|+|+| +resnet50_v1d_0.11|-|-| +resnet50_v1d_0.37|-|-| +resnet50_v1d_0.48|-|-| +resnet50_v1d_0.86|-|-| +resnet50_v1e|-|-| +resnet50_v1s|-|-| +resnet50_v2|+|+| +resnext101_32x4d|+|+| +resnext101_64x4d|-|-| +resnext101e_64x4d|-|-| +resnext50_32x4d|+|+| +se_resnet101_v1|-|-| +se_resnet101_v2|-|-| +se_resnet152_v1|-|-| +se_resnet152_v2|-|-| +se_resnet18_v1|-|-| +se_resnet18_v2|-|-| +se_resnet34_v1|-|-| +se_resnet34_v2|-|-| +se_resnet50_v1|-|-| +se_resnet50_v2|-|-| +se_resnext101_32x4d|+|+| +se_resnext101_64x4d|+|+| +se_resnext101e_64x4d|-|-| +se_resnext50_32x4d|+|+| +senet_154|+|+| +senet_154e|-|-| +squeezenet1.0|+|+| +squeezenet1.1|+|+| +shufflenet_v1|-|-| +shufflenet_v2|-|-| +vgg11|+|+| +vgg11_bn|+|+| +vgg13|+|+| +vgg13_bn|+|+| +vgg16|+|+| +vgg16_bn|+|+| +vgg19|+|+| +vgg19_bn|+|+| +xception|+|+| +xception71|-|-| + +# Object detection + +Model | Availability in OMZ (2023.02.24)| Availability in the validation table | +-|-|-| +custom_faster_rcnn_fpn|-|-| +center_net_dla34_coco|-|-| +center_net_dla34_dcnv2_coco|-|-| +center_net_dla34_dcnv2_voc|-|-| +center_net_dla34_voc|-|-| +center_net_mobilenetv3_large_duc_coco|-|-| +center_net_mobilenetv3_large_duc_voc|-|-| +center_net_mobilenetv3_small_duc_coco|-|-| +center_net_mobilenetv3_small_duc_voc|-|-| +center_net_resnet101_v1b_coco|-|-| +center_net_resnet101_v1b_dcnv2_coco|-|-| +center_net_resnet101_v1b_dcnv2_voc|-|-| +center_net_resnet101_v1b_voc|-|-| +center_net_resnet18_v1b_coco|-|-| +center_net_resnet18_v1b_dcnv2_coco|-|-| +center_net_resnet18_v1b_dcnv2_voc|-|-| +center_net_resnet18_v1b_voc|-|-| +center_net_resnet50_v1b_coco|-|-| +center_net_resnet50_v1b_dcnv2_coco|-|-| +center_net_resnet50_v1b_dcnv2_voc|-|-| +center_net_resnet50_v1b_voc|-|-| +doublehead_rcnn_resnet50_v1b_voc|-|-| +dla34|-|-| +faster_rcnn_fpn_resnet101_v1d_coco|-|-| +faster_rcnn_fpn_resnet50_v1b_coco|-|-| +faster_rcnn_fpn_syncbn_resnest101_coco|-|-| +faster_rcnn_fpn_syncbn_resnest269_coco|-|-| +faster_rcnn_fpn_syncbn_resnest50_coco|-|-| +faster_rcnn_fpn_syncbn_resnet101_v1d_coco|-|-| +faster_rcnn_fpn_syncbn_resnet50_v1b_coco|-|-| +faster_rcnn_resnet101_v1d_coco|-|-| +faster_rcnn_resnet101_v1d_custom|-|-| +faster_rcnn_resnet101_v1d_voc|-|-| +faster_rcnn_resnet50_v1b_coco|-|-| +faster_rcnn_resnet50_v1b_custom|-|-| +faster_rcnn_resnet50_v1b_voc|-|-| +ssd_300_mobilenet0.25_coco|-|-| +ssd_300_mobilenet0.25_custom|-|-| +ssd_300_mobilenet0.25_voc|-|-| +ssd_300_mobilenet1.0_lite_coco|-|-| +ssd_300_resnet34_v1b_coco|-|-| +ssd_300_resnet34_v1b_custom|-|-| +ssd_300_resnet34_v1b_voc|-|-| +ssd_300_vgg16_atrous_coco|-|-| +ssd_300_vgg16_atrous_custom|-|-| +ssd_300_vgg16_atrous_voc|-|-| +ssd_300_vgg16_atrous_voc_int8|-|-| +ssd_512_mobilenet1.0_coco|-|-| +ssd_512_mobilenet1.0_custom|-|-| +ssd_512_mobilenet1.0_voc|-|-| +ssd_512_mobilenet1.0_voc_int8|-|-| +ssd_512_resnet101_v2_voc|-|-| +ssd_512_resnet152_v2_voc|-|-| +ssd_512_resnet18_v1_coco|-|-| +ssd_512_resnet18_v1_voc|-|-| +ssd_512_resnet50_v1_coco|-|-| +ssd_512_resnet50_v1_custom|-|-| +ssd_512_resnet50_v1_voc|-|-| +ssd_512_resnet50_v1_voc_int8|-|-| +ssd_512_vgg16_atrous_coco|-|-| +ssd_512_vgg16_atrous_custom|-|-| +ssd_512_vgg16_atrous_voc|-|-| +ssd_512_vgg16_atrous_voc_int8|-|-| +yolo3_darknet53_coco|-|-| +yolo3_darknet53_custom|-|-| +yolo3_darknet53_voc|-|-| +yolo3_mobilenet0.25_coco|-|-| +yolo3_mobilenet0.25_custom|-|-| +yolo3_mobilenet0.25_voc|-|-| +yolo3_mobilenet1.0_coco|-|-| +yolo3_mobilenet1.0_custom|-|-| +yolo3_mobilenet1.0_voc|-|-| + +# Semantic segmentation + +Model | Availability in OMZ (2023.02.24)| Availability in the validation table | +-|-|-| +danet_resnet101_citys|-|-| +danet_resnet50_citys|-|-| +deeplab_resnest101_ade|-|-| +deeplab_resnest200_ade|-|-| +deeplab_resnest269_ade|-|-| +deeplab_resnest50_ade|-|-| +deeplab_resnet101_ade|-|-| +deeplab_resnet101_citys|-|-| +deeplab_resnet101_coco|-|-| +deeplab_resnet101_coco_int8|-|-| +deeplab_resnet101_voc|-|-| +deeplab_resnet101_voc_int8|-|-| +deeplab_resnet152_coco|-|-| +deeplab_resnet152_voc|-|-| +deeplab_resnet50_ade|-|-| +deeplab_resnet50_citys|-|-| +deeplab_v3b_plus_wideresnet_citys|-|-| +fastscnn_citys|-|-| +fcn_resnet101_ade|-|-| +fcn_resnet101_coco|-|-| +fcn_resnet101_coco_int8|-|-| +fcn_resnet101_voc|-|-| +fcn_resnet101_voc_int8|-|-| +fcn_resnet50_ade|-|-| +fcn_resnet50_voc|-|-| +hrnet_w18_small_v1_s|-|-| +hrnet_w18_small_v2_s|-|-| +hrnet_w48_s|-|-| +icnet_resnet50_citys|-|-| +icnet_resnet50_mhpv1|-|-| +nasnet_4_1056|-|-| +nasnet_5_1538|-|-| +nasnet_6_4032|-|-| +nasnet_7_1920|-|-| +psp_resnet101_ade|-|-| +psp_resnet101_citys|-|-| +psp_resnet101_coco|-|-| +psp_resnet101_coco_int8|-|-| +psp_resnet101_voc|-|-| +psp_resnet101_voc_int8|-|-| +psp_resnet50_ade|-|-| + +### Instance segmentation + +Model | Availability in OMZ (2023.02.24)| Availability in the validation table | +-|-|-| +custom_mask_rcnn_fpn|-|-| +mask_rcnn_fpn_resnet101_v1d_coco|-|-| +mask_rcnn_fpn_resnet18_v1b_coco|-|-| +mask_rcnn_fpn_resnet50_v1b_coco|-|-| +mask_rcnn_fpn_syncbn_mobilenet1_0_coco|-|-| +mask_rcnn_fpn_syncbn_resnet18_v1b_coco|-|-| +mask_rcnn_resnet101_v1d_coco|-|-| +mask_rcnn_resnet18_v1b_coco|-|-| +mask_rcnn_resnet50_v1b_coco|-|-| + +### Pose estimation + +Model | Availability in OMZ (2023.02.24)| Availability in the validation table | +-|-|-| +alpha_pose_resnet101_v1b_coco|-|-| +mobile_pose_mobilenet1.0|-|-| +mobile_pose_mobilenetv2_1.0|-|-| +mobile_pose_mobilenetv3_large|-|-| +mobile_pose_mobilenetv3_small|-|-| +mobile_pose_resnet18_v1b|-|-| +mobile_pose_resnet50_v1b|-|-| +simple_pose_resnet101_v1b|-|-| +simple_pose_resnet101_v1b_int8|-|-| +simple_pose_resnet101_v1d|-|-| +simple_pose_resnet101_v1d_int8|-|-| +simple_pose_resnet152_v1b|-|-| +simple_pose_resnet152_v1d|-|-| +simple_pose_resnet18_v1b|-|-| +simple_pose_resnet18_v1b_int8|-|-| +simple_pose_resnet50_v1b|-|-| +simple_pose_resnet50_v1b_int8|-|-| +simple_pose_resnet50_v1d|-|-| +simple_pose_resnet50_v1d_int8|-|-| + +### Action recognition + +Model | Availability in OMZ (2023.02.24)| Availability in the validation table | +-|-|-| +c3d_kinetics400|-|-| +i3d_inceptionv1_kinetics400|-|-| +i3d_inceptionv3_kinetics400|-|-| +i3d_nl10_resnet101_v1_kinetics400|-|-| +i3d_nl10_resnet50_v1_kinetics400|-|-| +i3d_nl5_resnet101_v1_kinetics400|-|-| +i3d_nl5_resnet50_v1_kinetics400|-|-| +i3d_resnet101_v1_kinetics400|-|-| +i3d_resnet50_v1_custom|-|-| +i3d_resnet50_v1_hmdb51|-|-| +i3d_resnet50_v1_kinetics400|-|-| +i3d_resnet50_v1_sthsthv2|-|-| +i3d_resnet50_v1_ucf101|-|-| +i3d_slow_resnet101_f16s4_kinetics700|-|-| +inceptionv1_hmdb51|-|-| +inceptionv1_kinetics400|-|-| +inceptionv1_sthsthv2|-|-| +inceptionv1_ucf101|-|-| +inceptionv3_hmdb51|-|-| +inceptionv3_kinetics400|-|-| +inceptionv3_kinetics400_int8|-|-| +inceptionv3_sthsthv2|-|-| +inceptionv3_ucf101|-|-| +inceptionv3_ucf101_int8|-|-| +p3d_resnet101_kinetics400|-|-| +p3d_resnet50_kinetics400|-|-| +r2plus1d_resnet101_kinetics400|-|-| +r2plus1d_resnet152_kinetics400|-|-| +r2plus1d_resnet18_kinetics400|-|-| +r2plus1d_resnet34_kinetics400|-|-| +r2plus1d_resnet50_kinetics400|-|-| +resnet101_v1b_kinetics400|-|-| +resnet101_v1b_sthsthv2|-|-| +resnet152_v1b_kinetics400|-|-| +resnet152_v1b_sthsthv2|-|-| +resnet18_v1b_kinetics400|-|-| +resnet18_v1b_kinetics400_int8|-|-| +resnet18_v1b_sthsthv2|-|-| +resnet34_v1b_kinetics400|-|-| +resnet34_v1b_sthsthv2|-|-| +resnet50_v1b_hmdb51|-|-| +resnet50_v1b_kinetics400|-|-| +resnet50_v1b_kinetics400_int8|-|-| +resnet50_v1b_sthsthv2|-|-| +resnet50_v1b_ucf101|-|-| +slowfast_16x8_resnet101_50_50_kinetics400|-|-| +slowfast_16x8_resnet101_kinetics400|-|-| +slowfast_4x16_resnet101_kinetics400|-|-| +slowfast_4x16_resnet50_custom|-|-| +slowfast_4x16_resnet50_kinetics400|-|-| +slowfast_8x8_resnet101_kinetics400|-|-| +slowfast_8x8_resnet50_kinetics400|-|-| +vgg16_hmdb51|-|-| +vgg16_kinetics400|-|-| +vgg16_sthsthv2|-|-| +vgg16_ucf101|-|-| +vgg16_ucf101_int8|-|-| + +### Depth prediction + +Model | Availability in OMZ (2023.02.24)| Availability in the validation table | +-|-|-| +monodepth2_resnet18_kitti_mono_640x192|-|-| +monodepth2_resnet18_kitti_mono_stereo_640x192|-|-| +monodepth2_resnet18_kitti_stereo_640x192|-|-| +monodepth2_resnet18_posenet_kitti_mono_640x192|-|-| +monodepth2_resnet18_posenet_kitti_mono_stereo_640x192|-|-| + +### Image classification on Cifar-10 + +Model | Availability in OMZ (2023.02.24)| Availability in the validation table | +-|-|-| +cifar_residualattentionnet452|-|-| +cifar_residualattentionnet56|-|-| +cifar_residualattentionnet92|-|-| +cifar_resnet110_v1|-|-| +cifar_resnet110_v2|-|-| +cifar_resnet20_v1|-|-| +cifar_resnet20_v2|-|-| +cifar_resnet56_v1|-|-| +cifar_resnet56_v2|-|-| +cifar_resnext29_16x64d|-|-| +cifar_resnext29_32x4d|-|-| +cifar_wideresnet16_10|-|-| +cifar_wideresnet28_10|-|-| +cifar_wideresnet40_8|-|-| + +### Object tracking + +Model | Availability in OMZ (2023.02.24)| Availability in the validation table | +-|-|-| +siamrpn_alexnet_v2_otb15|-|-| diff --git a/results/models_checklist.md b/results/openvino_models_checklist.md similarity index 100% rename from results/models_checklist.md rename to results/openvino_models_checklist.md From d2a05be177c4026afc5c939f323adf5597201e6d Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Sat, 25 Feb 2023 15:09:46 +0300 Subject: [PATCH 29/63] Fix formatting in checklist --- results/mxnet_models_checklist.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/results/mxnet_models_checklist.md b/results/mxnet_models_checklist.md index 2968d9310..9783338b6 100644 --- a/results/mxnet_models_checklist.md +++ b/results/mxnet_models_checklist.md @@ -121,7 +121,7 @@ vgg19_bn|+|+| xception|+|+| xception71|-|-| -# Object detection +### Object detection Model | Availability in OMZ (2023.02.24)| Availability in the validation table | -|-|-| From e06509e68a67ce402d82d31325f4b0c9a5fb3047 Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Sat, 25 Feb 2023 17:29:37 +0300 Subject: [PATCH 30/63] Add validated models --- results/mxnet_models_checklist.md | 89 ++++++++----------- ...validation_results_mxnet_gluon_modelzoo.md | 63 +++++++++++++ 2 files changed, 101 insertions(+), 51 deletions(-) diff --git a/results/mxnet_models_checklist.md b/results/mxnet_models_checklist.md index 9783338b6..f0993665a 100644 --- a/results/mxnet_models_checklist.md +++ b/results/mxnet_models_checklist.md @@ -7,40 +7,40 @@ Model | Availability in OMZ (2023.02.24)| Availability in the validation table | -|-|-| alexnet|+|+| -darknet53|-|-| +darknet53|+|+| densenet121|+|+| densenet161|+|+| densenet169|+|+| densenet201|+|+| googlenet|+|+| -hrnet_w18_c|-|-| -hrnet_w18_small_v1_c|-|-| -hrnet_w18_small_v2_c|-|-| -hrnet_w30_c|-|-| -hrnet_w32_c|-|-| -hrnet_w40_c|-|-| -hrnet_w44_c|-|-| -hrnet_w48_c|-|-| -hrnet_w64_c|-|-| +hrnet_w18_c|+|+| +hrnet_w18_small_v1_c|+|+| +hrnet_w18_small_v2_c|+|+| +hrnet_w30_c|+|+| +hrnet_w32_c|+|+| +hrnet_w40_c|+|+| +hrnet_w44_c|+|+| +hrnet_w48_c|+|+| +hrnet_w64_c|+|+| inceptionv3|+|+| mobilenet0.25|+|+| mobilenet0.5|+|+| mobilenet0.75|+|+| mobilenet1.0|+|+| -mobilenet1.0_int8|-|-| +mobilenet1.0_int8|+|-| mobilenetv2_0.25|+|+| mobilenetv2_0.5|+|+| mobilenetv2_0.75|+|+| mobilenetv2_1.0|+|+| mobilenetv3_large|+|+| mobilenetv3_small|+|+| -residualattentionnet128|-|-| -residualattentionnet164|-|-| -residualattentionnet200|-|-| -residualattentionnet236|-|-| -residualattentionnet452|-|-| -residualattentionnet56|-|-| -residualattentionnet92|-|-| +residualattentionnet128|+|-| +residualattentionnet164|+|-| +residualattentionnet200|+|-| +residualattentionnet236|+|-| +residualattentionnet452|+|-| +residualattentionnet56|+|-| +residualattentionnet92|+|-| resnest101|+|+| resnest14|+|+| resnest200|+|+| @@ -49,25 +49,25 @@ resnest269|+|+| resnest50|+|+| resnet101_v1|+|+| resnet101_v1b|+|+| -resnet101_v1b_gn|-|-| +resnet101_v1b_gn|+|-| resnet101_v1c|+|+| resnet101_v1d|+|+| -resnet101_v1d_0.73|-|-| -resnet101_v1d_0.76|-|-| -resnet101_v1e|-|-| -resnet101_v1s|-|-| +resnet101_v1d_0.73|+|+| +resnet101_v1d_0.76|+|+| +resnet101_v1e|+|-| +resnet101_v1s|+|+| resnet101_v2|+|+| resnet152_v1|+|+| resnet152_v1b|+|+| resnet152_v1c|+|+| resnet152_v1d|+|+| -resnet152_v1e|-|-| -resnet152_v1s|-|-| +resnet152_v1e|+|-| +resnet152_v1s|+|+| resnet152_v2|+|+| resnet18_v1|+|+| resnet18_v1b|+|+| -resnet18_v1b_0.89|-|-| -resnet18_v1b_custom|-|-| +resnet18_v1b_0.89|+|+| +resnet18_v1b_custom|+|-| resnet18_v2|+|+| resnet34_v1|+|+| resnet34_v1b|+|+| @@ -75,41 +75,29 @@ resnet34_v2|+|+| resnet50_v1|+|+| resnet50_v1_int8|-|-| resnet50_v1b|+|+| -resnet50_v1b_custom|-|-| -resnet50_v1b_gn|-|-| +resnet50_v1b_custom|+|-| +resnet50_v1b_gn|+|-| resnet50_v1c|+|+| resnet50_v1d|+|+| -resnet50_v1d_0.11|-|-| -resnet50_v1d_0.37|-|-| -resnet50_v1d_0.48|-|-| -resnet50_v1d_0.86|-|-| -resnet50_v1e|-|-| -resnet50_v1s|-|-| +resnet50_v1d_0.11|+|+| +resnet50_v1d_0.37|+|+| +resnet50_v1d_0.48|+|+| +resnet50_v1d_0.86|+|+| +resnet50_v1e|+|-| +resnet50_v1s|+|+| resnet50_v2|+|+| resnext101_32x4d|+|+| -resnext101_64x4d|-|-| -resnext101e_64x4d|-|-| +resnext101_64x4d|+|+| resnext50_32x4d|+|+| -se_resnet101_v1|-|-| -se_resnet101_v2|-|-| -se_resnet152_v1|-|-| -se_resnet152_v2|-|-| -se_resnet18_v1|-|-| -se_resnet18_v2|-|-| -se_resnet34_v1|-|-| -se_resnet34_v2|-|-| -se_resnet50_v1|-|-| -se_resnet50_v2|-|-| se_resnext101_32x4d|+|+| se_resnext101_64x4d|+|+| -se_resnext101e_64x4d|-|-| se_resnext50_32x4d|+|+| senet_154|+|+| senet_154e|-|-| squeezenet1.0|+|+| squeezenet1.1|+|+| -shufflenet_v1|-|-| -shufflenet_v2|-|-| +shufflenet_v1|+|-| +shufflenet_v2|+|-| vgg11|+|+| vgg11_bn|+|+| vgg13|+|+| @@ -119,7 +107,6 @@ vgg16_bn|+|+| vgg19|+|+| vgg19_bn|+|+| xception|+|+| -xception71|-|-| ### Object detection diff --git a/results/validation/validation_results_mxnet_gluon_modelzoo.md b/results/validation/validation_results_mxnet_gluon_modelzoo.md index 029e941d5..1cb16ba42 100644 --- a/results/validation/validation_results_mxnet_gluon_modelzoo.md +++ b/results/validation/validation_results_mxnet_gluon_modelzoo.md @@ -23,11 +23,21 @@ Standard deviation: [0.229, 0.224, 0.225] Model | Python (implementation) | -------------|---------------------------| alexnet |0.4499776 Granny Smith
0.0933101 dumbbell
0.0876726 ocarina, sweet potato
0.0628703 hair slide
0.0484683 bottlecap| +darknet53|0.5883058 Granny Smith
0.0645481 candle, taper, wax light
0.0236042 piggy bank, penny bank
0.0160968 pencil sharpener
0.0060462 vase| densenet121 |0.9523344 Granny Smith
0.0132273 orange
0.0125171 lemon
0.0027910 banana
0.0020333 piggy bank, penny bank| densenet161 |0.9372969 Granny Smith
0.0082274 dumbbell
0.0056475 piggy bank, penny bank
0.0055374 ping-pong ball
0.0041915 pitcher, ewer| densenet169 |0.9811631 Granny Smith
0.0033828 piggy bank, penny bank
0.0021366 orange
0.0019196 lemon
0.0017232 pomegranate| densenet201 |0.9119797 Granny Smith
0.0533454 piggy bank, penny bank
0.0056832 lemon
0.0017810 pool table, billiard table, snooker table
0.0015689 tennis ball| googlenet |0.2217809 Granny Smith
0.2117919 piggy bank, penny bank
0.0270375 dumbbell
0.0116782 saltshaker, salt shaker
0.0108081 candle, taper, wax light| +hrnet_w18_c|0.5520265 Granny Smith
0.3595940 piggy bank, penny bank
0.0101424 soap dispenser
0.0099333 orange
0.0080332 analog clock| +hrnet_w18_small_v1_c|0.2498852 piggy bank, penny bank
0.2331266 Granny Smith
0.0426585 saltshaker, salt shaker
0.0386379 soap dispenser
0.0379136 rubber eraser, rubber, pencil eraser| +hrnet_w18_small_v2_c|0.6248410 Granny Smith
0.1713923 piggy bank, penny bank
0.0333320 hair slide
0.0208897 dumbbell
0.0133955 lemon| +hrnet_w30_c|0.7287291 Granny Smith
0.0418399 safety pin
0.0235450 piggy bank, penny bank
0.0177420 hook, claw
0.0166939 candle, taper, wax light| +hrnet_w32_c|0.7040173 Granny Smith
0.0617449 lemon
0.0479103 orange
0.0105366 piggy bank, penny bank
0.0091770 water bottle| +hrnet_w40_c|0.8720709 Granny Smith
0.0261122 piggy bank, penny bank
0.0044007 rubber eraser, rubber, pencil eraser
0.0035225 orange
0.0033742 pencil box, pencil case| +hrnet_w44_c|0.4269260 Granny Smith
0.0949395 piggy bank, penny bank
0.0170920 lemon
0.0153640 digital clock
0.0148546 orange| +hrnet_w48_c|0.6867532 Granny Smith
0.0240540 lemon
0.0171518 tennis ball
0.0120418 orange
0.0086221 banana| +hrnet_w64_c|0.9524294 Granny Smith
0.0103368 orange
0.0050601 syringe
0.0020366 lemon
0.0015060 tennis ball| inceptionv3 |0.9185177 Granny Smith
0.0018308 candle, taper, wax light
0.0012390 crane
0.0009742 orange
0.0007275 syringe| mobilenet0.25|0.1413059 Granny Smith
0.0491460 necklace
0.0399561 bell pepper
0.0370546 hair slide
0.0301661 piggy bank, penny bank| mobilenet0.5 |0.0988460 teapot
0.0556895 piggy bank, penny bank
0.0552285 saltshaker, salt shaker
0.0383402 pitcher, ewer
0.0319066 necklace| @@ -42,6 +52,7 @@ mobilenetV3_small |0.5579336 Granny Smith
0.0599188 bell pepper
0.0381997 resnet18_v1 |0.7145669 Granny Smith
0.0433350 piggy bank, penny bank
0.0343973 saltshaker, salt shaker
0.0215942 fig
0.0212160 banana| resnet18_v2 |0.2885310 Granny Smith
0.1816195 piggy bank, penny bank
0.0722676 saltshaker, salt shaker
0.0635361 rubber eraser, rubber, pencil eraser
0.0440725 soap dispenser| resnet18_v1b |0.9093145 Granny Smith
0.0187165 saltshaker, salt shaker
0.0117636 piggy bank, penny bank
0.0104429 pomegranate
0.0058578 perfume, essence| +resnet18_v1b_0.89|0.5232244 piggy bank, penny bank
0.1546666 Granny Smith
0.1040182 saltshaker, salt shaker
0.0313044 soap dispenser
0.0295205 vase| resnet34_v1 |0.5898067 piggy bank, penny bank
0.3150526 Granny Smith
0.0128028 saltshaker, salt shaker
0.0093141 candle, taper, wax light
0.0089791 pencil sharpener| resnet34_v2 |0.5082690 Granny Smith
0.3873709 piggy bank, penny bank
0.0163602 pencil sharpener
0.0137501 saltshaker, salt shaker
0.0071418 dumbbell| resnet34_v1b |0.8592082 Granny Smith
0.0595315 piggy bank, penny bank
0.0257422 pitcher, ewer
0.0107621 saltshaker, salt shaker
0.0081604 water jug| @@ -50,18 +61,28 @@ resnet50_v2 |0.9931253 Granny Smith
0.0017001 piggy bank, penny bank
0.000 resnet50_v1b |0.5989549 Granny Smith
0.1451391 piggy bank, penny bank
0.0253168 pitcher, ewer
0.0129785 candle, taper, wax light
0.0104263 saltshaker, salt shaker| resnet50_v1c |0.8618550 Granny Smith
0.0188060 candle, taper, wax light
0.0055579 orange
0.0039412 pitcher, ewer
0.0034311 lemon| resnet50_v1d |0.7543250 Granny Smith
0.0171997 dumbbell
0.0116827 candle, taper, wax light
0.0074653 spindle
0.0059659 piggy bank, penny bank| +resnet50_v1d_0.11|0.0547115 necklace
0.0466139 hair slide
0.0440734 dumbbell
0.0317605 piggy bank, penny bank
0.0290907 bell pepper| +resnet50_v1d_0.37|0.1162345 piggy bank, penny bank
0.0520882 pitcher, ewer
0.0499831 saltshaker, salt shaker
0.0374144 Granny Smith
0.0282163 teapot| +resnet50_v1d_0.48|0.1693304 piggy bank, penny bank
0.1179273 pitcher, ewer
0.0780331 teapot
0.0759569 Granny Smith
0.0351384 soap dispenser| +resnet50_v1d_0.86|0.1912429 Granny Smith
0.1522044 vase
0.0466547 pitcher, ewer
0.0438114 piggy bank, penny bank
0.0387991 teapot| +resnet50_v1s|0.6378097 Granny Smith
0.0955320 piggy bank, penny bank
0.0187408 candle, taper, wax light
0.0173063 pencil sharpener
0.0085729 bell pepper| resnet101_v1 |0.8556004 Granny Smith
0.0572233 piggy bank, penny bank
0.0485183 saltshaker, salt shaker
0.0053788 hair slide
0.0048319 dumbbell| resnet101_v2 |0.8972552 Granny Smith
0.0401729 candle, taper, wax light
0.0074956 nail
0.0072347 screw
0.0056376 hair slide| resnet101_v1b |0.7857985 Granny Smith
0.0282332 piggy bank, penny bank
0.0049071 pool table, billiard table, snooker table
0.0044100 nail
0.0031941 orange| resnet101_v1c |0.8046857 Granny Smith
0.0042279 lemon
0.0039940 orange
0.0035649 piggy bank, penny bank
0.0026069 pool table, billiard table, snooker table| resnet101_v1d |0.8904037 Granny Smith
0.0013014 orange
0.0011862 candle, taper, wax light
0.0011250 lemon
0.0007383 syringe| +resnet101_v1d_0.73|0.8738712 Granny Smith
0.0117864 orange
0.0059453 lemon
0.0025753 banana
0.0018348 pitcher, ewer| +resnet101_v1d_0.76|0.7294459 Granny Smith
0.0140307 pool table, billiard table, snooker table
0.0060903 lemon
0.0042248 ping-pong ball
0.0035100 saltshaker, salt shaker| +resnet101_v1s|0.9420365 Granny Smith
0.0060981 piggy bank, penny bank
0.0039682 nail
0.0038415 tennis ball
0.0032590 candle, taper, wax light| resnet152_v1 |0.9902509 Granny Smith
0.0005738 lemon
0.0005411 orange
0.0004003 banana
0.0001558 paper towel| resnet152_v2 |0.9972478 Granny Smith
0.0010769 piggy bank, penny bank
0.0002827 orange
0.0002033 pitcher, ewer
0.0001509 lemon| resnet152_v1b |0.9882135 Granny Smith
0.0015362 orange
0.0005067 lemon
0.0004334 candle, taper, wax light
0.0003944 banana| resnet152_v1c |0.9887015 Granny Smith
0.0004545 lemon
0.0002470 orange
0.0001788 piggy bank, penny bank
0.0001443 banana| resnet152_v1d |0.9998449 Granny Smith
0.0000251 candle, taper, wax light
0.0000094 piggy bank, penny bank
0.0000013 orange
0.0000013 nail| +resnet152_v1s|0.9897671 Granny Smith
0.0006710 orange
0.0005886 lemon
0.0003691 banana
0.0002897 paper towel| resnext50_32x4d |0.5039393 Granny Smith
0.1542056 piggy bank, penny bank
0.0462400 candle, taper, wax light
0.0045215 pitcher, ewer
0.0038302 necklace| resnext101_32x4d |0.9283170 Granny Smith
0.0027030 candle, taper, wax light
0.0021046 spindle
0.0011345 lemon
0.0008427 orange| +resnext101_64x4d|0.8459962 Granny Smith
0.0009382 syringe
0.0007001 lemon
0.0006334 orange
0.0005562 shower curtain| se_resnext50_32x4d |0.9321796 Granny Smith
0.0013750 candle, taper, wax light
0.0006428 piggy bank, penny bank
0.0004036 tennis ball
0.0004002 lemon| se_resnext101_32x4d |0.8162102 Granny Smith
0.0103485 saltshaker, salt shaker
0.0090229 lemon
0.0089084 orange
0.0052811 candle, taper, wax light| se_resnext101_64x4d |0.8560185 Granny Smith
0.0008338 orange
0.0006294 banana
0.0005233 syringe
0.0005156 piggy bank, penny bank| @@ -98,11 +119,21 @@ Image resolution: 500 x 500 Model | Python (implementation) | -------------|---------------------------| alexnet |0.9947649 junco, snowbird
0.0043087 chickadee
0.0002780 water ouzel, dipper
0.0002770 bulbul
0.0001244 brambling, Fringilla montifringilla| +darknet53|0.8250283 junco, snowbird
0.0037126 indigo bunting, indigo finch, indigo bird, Passerina cyanea
0.0019864 brambling, Fringilla montifringilla
0.0017965 water ouzel, dipper
0.0015356 American coot, marsh hen, mud hen, water hen, Fulica americana| densenet121 |0.9841599 junco, snowbird
0.0072199 chickadee
0.0034962 brambling, Fringilla montifringilla
0.0016226 water ouzel, dipper
0.0012858 indigo bunting, indigo finch, indigo bird, Passerina cyanea| densenet161 |0.9932058 junco, snowbird
0.0015922 chickadee
0.0012295 brambling, Fringilla montifringilla
0.0011838 indigo bunting, indigo finch, indigo bird, Passerina cyanea
0.0008891 goldfinch, Carduelis carduelis| densenet169 |0.9640697 junco, snowbird
0.0201313 brambling, Fringilla montifringilla
0.0044098 chickadee
0.0032345 goldfinch, Carduelis carduelis
0.0026739 water ouzel, dipper| densenet201 |0.9515250 junco, snowbird
0.0178252 water ouzel, dipper
0.0109119 brambling, Fringilla montifringilla
0.0077980 house finch, linnet, Carpodacus mexicanus
0.0044695 chickadee| googlenet |0.8450467 junco, snowbird
0.0073040 brambling, Fringilla montifringilla
0.0059225 chickadee
0.0033832 goldfinch, Carduelis carduelis
0.0031529 indigo bunting, indigo finch, indigo bird, Passerina cyanea| +hrnet_w18_c|0.9188430 junco, snowbird
0.0296666 goldfinch, Carduelis carduelis
0.0161022 brambling, Fringilla montifringilla
0.0122709 chickadee
0.0079102 water ouzel, dipper| +hrnet_w18_small_v1_c|0.9759023 junco, snowbird
0.0106352 chickadee
0.0046215 water ouzel, dipper
0.0039147 brambling, Fringilla montifringilla
0.0010362 goldfinch, Carduelis carduelis| +hrnet_w18_small_v2_c|0.9787474 junco, snowbird
0.0064078 brambling, Fringilla montifringilla
0.0033237 chickadee
0.0029915 red-backed sandpiper, dunlin, Erolia alpina
0.0026367 goldfinch, Carduelis carduelis| +hrnet_w30_c|0.8702929 junco, snowbird
0.0613190 water ouzel, dipper
0.0193866 goldfinch, Carduelis carduelis
0.0158093 brambling, Fringilla montifringilla
0.0101078 house finch, linnet, Carpodacus mexicanus| +hrnet_w32_c|0.9775968 junco, snowbird
0.0055817 brambling, Fringilla montifringilla
0.0031635 water ouzel, dipper
0.0028534 chickadee
0.0028298 goldfinch, Carduelis carduelis| +hrnet_w40_c|0.9487124 junco, snowbird
0.0197902 brambling, Fringilla montifringilla
0.0128434 water ouzel, dipper
0.0024431 chickadee
0.0020419 red-backed sandpiper, dunlin, Erolia alpina| +hrnet_w44_c|0.9505839 junco, snowbird
0.0129691 water ouzel, dipper
0.0122351 brambling, Fringilla montifringilla
0.0042923 goldfinch, Carduelis carduelis
0.0020167 house finch, linnet, Carpodacus mexicanus| +hrnet_w48_c|0.9621018 junco, snowbird
0.0160337 water ouzel, dipper
0.0087758 chickadee
0.0063452 brambling, Fringilla montifringilla
0.0013683 red-backed sandpiper, dunlin, Erolia alpina| +hrnet_w64_c|0.9867093 junco, snowbird
0.0050804 water ouzel, dipper
0.0014207 brambling, Fringilla montifringilla
0.0012793 chickadee
0.0007804 goldfinch, Carduelis carduelis| inceptionv3 |0.8331636 junco, snowbird
0.0094580 water ouzel, dipper
0.0049014 brambling, Fringilla montifringilla
0.0012859 chickadee
0.0010424 robin, American robin, Turdus migratorius| mobilenet0.25|0.7976559 junco, snowbird
0.0642661 brambling, Fringilla montifringilla
0.0495699 chickadee
0.0412341 house finch, linnet, Carpodacus mexicanus
0.0166411 goldfinch, Carduelis carduelis| mobilenet0.5 |0.6876235 junco, snowbird
0.0567821 chickadee
0.0440355 house finch, linnet, Carpodacus mexicanus
0.0327330 brambling, Fringilla montifringilla
0.0170564 hummingbird| @@ -117,6 +148,7 @@ mobilenetV3_small |0.4654124 junco, snowbird
0.0976697 brambling, Fringilla m resnet18_v1 |0.9597536 junco, snowbird
0.0103961 chickadee
0.0075481 goldfinch, Carduelis carduelis
0.0054579 house finch, linnet, Carpodacus mexicanus
0.0053979 water ouzel, dipper| resnet18_v2 |0.9460587 junco, snowbird
0.0180775 brambling, Fringilla montifringilla
0.0139834 chickadee
0.0075303 goldfinch, Carduelis carduelis
0.0037932 water ouzel, dipper| resnet18_v1b |0.9505776 junco, snowbird
0.0350148 brambling, Fringilla montifringilla
0.0052746 goldfinch, Carduelis carduelis
0.0044680 chickadee
0.0029356 house finch, linnet, Carpodacus mexicanus| +resnet18_v1b_0.89|0.9328545 junco, snowbird
0.0446809 brambling, Fringilla montifringilla
0.0101631 house finch, linnet, Carpodacus mexicanus
0.0063039 chickadee
0.0027250 indigo bunting, indigo finch, indigo bird, Passerina cyanea| resnet34_v1 |0.9352032 junco, snowbird
0.0226504 water ouzel, dipper
0.0129960 brambling, Fringilla montifringilla
0.0050841 chickadee
0.0037434 goldfinch, Carduelis carduelis| resnet34_v2 |0.6477929 junco, snowbird
0.0750227 water ouzel, dipper
0.0672589 brambling, Fringilla montifringilla
0.0443260 chickadee
0.0321975 goldfinch, Carduelis carduelis| resnet34_v1b |0.9445685 junco, snowbird
0.0109547 water ouzel, dipper
0.0106829 goldfinch, Carduelis carduelis
0.0075599 brambling, Fringilla montifringilla
0.0043267 chickadee| @@ -125,18 +157,28 @@ resnet50_v2 |0.9820374 junco, snowbird
0.0070083 water ouzel, dipper
0.003 resnet50_v1b |0.9325832 junco, snowbird
0.0022530 water ouzel, dipper
0.0019862 chickadee
0.0011287 brambling, Fringilla montifringilla
0.0005958 goldfinch, Carduelis carduelis| resnet50_v1c |0.9273983 junco, snowbird
0.0029364 chickadee
0.0021531 water ouzel, dipper
0.0007963 brambling, Fringilla montifringilla
0.0007281 indigo bunting, indigo finch, indigo bird, Passerina cyanea| resnet50_v1d |0.8592040 junco, snowbird
0.0012552 trilobite
0.0008265 indigo bunting, indigo finch, indigo bird, Passerina cyanea
0.0007142 brambling, Fringilla montifringilla
0.0006234 chickadee| +resnet50_v1d_0.11|0.8141757 junco, snowbird
0.0476836 brambling, Fringilla montifringilla
0.0176680 house finch, linnet, Carpodacus mexicanus
0.0174768 goldfinch, Carduelis carduelis
0.0137937 chickadee| +resnet50_v1d_0.37|0.7955161 junco, snowbird
0.0112076 brambling, Fringilla montifringilla
0.0072924 house finch, linnet, Carpodacus mexicanus
0.0043970 goldfinch, Carduelis carduelis
0.0027637 indigo bunting, indigo finch, indigo bird, Passerina cyanea| +resnet50_v1d_0.48|0.8902636 junco, snowbird
0.0054964 brambling, Fringilla montifringilla
0.0040026 chickadee
0.0018189 water ouzel, dipper
0.0011725 indigo bunting, indigo finch, indigo bird, Passerina cyanea| +resnet50_v1d_0.86|0.8838070 junco, snowbird
0.0034641 chickadee
0.0034094 water ouzel, dipper
0.0015349 brambling, Fringilla montifringilla
0.0011060 trilobite| +resnet50_v1s|0.9927825 junco, snowbird
0.0009136 chickadee
0.0007485 brambling, Fringilla montifringilla
0.0006094 indigo bunting, indigo finch, indigo bird, Passerina cyanea
0.0003606 water ouzel, dipper| resnet101_v1 |0.9215414 junco, snowbird
0.0161461 brambling, Fringilla montifringilla
0.0113413 water ouzel, dipper
0.0096166 chickadee
0.0087583 house finch, linnet, Carpodacus mexicanus| resnet101_v2 |0.9053523 junco, snowbird
0.0451527 water ouzel, dipper
0.0106367 chickadee
0.0089987 brambling, Fringilla montifringilla
0.0037001 goldfinch, Carduelis carduelis| resnet101_v1b |0.9276263 junco, snowbird
0.0049964 water ouzel, dipper
0.0046615 chickadee
0.0030906 brambling, Fringilla montifringilla
0.0008694 goldfinch, Carduelis carduelis| resnet101_v1c |0.8793292 junco, snowbird
0.0013544 water ouzel, dipper
0.0010500 indigo bunting, indigo finch, indigo bird, Passerina cyanea
0.0009873 chickadee
0.0009162 brambling, Fringilla montifringilla| resnet101_v1d |0.8917584 junco, snowbird
0.0015399 water ouzel, dipper
0.0006728 indigo bunting, indigo finch, indigo bird, Passerina cyanea
0.0006276 loupe, jeweler's loupe
0.0005075 brambling, Fringilla montifringilla| +resnet101_v1d_0.73|0.8549845 junco, snowbird
0.0013958 water ouzel, dipper
0.0009261 chickadee
0.0008926 indigo bunting, indigo finch, indigo bird, Passerina cyanea
0.0007149 brambling, Fringilla montifringilla| +resnet101_v1d_0.76|0.8729864 junco, snowbird
0.0017605 water ouzel, dipper
0.0005628 chickadee
0.0004765 saltshaker, salt shaker
0.0004645 American coot, marsh hen, mud hen, water hen, Fulica americana| +resnet101_v1s|0.9577293 junco, snowbird
0.0045800 chickadee
0.0034216 indigo bunting, indigo finch, indigo bird, Passerina cyanea
0.0014765 brambling, Fringilla montifringilla
0.0012217 water ouzel, dipper| resnet152_v1 |0.9541085 junco, snowbird
0.0007439 indigo bunting, indigo finch, indigo bird, Passerina cyanea
0.0006390 brambling, Fringilla montifringilla
0.0005719 chickadee
0.0004141 water ouzel, dipper| resnet152_v2 |0.9695247 junco, snowbird
0.0054735 brambling, Fringilla montifringilla
0.0041750 water ouzel, dipper
0.0027560 goldfinch, Carduelis carduelis
0.0024612 indigo bunting, indigo finch, indigo bird, Passerina cyanea| resnet152_v1b |0.9181211 junco, snowbird
0.0008047 water ouzel, dipper
0.0006730 chickadee
0.0004612 indigo bunting, indigo finch, indigo bird, Passerina cyanea
0.0004567 brambling, Fringilla montifringilla| resnet152_v1c |0.9407274 junco, snowbird
0.0019899 water ouzel, dipper
0.0008130 chickadee
0.0008025 magpie
0.0006756 brambling, Fringilla montifringilla| resnet152_v1d |0.9677259 junco, snowbird
0.0004674 water ouzel, dipper
0.0003674 brambling, Fringilla montifringilla
0.0003074 indigo bunting, indigo finch, indigo bird, Passerina cyanea
0.0001905 redshank, Tringa totanus| +resnet152_v1s|0.9894158 junco, snowbird
0.0002875 chickadee
0.0002700 indigo bunting, indigo finch, indigo bird, Passerina cyanea
0.0002608 brambling, Fringilla montifringilla
0.0002240 American coot, marsh hen, mud hen, water hen, Fulica americana| resnext50_32x4d |0.8792271 junco, snowbird
0.0019800 water ouzel, dipper
0.0006261 brambling, Fringilla montifringilla
0.0005920 indigo bunting, indigo finch, indigo bird, Passerina cyanea
0.0005325 American coot, marsh hen, mud hen, water hen, Fulica americana| resnext101_32x4d |0.9276594 junco, snowbird
0.0029483 water ouzel, dipper
0.0007209 chickadee
0.0006452 brambling, Fringilla montifringilla
0.0004491 indigo bunting, indigo finch, indigo bird, Passerina cyanea| +resnext101_64x4d|0.8758404 junco, snowbird
0.0008728 tiger shark, Galeocerdo cuvieri
0.0005660 water ouzel, dipper
0.0005565 brambling, Fringilla montifringilla
0.0004355 indigo bunting, indigo finch, indigo bird, Passerina cyanea| se_resnext50_32x4d |0.9159099 junco, snowbird
0.0075064 chickadee
0.0031539 water ouzel, dipper
0.0008047 brambling, Fringilla montifringilla
0.0006744 indigo bunting, indigo finch, indigo bird, Passerina cyanea| se_resnext101_32x4d |0.8849785 junco, snowbird
0.0082190 water ouzel, dipper
0.0021819 brambling, Fringilla montifringilla
0.0016918 chickadee
0.0008970 indigo bunting, indigo finch, indigo bird, Passerina cyanea| se_resnext101_64x4d |0.9599983 junco, snowbird
0.0001568 container ship, containership, container vessel
0.0001457 water ouzel, dipper
0.0001314 oystercatcher, oyster catcher
0.0001283 dugong, Dugong dugon| @@ -173,11 +215,21 @@ Image resolution: 333 x 500 Model | Python (implementation) | -------------|---------------------------| alexnet |0.3216886 container ship, containership, container vessel
0.1360614 drilling platform, offshore rig
0.1140693 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.1057479 beacon, lighthouse, beacon light, pharos
0.0471224 liner, ocean liner| +darknet53|0.1329412 liner, ocean liner
0.1074721 dock, dockage, docking facility
0.1047859 drilling platform, offshore rig
0.0996367 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0419424 lifeboat| densenet121 |0.3022410 liner, ocean liner
0.1322484 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.1194606 container ship, containership, container vessel
0.0795041 drilling platform, offshore rig
0.0723068 dock, dockage, docking facility| densenet161 |0.4418391 lifeboat
0.1824287 liner, ocean liner
0.0596467 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0325274 submarine, pigboat, sub, U-boat
0.0298845 dock, dockage, docking facility| densenet169 |0.2955866 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.2342385 drilling platform, offshore rig
0.0940928 liner, ocean liner
0.0876009 container ship, containership, container vessel
0.0717737 dock, dockage, docking facility| densenet201 |0.5008176 fireboat
0.0950196 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0701646 lifeboat
0.0622607 liner, ocean liner
0.0582344 container ship, containership, container vessel| googlenet |0.0838070 lifeboat
0.0731668 container ship, containership, container vessel
0.0730509 liner, ocean liner
0.0729539 fireboat
0.0689271 drilling platform, offshore rig| +hrnet_w18_c|0.2656096 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.1616489 liner, ocean liner
0.0651337 beacon, lighthouse, beacon light, pharos
0.0627509 dock, dockage, docking facility
0.0541241 lifeboat| +hrnet_w18_small_v1_c|0.2587051 liner, ocean liner
0.2280385 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0848167 beacon, lighthouse, beacon light, pharos
0.0490974 fireboat
0.0474460 lifeboat| +hrnet_w18_small_v2_c|0.3427408 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.1242335 beacon, lighthouse, beacon light, pharos
0.0695650 liner, ocean liner
0.0620095 dock, dockage, docking facility
0.0403364 drilling platform, offshore rig| +hrnet_w30_c|0.2000750 drilling platform, offshore rig
0.1365042 liner, ocean liner
0.1290682 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0799346 catamaran
0.0686851 beacon, lighthouse, beacon light, pharos| +hrnet_w32_c|0.3427320 liner, ocean liner
0.2963618 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.1066358 container ship, containership, container vessel
0.0607251 beacon, lighthouse, beacon light, pharos
0.0566134 submarine, pigboat, sub, U-boat| +hrnet_w40_c|0.1606531 submarine, pigboat, sub, U-boat
0.1538070 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.1193006 drilling platform, offshore rig
0.1085498 dock, dockage, docking facility
0.0998095 liner, ocean liner| +hrnet_w44_c|0.6380179 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.1515551 beacon, lighthouse, beacon light, pharos
0.0499784 drilling platform, offshore rig
0.0288597 lifeboat
0.0177580 dock, dockage, docking facility| +hrnet_w48_c|0.3150678 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.1881291 lifeboat
0.1002933 liner, ocean liner
0.0560864 container ship, containership, container vessel
0.0427639 fireboat| +hrnet_w64_c|0.4080228 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.2256251 lifeboat
0.0774634 beacon, lighthouse, beacon light, pharos
0.0533113 dock, dockage, docking facility
0.0362752 liner, ocean liner| inceptionv3 |0.1505833 dock, dockage, docking facility
0.1179259 drilling platform, offshore rig
0.1058787 liner, ocean liner
0.0981266 container ship, containership, container vessel
0.0621234 lifeboat| mobilenet0.25|0.1772091 water bottle
0.1686604 liner, ocean liner
0.0955276 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0937019 submarine, pigboat, sub, U-boat
0.0643868 beacon, lighthouse, beacon light, pharos| mobilenet0.5 |0.1413260 submarine, pigboat, sub, U-boat
0.1064001 aircraft carrier, carrier, flattop, attack aircraft carrier
0.0897172 container ship, containership, container vessel
0.0690325 liner, ocean liner
0.0603464 breakwater, groin, groyne, mole, bulwark, seawall, jetty| @@ -192,6 +244,7 @@ mobilenetV3_small |0.0824172 breakwater, groin, groyne, mole, bulwark, seawall, resnet18_v1 |0.3416699 container ship, containership, container vessel
0.1224417 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.1104409 liner, ocean liner
0.0661493 lifeboat
0.0649564 pirate, pirate ship| resnet18_v2 |0.2872601 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.1359790 beacon, lighthouse, beacon light, pharos
0.1248601 container ship, containership, container vessel
0.1217950 dock, dockage, docking facility
0.0381382 fireboat| resnet18_v1b |0.1451897 liner, ocean liner
0.1121958 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0984692 water bottle
0.0750244 pop bottle, soda bottle
0.0602022 dock, dockage, docking facility| +resnet18_v1b_0.89|0.3873618 liner, ocean liner
0.1239402 container ship, containership, container vessel
0.1021361 fireboat
0.0560824 dock, dockage, docking facility
0.0553757 submarine, pigboat, sub, U-boat| resnet34_v1 |0.5147043 liner, ocean liner
0.0504066 submarine, pigboat, sub, U-boat
0.0452913 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0438951 sandbar, sand bar
0.0422814 fireboat| resnet34_v2 |0.4267518 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.2477035 container ship, containership, container vessel
0.0763714 liner, ocean liner
0.0696334 dock, dockage, docking facility
0.0447001 beacon, lighthouse, beacon light, pharos| resnet34_v1b |0.3476998 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.1918807 liner, ocean liner
0.1346360 drilling platform, offshore rig
0.1013912 container ship, containership, container vessel
0.0379062 aircraft carrier, carrier, flattop, attack aircraft carrier| @@ -200,18 +253,28 @@ resnet50_v2 |0.4814195 container ship, containership, container vessel
0.1134 resnet50_v1b |0.3147515 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.1076186 beacon, lighthouse, beacon light, pharos
0.0938480 liner, ocean liner
0.0601708 container ship, containership, container vessel
0.0513268 catamaran| resnet50_v1c |0.1537983 liner, ocean liner
0.1531646 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.1442180 catamaran
0.0900431 container ship, containership, container vessel
0.0479482 dock, dockage, docking facility| resnet50_v1d |0.2062457 lifeboat
0.0950574 beacon, lighthouse, beacon light, pharos
0.0893152 liner, ocean liner
0.0759133 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0445664 submarine, pigboat, sub, U-boat| +resnet50_v1d_0.11|0.1494648 container ship, containership, container vessel
0.0635204 pirate, pirate ship
0.0557595 drilling platform, offshore rig
0.0280262 liner, ocean liner
0.0257161 lifeboat| +resnet50_v1d_0.37|0.1321165 liner, ocean liner
0.0722743 container ship, containership, container vessel
0.0718901 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0668206 dock, dockage, docking facility
0.0600580 catamaran| +resnet50_v1d_0.48|0.2222678 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0869509 beacon, lighthouse, beacon light, pharos
0.0669275 aircraft carrier, carrier, flattop, attack aircraft carrier
0.0598991 liner, ocean liner
0.0556236 submarine, pigboat, sub, U-boat| +resnet50_v1d_0.86|0.1930386 liner, ocean liner
0.1047459 container ship, containership, container vessel
0.0845692 dock, dockage, docking facility
0.0839940 lifeboat
0.0681260 submarine, pigboat, sub, U-boat| +resnet50_v1s|0.2640891 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.2070212 lifeboat
0.0966062 dock, dockage, docking facility
0.0908102 container ship, containership, container vessel
0.0581087 beacon, lighthouse, beacon light, pharos| resnet101_v1 |0.2196445 beacon, lighthouse, beacon light, pharos
0.1348773 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0985300 aircraft carrier, carrier, flattop, attack aircraft carrier
0.0834722 fireboat
0.0616940 drilling platform, offshore rig| resnet101_v2 |0.2760630 pirate, pirate ship
0.1350921 wreck
0.0808766 liner, ocean liner
0.0636320 drilling platform, offshore rig
0.0605572 breakwater, groin, groyne, mole, bulwark, seawall, jetty| resnet101_v1b |0.0885735 beacon, lighthouse, beacon light, pharos
0.0653725 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0633635 catamaran
0.0506628 seashore, coast, seacoast, sea-coast
0.0471093 pirate, pirate ship| resnet101_v1c |0.1898097 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.1086092 beacon, lighthouse, beacon light, pharos
0.0697590 liner, ocean liner
0.0553854 catamaran
0.0528718 dock, dockage, docking facility| resnet101_v1d |0.3125349 beacon, lighthouse, beacon light, pharos
0.0871910 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0688296 drilling platform, offshore rig
0.0468906 liner, ocean liner
0.0428201 container ship, containership, container vessel| +resnet101_v1d_0.73|0.2870496 beacon, lighthouse, beacon light, pharos
0.1222293 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0537432 liner, ocean liner
0.0345260 container ship, containership, container vessel
0.0309129 water bottle| +resnet101_v1d_0.76|0.2803617 lifeboat
0.0701866 beacon, lighthouse, beacon light, pharos
0.0653023 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0499815 container ship, containership, container vessel
0.0268924 liner, ocean liner| +resnet101_v1s|0.3312930 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0998191 dock, dockage, docking facility
0.0886510 beacon, lighthouse, beacon light, pharos
0.0830901 lifeboat
0.0584256 liner, ocean liner| resnet152_v1 |0.5388135 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.2476967 beacon, lighthouse, beacon light, pharos
0.0155439 drilling platform, offshore rig
0.0131202 fireboat
0.0130310 lifeboat| resnet152_v2 |0.1715948 drilling platform, offshore rig
0.1319761 beacon, lighthouse, beacon light, pharos
0.0740682 dock, dockage, docking facility
0.0558942 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0398144 container ship, containership, container vessel| resnet152_v1b |0.1880362 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.1045246 dock, dockage, docking facility
0.0919990 liner, ocean liner
0.0881177 beacon, lighthouse, beacon light, pharos
0.0596821 container ship, containership, container vessel| resnet152_v1c |0.6195551 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0571322 drilling platform, offshore rig
0.0511749 beacon, lighthouse, beacon light, pharos
0.0398432 dock, dockage, docking facility
0.0232956 liner, ocean liner| resnet152_v1d |0.2989854 beacon, lighthouse, beacon light, pharos
0.0665542 lifeboat
0.0560873 catamaran
0.0472852 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0300369 drilling platform, offshore rig| +resnet152_v1s|0.6420261 lifeboat
0.0874910 dock, dockage, docking facility
0.0536488 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0530122 fireboat
0.0286627 beacon, lighthouse, beacon light, pharos| resnext50_32x4d |0.2343378 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.1751438 beacon, lighthouse, beacon light, pharos
0.0791734 dock, dockage, docking facility
0.0537418 liner, ocean liner
0.0213372 container ship, containership, container vessel| resnext101_32x4d |0.4391710 drilling platform, offshore rig
0.0688623 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0503909 beacon, lighthouse, beacon light, pharos
0.0230537 liner, ocean liner
0.0186267 container ship, containership, container vessel| +resnext101_64x4d|0.1538679 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0682765 drilling platform, offshore rig
0.0661227 beacon, lighthouse, beacon light, pharos
0.0637413 fireboat
0.0429024 liner, ocean liner| se_resnext50_32x4d |0.1261469 beacon, lighthouse, beacon light, pharos
0.1138437 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0567448 sandbar, sand bar
0.0548189 dock, dockage, docking facility
0.0368647 drilling platform, offshore rig| se_resnext101_32x4d |0.4807236 drilling platform, offshore rig
0.0311037 beacon, lighthouse, beacon light, pharos
0.0283869 liner, ocean liner
0.0250085 lifeboat
0.0242792 breakwater, groin, groyne, mole, bulwark, seawall, jetty| se_resnext101_64x4d |0.4892049 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.1304153 beacon, lighthouse, beacon light, pharos
0.0428700 lifeboat
0.0310130 drilling platform, offshore rig
0.0167582 container ship, containership, container vessel| From 06391aea6bb9dea4f261d677ebd305859d6f1137 Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Sat, 25 Feb 2023 17:34:30 +0300 Subject: [PATCH 31/63] Fix list of available MXNet models --- results/mxnet_models_checklist.md | 440 +++++++++++++++--------------- 1 file changed, 220 insertions(+), 220 deletions(-) diff --git a/results/mxnet_models_checklist.md b/results/mxnet_models_checklist.md index f0993665a..d7e5465f4 100644 --- a/results/mxnet_models_checklist.md +++ b/results/mxnet_models_checklist.md @@ -73,7 +73,7 @@ resnet34_v1|+|+| resnet34_v1b|+|+| resnet34_v2|+|+| resnet50_v1|+|+| -resnet50_v1_int8|-|-| +resnet50_v1_int8|+|-| resnet50_v1b|+|+| resnet50_v1b_custom|+|-| resnet50_v1b_gn|+|-| @@ -93,7 +93,7 @@ se_resnext101_32x4d|+|+| se_resnext101_64x4d|+|+| se_resnext50_32x4d|+|+| senet_154|+|+| -senet_154e|-|-| +senet_154e|+|-| squeezenet1.0|+|+| squeezenet1.1|+|+| shufflenet_v1|+|-| @@ -112,256 +112,256 @@ xception|+|+| Model | Availability in OMZ (2023.02.24)| Availability in the validation table | -|-|-| -custom_faster_rcnn_fpn|-|-| -center_net_dla34_coco|-|-| -center_net_dla34_dcnv2_coco|-|-| -center_net_dla34_dcnv2_voc|-|-| -center_net_dla34_voc|-|-| -center_net_mobilenetv3_large_duc_coco|-|-| -center_net_mobilenetv3_large_duc_voc|-|-| -center_net_mobilenetv3_small_duc_coco|-|-| -center_net_mobilenetv3_small_duc_voc|-|-| -center_net_resnet101_v1b_coco|-|-| -center_net_resnet101_v1b_dcnv2_coco|-|-| -center_net_resnet101_v1b_dcnv2_voc|-|-| -center_net_resnet101_v1b_voc|-|-| -center_net_resnet18_v1b_coco|-|-| -center_net_resnet18_v1b_dcnv2_coco|-|-| -center_net_resnet18_v1b_dcnv2_voc|-|-| -center_net_resnet18_v1b_voc|-|-| -center_net_resnet50_v1b_coco|-|-| -center_net_resnet50_v1b_dcnv2_coco|-|-| -center_net_resnet50_v1b_dcnv2_voc|-|-| -center_net_resnet50_v1b_voc|-|-| -doublehead_rcnn_resnet50_v1b_voc|-|-| -dla34|-|-| -faster_rcnn_fpn_resnet101_v1d_coco|-|-| -faster_rcnn_fpn_resnet50_v1b_coco|-|-| -faster_rcnn_fpn_syncbn_resnest101_coco|-|-| -faster_rcnn_fpn_syncbn_resnest269_coco|-|-| -faster_rcnn_fpn_syncbn_resnest50_coco|-|-| -faster_rcnn_fpn_syncbn_resnet101_v1d_coco|-|-| -faster_rcnn_fpn_syncbn_resnet50_v1b_coco|-|-| -faster_rcnn_resnet101_v1d_coco|-|-| -faster_rcnn_resnet101_v1d_custom|-|-| -faster_rcnn_resnet101_v1d_voc|-|-| -faster_rcnn_resnet50_v1b_coco|-|-| -faster_rcnn_resnet50_v1b_custom|-|-| -faster_rcnn_resnet50_v1b_voc|-|-| -ssd_300_mobilenet0.25_coco|-|-| -ssd_300_mobilenet0.25_custom|-|-| -ssd_300_mobilenet0.25_voc|-|-| -ssd_300_mobilenet1.0_lite_coco|-|-| -ssd_300_resnet34_v1b_coco|-|-| -ssd_300_resnet34_v1b_custom|-|-| -ssd_300_resnet34_v1b_voc|-|-| -ssd_300_vgg16_atrous_coco|-|-| -ssd_300_vgg16_atrous_custom|-|-| -ssd_300_vgg16_atrous_voc|-|-| -ssd_300_vgg16_atrous_voc_int8|-|-| -ssd_512_mobilenet1.0_coco|-|-| -ssd_512_mobilenet1.0_custom|-|-| -ssd_512_mobilenet1.0_voc|-|-| -ssd_512_mobilenet1.0_voc_int8|-|-| -ssd_512_resnet101_v2_voc|-|-| -ssd_512_resnet152_v2_voc|-|-| -ssd_512_resnet18_v1_coco|-|-| -ssd_512_resnet18_v1_voc|-|-| -ssd_512_resnet50_v1_coco|-|-| -ssd_512_resnet50_v1_custom|-|-| -ssd_512_resnet50_v1_voc|-|-| -ssd_512_resnet50_v1_voc_int8|-|-| -ssd_512_vgg16_atrous_coco|-|-| -ssd_512_vgg16_atrous_custom|-|-| -ssd_512_vgg16_atrous_voc|-|-| -ssd_512_vgg16_atrous_voc_int8|-|-| -yolo3_darknet53_coco|-|-| -yolo3_darknet53_custom|-|-| -yolo3_darknet53_voc|-|-| -yolo3_mobilenet0.25_coco|-|-| -yolo3_mobilenet0.25_custom|-|-| -yolo3_mobilenet0.25_voc|-|-| -yolo3_mobilenet1.0_coco|-|-| -yolo3_mobilenet1.0_custom|-|-| -yolo3_mobilenet1.0_voc|-|-| +custom_faster_rcnn_fpn|+|-| +center_net_dla34_coco|+|-| +center_net_dla34_dcnv2_coco|+|-| +center_net_dla34_dcnv2_voc|+|-| +center_net_dla34_voc|+|-| +center_net_mobilenetv3_large_duc_coco|+|-| +center_net_mobilenetv3_large_duc_voc|+|-| +center_net_mobilenetv3_small_duc_coco|+|-| +center_net_mobilenetv3_small_duc_voc|+|-| +center_net_resnet101_v1b_coco|+|-| +center_net_resnet101_v1b_dcnv2_coco|+|-| +center_net_resnet101_v1b_dcnv2_voc|+|-| +center_net_resnet101_v1b_voc|+|-| +center_net_resnet18_v1b_coco|+|-| +center_net_resnet18_v1b_dcnv2_coco|+|-| +center_net_resnet18_v1b_dcnv2_voc|+|-| +center_net_resnet18_v1b_voc|+|-| +center_net_resnet50_v1b_coco|+|-| +center_net_resnet50_v1b_dcnv2_coco|+|-| +center_net_resnet50_v1b_dcnv2_voc|+|-| +center_net_resnet50_v1b_voc|+|-| +doublehead_rcnn_resnet50_v1b_voc|+|-| +dla34|+|-| +faster_rcnn_fpn_resnet101_v1d_coco|+|-| +faster_rcnn_fpn_resnet50_v1b_coco|+|-| +faster_rcnn_fpn_syncbn_resnest101_coco|+|-| +faster_rcnn_fpn_syncbn_resnest269_coco|+|-| +faster_rcnn_fpn_syncbn_resnest50_coco|+|-| +faster_rcnn_fpn_syncbn_resnet101_v1d_coco|+|-| +faster_rcnn_fpn_syncbn_resnet50_v1b_coco|+|-| +faster_rcnn_resnet101_v1d_coco|+|-| +faster_rcnn_resnet101_v1d_custom|+|-| +faster_rcnn_resnet101_v1d_voc|+|-| +faster_rcnn_resnet50_v1b_coco|+|-| +faster_rcnn_resnet50_v1b_custom|+|-| +faster_rcnn_resnet50_v1b_voc|+|-| +ssd_300_mobilenet0.25_coco|+|-| +ssd_300_mobilenet0.25_custom|+|-| +ssd_300_mobilenet0.25_voc|+|-| +ssd_300_mobilenet1.0_lite_coco|+|-| +ssd_300_resnet34_v1b_coco|+|-| +ssd_300_resnet34_v1b_custom|+|-| +ssd_300_resnet34_v1b_voc|+|-| +ssd_300_vgg16_atrous_coco|+|-| +ssd_300_vgg16_atrous_custom|+|-| +ssd_300_vgg16_atrous_voc|+|-| +ssd_300_vgg16_atrous_voc_int8|+|-| +ssd_512_mobilenet1.0_coco|+|-| +ssd_512_mobilenet1.0_custom|+|-| +ssd_512_mobilenet1.0_voc|+|-| +ssd_512_mobilenet1.0_voc_int8|+|-| +ssd_512_resnet101_v2_voc|+|-| +ssd_512_resnet152_v2_voc|+|-| +ssd_512_resnet18_v1_coco|+|-| +ssd_512_resnet18_v1_voc|+|-| +ssd_512_resnet50_v1_coco|+|-| +ssd_512_resnet50_v1_custom|+|-| +ssd_512_resnet50_v1_voc|+|-| +ssd_512_resnet50_v1_voc_int8|+|-| +ssd_512_vgg16_atrous_coco|+|-| +ssd_512_vgg16_atrous_custom|+|-| +ssd_512_vgg16_atrous_voc|+|-| +ssd_512_vgg16_atrous_voc_int8|+|-| +yolo3_darknet53_coco|+|-| +yolo3_darknet53_custom|+|-| +yolo3_darknet53_voc|+|-| +yolo3_mobilenet0.25_coco|+|-| +yolo3_mobilenet0.25_custom|+|-| +yolo3_mobilenet0.25_voc|+|-| +yolo3_mobilenet1.0_coco|+|-| +yolo3_mobilenet1.0_custom|+|-| +yolo3_mobilenet1.0_voc|+|-| # Semantic segmentation Model | Availability in OMZ (2023.02.24)| Availability in the validation table | -|-|-| -danet_resnet101_citys|-|-| -danet_resnet50_citys|-|-| -deeplab_resnest101_ade|-|-| -deeplab_resnest200_ade|-|-| -deeplab_resnest269_ade|-|-| -deeplab_resnest50_ade|-|-| -deeplab_resnet101_ade|-|-| -deeplab_resnet101_citys|-|-| -deeplab_resnet101_coco|-|-| -deeplab_resnet101_coco_int8|-|-| -deeplab_resnet101_voc|-|-| -deeplab_resnet101_voc_int8|-|-| -deeplab_resnet152_coco|-|-| -deeplab_resnet152_voc|-|-| -deeplab_resnet50_ade|-|-| -deeplab_resnet50_citys|-|-| -deeplab_v3b_plus_wideresnet_citys|-|-| -fastscnn_citys|-|-| -fcn_resnet101_ade|-|-| -fcn_resnet101_coco|-|-| -fcn_resnet101_coco_int8|-|-| -fcn_resnet101_voc|-|-| -fcn_resnet101_voc_int8|-|-| -fcn_resnet50_ade|-|-| -fcn_resnet50_voc|-|-| -hrnet_w18_small_v1_s|-|-| -hrnet_w18_small_v2_s|-|-| -hrnet_w48_s|-|-| -icnet_resnet50_citys|-|-| -icnet_resnet50_mhpv1|-|-| -nasnet_4_1056|-|-| -nasnet_5_1538|-|-| -nasnet_6_4032|-|-| -nasnet_7_1920|-|-| -psp_resnet101_ade|-|-| -psp_resnet101_citys|-|-| -psp_resnet101_coco|-|-| -psp_resnet101_coco_int8|-|-| -psp_resnet101_voc|-|-| -psp_resnet101_voc_int8|-|-| -psp_resnet50_ade|-|-| +danet_resnet101_citys|+|-| +danet_resnet50_citys|+|-| +deeplab_resnest101_ade|+|-| +deeplab_resnest200_ade|+|-| +deeplab_resnest269_ade|+|-| +deeplab_resnest50_ade|+|-| +deeplab_resnet101_ade|+|-| +deeplab_resnet101_citys|+|-| +deeplab_resnet101_coco|+|-| +deeplab_resnet101_coco_int8|+|-| +deeplab_resnet101_voc|+|-| +deeplab_resnet101_voc_int8|+|-| +deeplab_resnet152_coco|+|-| +deeplab_resnet152_voc|+|-| +deeplab_resnet50_ade|+|-| +deeplab_resnet50_citys|+|-| +deeplab_v3b_plus_wideresnet_citys|+|-| +fastscnn_citys|+|-| +fcn_resnet101_ade|+|-| +fcn_resnet101_coco|+|-| +fcn_resnet101_coco_int8|+|-| +fcn_resnet101_voc|+|-| +fcn_resnet101_voc_int8|+|-| +fcn_resnet50_ade|+|-| +fcn_resnet50_voc|+|-| +hrnet_w18_small_v1_s|+|-| +hrnet_w18_small_v2_s|+|-| +hrnet_w48_s|+|-| +icnet_resnet50_citys|+|-| +icnet_resnet50_mhpv1|+|-| +nasnet_4_1056|+|-| +nasnet_5_1538|+|-| +nasnet_6_4032|+|-| +nasnet_7_1920|+|-| +psp_resnet101_ade|+|-| +psp_resnet101_citys|+|-| +psp_resnet101_coco|+|-| +psp_resnet101_coco_int8|+|-| +psp_resnet101_voc|+|-| +psp_resnet101_voc_int8|+|-| +psp_resnet50_ade|+|-| ### Instance segmentation Model | Availability in OMZ (2023.02.24)| Availability in the validation table | -|-|-| -custom_mask_rcnn_fpn|-|-| -mask_rcnn_fpn_resnet101_v1d_coco|-|-| -mask_rcnn_fpn_resnet18_v1b_coco|-|-| -mask_rcnn_fpn_resnet50_v1b_coco|-|-| -mask_rcnn_fpn_syncbn_mobilenet1_0_coco|-|-| -mask_rcnn_fpn_syncbn_resnet18_v1b_coco|-|-| -mask_rcnn_resnet101_v1d_coco|-|-| -mask_rcnn_resnet18_v1b_coco|-|-| -mask_rcnn_resnet50_v1b_coco|-|-| +custom_mask_rcnn_fpn|+|-| +mask_rcnn_fpn_resnet101_v1d_coco|+|-| +mask_rcnn_fpn_resnet18_v1b_coco|+|-| +mask_rcnn_fpn_resnet50_v1b_coco|+|-| +mask_rcnn_fpn_syncbn_mobilenet1_0_coco|+|-| +mask_rcnn_fpn_syncbn_resnet18_v1b_coco|+|-| +mask_rcnn_resnet101_v1d_coco|+|-| +mask_rcnn_resnet18_v1b_coco|+|-| +mask_rcnn_resnet50_v1b_coco|+|-| ### Pose estimation Model | Availability in OMZ (2023.02.24)| Availability in the validation table | -|-|-| -alpha_pose_resnet101_v1b_coco|-|-| -mobile_pose_mobilenet1.0|-|-| -mobile_pose_mobilenetv2_1.0|-|-| -mobile_pose_mobilenetv3_large|-|-| -mobile_pose_mobilenetv3_small|-|-| -mobile_pose_resnet18_v1b|-|-| -mobile_pose_resnet50_v1b|-|-| -simple_pose_resnet101_v1b|-|-| -simple_pose_resnet101_v1b_int8|-|-| -simple_pose_resnet101_v1d|-|-| -simple_pose_resnet101_v1d_int8|-|-| -simple_pose_resnet152_v1b|-|-| -simple_pose_resnet152_v1d|-|-| -simple_pose_resnet18_v1b|-|-| -simple_pose_resnet18_v1b_int8|-|-| -simple_pose_resnet50_v1b|-|-| -simple_pose_resnet50_v1b_int8|-|-| -simple_pose_resnet50_v1d|-|-| -simple_pose_resnet50_v1d_int8|-|-| +alpha_pose_resnet101_v1b_coco|+|-| +mobile_pose_mobilenet1.0|+|-| +mobile_pose_mobilenetv2_1.0|+|-| +mobile_pose_mobilenetv3_large|+|-| +mobile_pose_mobilenetv3_small|+|-| +mobile_pose_resnet18_v1b|+|-| +mobile_pose_resnet50_v1b|+|-| +simple_pose_resnet101_v1b|+|-| +simple_pose_resnet101_v1b_int8|+|-| +simple_pose_resnet101_v1d|+|-| +simple_pose_resnet101_v1d_int8|+|-| +simple_pose_resnet152_v1b|+|-| +simple_pose_resnet152_v1d|+|-| +simple_pose_resnet18_v1b|+|-| +simple_pose_resnet18_v1b_int8|+|-| +simple_pose_resnet50_v1b|+|-| +simple_pose_resnet50_v1b_int8|+|-| +simple_pose_resnet50_v1d|+|-| +simple_pose_resnet50_v1d_int8|+|-| ### Action recognition Model | Availability in OMZ (2023.02.24)| Availability in the validation table | -|-|-| -c3d_kinetics400|-|-| -i3d_inceptionv1_kinetics400|-|-| -i3d_inceptionv3_kinetics400|-|-| -i3d_nl10_resnet101_v1_kinetics400|-|-| -i3d_nl10_resnet50_v1_kinetics400|-|-| -i3d_nl5_resnet101_v1_kinetics400|-|-| -i3d_nl5_resnet50_v1_kinetics400|-|-| -i3d_resnet101_v1_kinetics400|-|-| -i3d_resnet50_v1_custom|-|-| -i3d_resnet50_v1_hmdb51|-|-| -i3d_resnet50_v1_kinetics400|-|-| -i3d_resnet50_v1_sthsthv2|-|-| -i3d_resnet50_v1_ucf101|-|-| -i3d_slow_resnet101_f16s4_kinetics700|-|-| -inceptionv1_hmdb51|-|-| -inceptionv1_kinetics400|-|-| -inceptionv1_sthsthv2|-|-| -inceptionv1_ucf101|-|-| -inceptionv3_hmdb51|-|-| -inceptionv3_kinetics400|-|-| -inceptionv3_kinetics400_int8|-|-| -inceptionv3_sthsthv2|-|-| -inceptionv3_ucf101|-|-| -inceptionv3_ucf101_int8|-|-| -p3d_resnet101_kinetics400|-|-| -p3d_resnet50_kinetics400|-|-| -r2plus1d_resnet101_kinetics400|-|-| -r2plus1d_resnet152_kinetics400|-|-| -r2plus1d_resnet18_kinetics400|-|-| -r2plus1d_resnet34_kinetics400|-|-| -r2plus1d_resnet50_kinetics400|-|-| -resnet101_v1b_kinetics400|-|-| -resnet101_v1b_sthsthv2|-|-| -resnet152_v1b_kinetics400|-|-| -resnet152_v1b_sthsthv2|-|-| -resnet18_v1b_kinetics400|-|-| -resnet18_v1b_kinetics400_int8|-|-| -resnet18_v1b_sthsthv2|-|-| -resnet34_v1b_kinetics400|-|-| -resnet34_v1b_sthsthv2|-|-| -resnet50_v1b_hmdb51|-|-| -resnet50_v1b_kinetics400|-|-| -resnet50_v1b_kinetics400_int8|-|-| -resnet50_v1b_sthsthv2|-|-| -resnet50_v1b_ucf101|-|-| -slowfast_16x8_resnet101_50_50_kinetics400|-|-| -slowfast_16x8_resnet101_kinetics400|-|-| -slowfast_4x16_resnet101_kinetics400|-|-| -slowfast_4x16_resnet50_custom|-|-| -slowfast_4x16_resnet50_kinetics400|-|-| -slowfast_8x8_resnet101_kinetics400|-|-| -slowfast_8x8_resnet50_kinetics400|-|-| -vgg16_hmdb51|-|-| -vgg16_kinetics400|-|-| -vgg16_sthsthv2|-|-| -vgg16_ucf101|-|-| -vgg16_ucf101_int8|-|-| +c3d_kinetics400|+|-| +i3d_inceptionv1_kinetics400|+|-| +i3d_inceptionv3_kinetics400|+|-| +i3d_nl10_resnet101_v1_kinetics400|+|-| +i3d_nl10_resnet50_v1_kinetics400|+|-| +i3d_nl5_resnet101_v1_kinetics400|+|-| +i3d_nl5_resnet50_v1_kinetics400|+|-| +i3d_resnet101_v1_kinetics400|+|-| +i3d_resnet50_v1_custom|+|-| +i3d_resnet50_v1_hmdb51|+|-| +i3d_resnet50_v1_kinetics400|+|-| +i3d_resnet50_v1_sthsthv2|+|-| +i3d_resnet50_v1_ucf101|+|-| +i3d_slow_resnet101_f16s4_kinetics700|+|-| +inceptionv1_hmdb51|+|-| +inceptionv1_kinetics400|+|-| +inceptionv1_sthsthv2|+|-| +inceptionv1_ucf101|+|-| +inceptionv3_hmdb51|+|-| +inceptionv3_kinetics400|+|-| +inceptionv3_kinetics400_int8|+|-| +inceptionv3_sthsthv2|+|-| +inceptionv3_ucf101|+|-| +inceptionv3_ucf101_int8|+|-| +p3d_resnet101_kinetics400|+|-| +p3d_resnet50_kinetics400|+|-| +r2plus1d_resnet101_kinetics400|+|-| +r2plus1d_resnet152_kinetics400|+|-| +r2plus1d_resnet18_kinetics400|+|-| +r2plus1d_resnet34_kinetics400|+|-| +r2plus1d_resnet50_kinetics400|+|-| +resnet101_v1b_kinetics400|+|-| +resnet101_v1b_sthsthv2|+|-| +resnet152_v1b_kinetics400|+|-| +resnet152_v1b_sthsthv2|+|-| +resnet18_v1b_kinetics400|+|-| +resnet18_v1b_kinetics400_int8|+|-| +resnet18_v1b_sthsthv2|+|-| +resnet34_v1b_kinetics400|+|-| +resnet34_v1b_sthsthv2|+|-| +resnet50_v1b_hmdb51|+|-| +resnet50_v1b_kinetics400|+|-| +resnet50_v1b_kinetics400_int8|+|-| +resnet50_v1b_sthsthv2|+|-| +resnet50_v1b_ucf101|+|-| +slowfast_16x8_resnet101_50_50_kinetics400|+|-| +slowfast_16x8_resnet101_kinetics400|+|-| +slowfast_4x16_resnet101_kinetics400|+|-| +slowfast_4x16_resnet50_custom|+|-| +slowfast_4x16_resnet50_kinetics400|+|-| +slowfast_8x8_resnet101_kinetics400|+|-| +slowfast_8x8_resnet50_kinetics400|+|-| +vgg16_hmdb51|+|-| +vgg16_kinetics400|+|-| +vgg16_sthsthv2|+|-| +vgg16_ucf101|+|-| +vgg16_ucf101_int8|+|-| ### Depth prediction Model | Availability in OMZ (2023.02.24)| Availability in the validation table | -|-|-| -monodepth2_resnet18_kitti_mono_640x192|-|-| -monodepth2_resnet18_kitti_mono_stereo_640x192|-|-| -monodepth2_resnet18_kitti_stereo_640x192|-|-| -monodepth2_resnet18_posenet_kitti_mono_640x192|-|-| -monodepth2_resnet18_posenet_kitti_mono_stereo_640x192|-|-| +monodepth2_resnet18_kitti_mono_640x192|+|-| +monodepth2_resnet18_kitti_mono_stereo_640x192|+|-| +monodepth2_resnet18_kitti_stereo_640x192|+|-| +monodepth2_resnet18_posenet_kitti_mono_640x192|+|-| +monodepth2_resnet18_posenet_kitti_mono_stereo_640x192|+|-| ### Image classification on Cifar-10 Model | Availability in OMZ (2023.02.24)| Availability in the validation table | -|-|-| -cifar_residualattentionnet452|-|-| -cifar_residualattentionnet56|-|-| -cifar_residualattentionnet92|-|-| -cifar_resnet110_v1|-|-| -cifar_resnet110_v2|-|-| -cifar_resnet20_v1|-|-| -cifar_resnet20_v2|-|-| -cifar_resnet56_v1|-|-| -cifar_resnet56_v2|-|-| -cifar_resnext29_16x64d|-|-| -cifar_resnext29_32x4d|-|-| -cifar_wideresnet16_10|-|-| -cifar_wideresnet28_10|-|-| -cifar_wideresnet40_8|-|-| +cifar_residualattentionnet452|+|-| +cifar_residualattentionnet56|+|-| +cifar_residualattentionnet92|+|-| +cifar_resnet110_v1|+|-| +cifar_resnet110_v2|+|-| +cifar_resnet20_v1|+|-| +cifar_resnet20_v2|+|-| +cifar_resnet56_v1|+|-| +cifar_resnet56_v2|+|-| +cifar_resnext29_16x64d|+|-| +cifar_resnext29_32x4d|+|-| +cifar_wideresnet16_10|+|-| +cifar_wideresnet28_10|+|-| +cifar_wideresnet40_8|+|-| ### Object tracking Model | Availability in OMZ (2023.02.24)| Availability in the validation table | -|-|-| -siamrpn_alexnet_v2_otb15|-|-| +siamrpn_alexnet_v2_otb15|+|-| From 4c44514cd1f906458275c04dfb2e4c2241eff8dd Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Sun, 26 Feb 2023 14:44:11 +0300 Subject: [PATCH 32/63] Update validation table (add int8 models) --- results/mxnet_models_checklist.md | 4 ++-- .../validation/validation_results_mxnet_gluon_modelzoo.md | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/results/mxnet_models_checklist.md b/results/mxnet_models_checklist.md index d7e5465f4..f3655d90b 100644 --- a/results/mxnet_models_checklist.md +++ b/results/mxnet_models_checklist.md @@ -27,7 +27,7 @@ mobilenet0.25|+|+| mobilenet0.5|+|+| mobilenet0.75|+|+| mobilenet1.0|+|+| -mobilenet1.0_int8|+|-| +mobilenet1.0_int8|+|+| mobilenetv2_0.25|+|+| mobilenetv2_0.5|+|+| mobilenetv2_0.75|+|+| @@ -73,7 +73,7 @@ resnet34_v1|+|+| resnet34_v1b|+|+| resnet34_v2|+|+| resnet50_v1|+|+| -resnet50_v1_int8|+|-| +resnet50_v1_int8|+|+| resnet50_v1b|+|+| resnet50_v1b_custom|+|-| resnet50_v1b_gn|+|-| diff --git a/results/validation/validation_results_mxnet_gluon_modelzoo.md b/results/validation/validation_results_mxnet_gluon_modelzoo.md index 1cb16ba42..583c7b0a0 100644 --- a/results/validation/validation_results_mxnet_gluon_modelzoo.md +++ b/results/validation/validation_results_mxnet_gluon_modelzoo.md @@ -43,6 +43,7 @@ mobilenet0.25|0.1413059 Granny Smith
0.0491460 necklace
0.0399561 bell pep mobilenet0.5 |0.0988460 teapot
0.0556895 piggy bank, penny bank
0.0552285 saltshaker, salt shaker
0.0383402 pitcher, ewer
0.0319066 necklace| mobilenet0.75|0.1711094 Granny Smith
0.1602635 piggy bank, penny bank
0.0857521 teapot
0.0533995 pitcher, ewer
0.0354786 soap dispenser| mobilenet1.0 |0.4475225 Granny Smith
0.0954533 piggy bank, penny bank
0.0523449 saltshaker, salt shaker
0.0358669 pencil sharpener
0.0232730 dumbbell| +mobilenet1.0_int8 |0.4552936 Granny Smith
0.0632280 piggy bank, penny bank
0.0523604 saltshaker, salt shaker
0.0327314 pencil sharpener
0.0265849 dumbbell| mobilenetv2_0.25|0.1189503 saltshaker, salt shaker
0.0919519 bell pepper
0.0862974 Granny Smith
0.0828691 piggy bank, penny bank
0.0527703 hair slide| mobilenetv2_0.5 |0.1700138 hair slide
0.0890763 dumbbell
0.0777359 piggy bank, penny bank
0.0521625 saltshaker, salt shaker
0.0489330 necklace| mobilenetv2_0.75|0.3621189 saltshaker, salt shaker
0.2135993 Granny Smith
0.1286786 piggy bank, penny bank
0.0391913 lemon
0.0134064 ocarina, sweet potato| @@ -57,6 +58,7 @@ resnet34_v1 |0.5898067 piggy bank, penny bank
0.3150526 Granny Smith
0.012 resnet34_v2 |0.5082690 Granny Smith
0.3873709 piggy bank, penny bank
0.0163602 pencil sharpener
0.0137501 saltshaker, salt shaker
0.0071418 dumbbell| resnet34_v1b |0.8592082 Granny Smith
0.0595315 piggy bank, penny bank
0.0257422 pitcher, ewer
0.0107621 saltshaker, salt shaker
0.0081604 water jug| resnet50_v1 |0.7377543 Granny Smith
0.0241721 piggy bank, penny bank
0.0123405 lemon
0.0061283 candle, taper, wax light
0.0051573 orange| +resnet50_v1_int8|0.6387886 Granny Smith
0.0295431 piggy bank, penny bank
0.0144967 lemon
0.0125546 candle, taper, wax light
0.0110684 saltshaker, salt shaker| resnet50_v2 |0.9931253 Granny Smith
0.0017001 piggy bank, penny bank
0.0007180 saltshaker, salt shaker
0.0006648 dumbbell
0.0002998 tennis ball| resnet50_v1b |0.5989549 Granny Smith
0.1451391 piggy bank, penny bank
0.0253168 pitcher, ewer
0.0129785 candle, taper, wax light
0.0104263 saltshaker, salt shaker| resnet50_v1c |0.8618550 Granny Smith
0.0188060 candle, taper, wax light
0.0055579 orange
0.0039412 pitcher, ewer
0.0034311 lemon| @@ -139,6 +141,7 @@ mobilenet0.25|0.7976559 junco, snowbird
0.0642661 brambling, Fringilla montif mobilenet0.5 |0.6876235 junco, snowbird
0.0567821 chickadee
0.0440355 house finch, linnet, Carpodacus mexicanus
0.0327330 brambling, Fringilla montifringilla
0.0170564 hummingbird| mobilenet0.75|0.7232150 junco, snowbird
0.0267700 brambling, Fringilla montifringilla
0.0224716 chickadee
0.0173853 water ouzel, dipper
0.0107710 house finch, linnet, Carpodacus mexicanus| mobilenet1.0 |0.3200681 junco, snowbird
0.0977401 chickadee
0.0255691 brambling, Fringilla montifringilla
0.0200887 water ouzel, dipper
0.0052986 indigo bunting, indigo finch, indigo bird, Passerina cyanea| +mobilenet1.0_int8 | 0.4458332 junco, snowbird
0.0757087 chickadee
0.0233120 water ouzel, dipper
0.0213673 brambling, Fringilla montifringilla
0.0034389 indigo bunting, indigo finch, indigo bird, Passerina cyanea| mobilenetv2_0.25|0.5445739 junco, snowbird
0.1674697 chickadee
0.0738745 brambling, Fringilla montifringilla
0.0237202 house finch, linnet, Carpodacus mexicanus
0.0177469 coucal| mobilenetv2_0.5 |0.9121019 junco, snowbird
0.0402663 chickadee
0.0144705 brambling, Fringilla montifringilla
0.0111383 house finch, linnet, Carpodacus mexicanus
0.0019353 bulbul| mobilenetv2_0.75|0.4884860 junco, snowbird
0.0884391 chickadee
0.0189599 brambling, Fringilla montifringilla
0.0151568 water ouzel, dipper
0.0111601 hummingbird| @@ -153,6 +156,7 @@ resnet34_v1 |0.9352032 junco, snowbird
0.0226504 water ouzel, dipper
0.012 resnet34_v2 |0.6477929 junco, snowbird
0.0750227 water ouzel, dipper
0.0672589 brambling, Fringilla montifringilla
0.0443260 chickadee
0.0321975 goldfinch, Carduelis carduelis| resnet34_v1b |0.9445685 junco, snowbird
0.0109547 water ouzel, dipper
0.0106829 goldfinch, Carduelis carduelis
0.0075599 brambling, Fringilla montifringilla
0.0043267 chickadee| resnet50_v1 |0.8778600 junco, snowbird
0.0045333 water ouzel, dipper
0.0018932 brambling, Fringilla montifringilla
0.0016121 chickadee
0.0005472 magpie| +resnet50_v1_int8 |0.8986668 junco, snowbird
0.0042505 water ouzel, dipper
0.0019016 brambling, Fringilla montifringilla
0.0012305 chickadee
0.0005350 indigo bunting, indigo finch, indigo bird, Passerina cyanea| resnet50_v2 |0.9820374 junco, snowbird
0.0070083 water ouzel, dipper
0.0032555 chickadee
0.0021856 indigo bunting, indigo finch, indigo bird, Passerina cyanea
0.0017496 brambling, Fringilla montifringilla| resnet50_v1b |0.9325832 junco, snowbird
0.0022530 water ouzel, dipper
0.0019862 chickadee
0.0011287 brambling, Fringilla montifringilla
0.0005958 goldfinch, Carduelis carduelis| resnet50_v1c |0.9273983 junco, snowbird
0.0029364 chickadee
0.0021531 water ouzel, dipper
0.0007963 brambling, Fringilla montifringilla
0.0007281 indigo bunting, indigo finch, indigo bird, Passerina cyanea| @@ -235,6 +239,7 @@ mobilenet0.25|0.1772091 water bottle
0.1686604 liner, ocean liner
0.095527 mobilenet0.5 |0.1413260 submarine, pigboat, sub, U-boat
0.1064001 aircraft carrier, carrier, flattop, attack aircraft carrier
0.0897172 container ship, containership, container vessel
0.0690325 liner, ocean liner
0.0603464 breakwater, groin, groyne, mole, bulwark, seawall, jetty| mobilenet0.75|0.1406201 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0317071 liner, ocean liner
0.0312595 beacon, lighthouse, beacon light, pharos
0.0274736 container ship, containership, container vessel
0.0195056 fireboat| mobilenet1.0 |0.0580719 liner, ocean liner
0.0576046 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0517998 aircraft carrier, carrier, flattop, attack aircraft carrier
0.0483820 lifeboat
0.0473302 water bottle| +mobilenet1.0_int8 | 0.0885448 liner, ocean liner
0.0556101 aircraft carrier, carrier, flattop, attack aircraft carrier
0.0483991 water bottle
0.0429513 lifeboat
0.0388312 breakwater, groin, groyne, mole, bulwark, seawall, jetty| mobilenetv2_0.25|0.2591519 liner, ocean liner
0.0933797 drilling platform, offshore rig
0.0799795 aircraft carrier, carrier, flattop, attack aircraft carrier
0.0531668 fireboat
0.0529574 container ship, containership, container vessel| mobilenetv2_0.5 |0.1369159 container ship, containership, container vessel
0.1195146 liner, ocean liner
0.0921707 drilling platform, offshore rig
0.0855153 submarine, pigboat, sub, U-boat
0.0492694 beacon, lighthouse, beacon light, pharos| mobilenetv2_0.75|0.1567852 liner, ocean liner
0.0603636 lifeboat
0.0488465 pirate, pirate ship
0.0427859 container ship, containership, container vessel
0.0276454 drilling platform, offshore rig| @@ -249,6 +254,7 @@ resnet34_v1 |0.5147043 liner, ocean liner
0.0504066 submarine, pigboat, sub, resnet34_v2 |0.4267518 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.2477035 container ship, containership, container vessel
0.0763714 liner, ocean liner
0.0696334 dock, dockage, docking facility
0.0447001 beacon, lighthouse, beacon light, pharos| resnet34_v1b |0.3476998 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.1918807 liner, ocean liner
0.1346360 drilling platform, offshore rig
0.1013912 container ship, containership, container vessel
0.0379062 aircraft carrier, carrier, flattop, attack aircraft carrier| resnet50_v1 |0.4411839 liner, ocean liner
0.0861827 container ship, containership, container vessel
0.0609572 speedboat
0.0587049 dock, dockage, docking facility
0.0369093 breakwater, groin, groyne, mole, bulwark, seawall, jetty| +resnet50_v1_int8 |0.3967755 liner, ocean liner
0.0905244 container ship, containership, container vessel
0.0735052 dock, dockage, docking facility
0.0657126 speedboat
0.0458605 breakwater, groin, groyne, mole, bulwark, seawall, jetty| resnet50_v2 |0.4814195 container ship, containership, container vessel
0.1134715 liner, ocean liner
0.0718216 drilling platform, offshore rig
0.0571725 dock, dockage, docking facility
0.0450075 lifeboat| resnet50_v1b |0.3147515 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.1076186 beacon, lighthouse, beacon light, pharos
0.0938480 liner, ocean liner
0.0601708 container ship, containership, container vessel
0.0513268 catamaran| resnet50_v1c |0.1537983 liner, ocean liner
0.1531646 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.1442180 catamaran
0.0900431 container ship, containership, container vessel
0.0479482 dock, dockage, docking facility| From b0432b672bbd89be40201c232ba0dd8ba9b199e0 Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Sun, 26 Feb 2023 15:24:18 +0300 Subject: [PATCH 33/63] Update validation table (add some models) --- results/mxnet_models_checklist.md | 33 +++++++++---------- ...validation_results_mxnet_gluon_modelzoo.md | 9 +++++ 2 files changed, 25 insertions(+), 17 deletions(-) diff --git a/results/mxnet_models_checklist.md b/results/mxnet_models_checklist.md index f3655d90b..66c7be7c4 100644 --- a/results/mxnet_models_checklist.md +++ b/results/mxnet_models_checklist.md @@ -34,13 +34,13 @@ mobilenetv2_0.75|+|+| mobilenetv2_1.0|+|+| mobilenetv3_large|+|+| mobilenetv3_small|+|+| -residualattentionnet128|+|-| -residualattentionnet164|+|-| -residualattentionnet200|+|-| -residualattentionnet236|+|-| -residualattentionnet452|+|-| -residualattentionnet56|+|-| -residualattentionnet92|+|-| +residualattentionnet128|+|Error ( Parameter 'residualattentionmodel0_hybridsequential0_conv0_weight' has not been initialized)| +residualattentionnet164|+|Error ( Parameter 'residualattentionmodel0_hybridsequential0_conv0_weight' has not been initialized)| +residualattentionnet200|+|Error ( Parameter 'residualattentionmodel0_hybridsequential0_conv0_weight' has not been initialized)| +residualattentionnet236|+|Error ( Parameter 'residualattentionmodel0_hybridsequential0_conv0_weight' has not been initialized)| +residualattentionnet452|+|Error ( Parameter 'residualattentionmodel0_hybridsequential0_conv0_weight' has not been initialized)| +residualattentionnet56|+|Error ( Parameter 'residualattentionmodel0_hybridsequential0_conv0_weight' has not been initialized)| +residualattentionnet92|+|Error ( Parameter 'residualattentionmodel0_hybridsequential0_conv0_weight' has not been initialized)| resnest101|+|+| resnest14|+|+| resnest200|+|+| @@ -49,25 +49,24 @@ resnest269|+|+| resnest50|+|+| resnet101_v1|+|+| resnet101_v1b|+|+| -resnet101_v1b_gn|+|-| resnet101_v1c|+|+| resnet101_v1d|+|+| resnet101_v1d_0.73|+|+| resnet101_v1d_0.76|+|+| -resnet101_v1e|+|-| +resnet101_v1e|+|Error (Failed loading Parameter 'resnetv1e_batchnorm2_gamma' from saved params: shape incompatible expected (128,) vs saved (64,))| resnet101_v1s|+|+| resnet101_v2|+|+| resnet152_v1|+|+| resnet152_v1b|+|+| resnet152_v1c|+|+| resnet152_v1d|+|+| -resnet152_v1e|+|-| +resnet152_v1e|+|Error (Failed loading Parameter 'resnetv1e_batchnorm2_gamma' from saved params: shape incompatible expected (128,) vs saved (64,))| resnet152_v1s|+|+| resnet152_v2|+|+| resnet18_v1|+|+| resnet18_v1b|+|+| resnet18_v1b_0.89|+|+| -resnet18_v1b_custom|+|-| +resnet18_v1b_custom|+|+| resnet18_v2|+|+| resnet34_v1|+|+| resnet34_v1b|+|+| @@ -75,15 +74,15 @@ resnet34_v2|+|+| resnet50_v1|+|+| resnet50_v1_int8|+|+| resnet50_v1b|+|+| -resnet50_v1b_custom|+|-| -resnet50_v1b_gn|+|-| +resnet50_v1b_custom|+|+| +resnet50_v1b_gn|+|+| resnet50_v1c|+|+| resnet50_v1d|+|+| resnet50_v1d_0.11|+|+| resnet50_v1d_0.37|+|+| resnet50_v1d_0.48|+|+| resnet50_v1d_0.86|+|+| -resnet50_v1e|+|-| +resnet50_v1e|+|Error (Failed loading Parameter 'resnetv1e_batchnorm0_gamma' from saved params: shape incompatible expected (64,) vs saved (32,))| resnet50_v1s|+|+| resnet50_v2|+|+| resnext101_32x4d|+|+| @@ -93,11 +92,11 @@ se_resnext101_32x4d|+|+| se_resnext101_64x4d|+|+| se_resnext50_32x4d|+|+| senet_154|+|+| -senet_154e|+|-| +senet_154e|+|Error (Parameter 'features.11.0.downsample.1.weight' is missing...)| squeezenet1.0|+|+| squeezenet1.1|+|+| -shufflenet_v1|+|-| -shufflenet_v2|+|-| +shufflenet_v1|+|Error (Parameter 'shufflenetv10_conv0_weight' has not been initialized)| +shufflenet_v2|+|Error (Parameter 'shufflenetv20_conv0_weight' has not been initialized)| vgg11|+|+| vgg11_bn|+|+| vgg13|+|+| diff --git a/results/validation/validation_results_mxnet_gluon_modelzoo.md b/results/validation/validation_results_mxnet_gluon_modelzoo.md index 583c7b0a0..768bea6cf 100644 --- a/results/validation/validation_results_mxnet_gluon_modelzoo.md +++ b/results/validation/validation_results_mxnet_gluon_modelzoo.md @@ -54,6 +54,7 @@ resnet18_v1 |0.7145669 Granny Smith
0.0433350 piggy bank, penny bank
0.034 resnet18_v2 |0.2885310 Granny Smith
0.1816195 piggy bank, penny bank
0.0722676 saltshaker, salt shaker
0.0635361 rubber eraser, rubber, pencil eraser
0.0440725 soap dispenser| resnet18_v1b |0.9093145 Granny Smith
0.0187165 saltshaker, salt shaker
0.0117636 piggy bank, penny bank
0.0104429 pomegranate
0.0058578 perfume, essence| resnet18_v1b_0.89|0.5232244 piggy bank, penny bank
0.1546666 Granny Smith
0.1040182 saltshaker, salt shaker
0.0313044 soap dispenser
0.0295205 vase| +resnet18_v1b_custom|0.0051828 African chameleon, Chamaeleo chamaeleon
0.0048727 Gila monster, Heloderma suspectum
0.0046023 colobus, colobus monkey
0.0045542 black widow, Latrodectus mactans
0.0044811 flatworm, platyhelminth| resnet34_v1 |0.5898067 piggy bank, penny bank
0.3150526 Granny Smith
0.0128028 saltshaker, salt shaker
0.0093141 candle, taper, wax light
0.0089791 pencil sharpener| resnet34_v2 |0.5082690 Granny Smith
0.3873709 piggy bank, penny bank
0.0163602 pencil sharpener
0.0137501 saltshaker, salt shaker
0.0071418 dumbbell| resnet34_v1b |0.8592082 Granny Smith
0.0595315 piggy bank, penny bank
0.0257422 pitcher, ewer
0.0107621 saltshaker, salt shaker
0.0081604 water jug| @@ -61,6 +62,8 @@ resnet50_v1 |0.7377543 Granny Smith
0.0241721 piggy bank, penny bank
0.012 resnet50_v1_int8|0.6387886 Granny Smith
0.0295431 piggy bank, penny bank
0.0144967 lemon
0.0125546 candle, taper, wax light
0.0110684 saltshaker, salt shaker| resnet50_v2 |0.9931253 Granny Smith
0.0017001 piggy bank, penny bank
0.0007180 saltshaker, salt shaker
0.0006648 dumbbell
0.0002998 tennis ball| resnet50_v1b |0.5989549 Granny Smith
0.1451391 piggy bank, penny bank
0.0253168 pitcher, ewer
0.0129785 candle, taper, wax light
0.0104263 saltshaker, salt shaker| +resnet50_v1b_custom |0.0051049 anemone fish
0.0050241 spider monkey, Ateles geoffroyi
0.0050122 ibex, Capra ibex
0.0049926 cicada, cicala
0.0048133 garter snake, grass snake +resnet50_v1b_gn|0.7398045 Granny Smith
0.0445737 candle, taper, wax light
0.0127276 piggy bank, penny bank
0.0120557 safety pin
0.0083770 pomegranate| resnet50_v1c |0.8618550 Granny Smith
0.0188060 candle, taper, wax light
0.0055579 orange
0.0039412 pitcher, ewer
0.0034311 lemon| resnet50_v1d |0.7543250 Granny Smith
0.0171997 dumbbell
0.0116827 candle, taper, wax light
0.0074653 spindle
0.0059659 piggy bank, penny bank| resnet50_v1d_0.11|0.0547115 necklace
0.0466139 hair slide
0.0440734 dumbbell
0.0317605 piggy bank, penny bank
0.0290907 bell pepper| @@ -152,6 +155,7 @@ resnet18_v1 |0.9597536 junco, snowbird
0.0103961 chickadee
0.0075481 goldf resnet18_v2 |0.9460587 junco, snowbird
0.0180775 brambling, Fringilla montifringilla
0.0139834 chickadee
0.0075303 goldfinch, Carduelis carduelis
0.0037932 water ouzel, dipper| resnet18_v1b |0.9505776 junco, snowbird
0.0350148 brambling, Fringilla montifringilla
0.0052746 goldfinch, Carduelis carduelis
0.0044680 chickadee
0.0029356 house finch, linnet, Carpodacus mexicanus| resnet18_v1b_0.89|0.9328545 junco, snowbird
0.0446809 brambling, Fringilla montifringilla
0.0101631 house finch, linnet, Carpodacus mexicanus
0.0063039 chickadee
0.0027250 indigo bunting, indigo finch, indigo bird, Passerina cyanea| +resnet18_v1b_custom |0.0060174 African chameleon, Chamaeleo chamaeleon
0.0054798 American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier
0.0051301 flatworm, platyhelminth
0.0050669 prairie chicken, prairie grouse, prairie fowl
0.0049658 Shih-Tzu| resnet34_v1 |0.9352032 junco, snowbird
0.0226504 water ouzel, dipper
0.0129960 brambling, Fringilla montifringilla
0.0050841 chickadee
0.0037434 goldfinch, Carduelis carduelis| resnet34_v2 |0.6477929 junco, snowbird
0.0750227 water ouzel, dipper
0.0672589 brambling, Fringilla montifringilla
0.0443260 chickadee
0.0321975 goldfinch, Carduelis carduelis| resnet34_v1b |0.9445685 junco, snowbird
0.0109547 water ouzel, dipper
0.0106829 goldfinch, Carduelis carduelis
0.0075599 brambling, Fringilla montifringilla
0.0043267 chickadee| @@ -159,6 +163,8 @@ resnet50_v1 |0.8778600 junco, snowbird
0.0045333 water ouzel, dipper
0.001 resnet50_v1_int8 |0.8986668 junco, snowbird
0.0042505 water ouzel, dipper
0.0019016 brambling, Fringilla montifringilla
0.0012305 chickadee
0.0005350 indigo bunting, indigo finch, indigo bird, Passerina cyanea| resnet50_v2 |0.9820374 junco, snowbird
0.0070083 water ouzel, dipper
0.0032555 chickadee
0.0021856 indigo bunting, indigo finch, indigo bird, Passerina cyanea
0.0017496 brambling, Fringilla montifringilla| resnet50_v1b |0.9325832 junco, snowbird
0.0022530 water ouzel, dipper
0.0019862 chickadee
0.0011287 brambling, Fringilla montifringilla
0.0005958 goldfinch, Carduelis carduelis| +resnet50_v1b_custom |0.0042406 bullfrog, Rana catesbeiana
0.0041815 thunder snake, worm snake, Carphophis amoenus
0.0041015 bulbul
0.0040439 bustard
0.0039271 partridge| +resnet50_v1b_gn|0.8502029 junco, snowbird
0.0078807 brambling, Fringilla montifringilla
0.0052344 water ouzel, dipper
0.0047941 chickadee
0.0025184 goldfinch, Carduelis carduelis| resnet50_v1c |0.9273983 junco, snowbird
0.0029364 chickadee
0.0021531 water ouzel, dipper
0.0007963 brambling, Fringilla montifringilla
0.0007281 indigo bunting, indigo finch, indigo bird, Passerina cyanea| resnet50_v1d |0.8592040 junco, snowbird
0.0012552 trilobite
0.0008265 indigo bunting, indigo finch, indigo bird, Passerina cyanea
0.0007142 brambling, Fringilla montifringilla
0.0006234 chickadee| resnet50_v1d_0.11|0.8141757 junco, snowbird
0.0476836 brambling, Fringilla montifringilla
0.0176680 house finch, linnet, Carpodacus mexicanus
0.0174768 goldfinch, Carduelis carduelis
0.0137937 chickadee| @@ -250,6 +256,7 @@ resnet18_v1 |0.3416699 container ship, containership, container vessel
0.1224 resnet18_v2 |0.2872601 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.1359790 beacon, lighthouse, beacon light, pharos
0.1248601 container ship, containership, container vessel
0.1217950 dock, dockage, docking facility
0.0381382 fireboat| resnet18_v1b |0.1451897 liner, ocean liner
0.1121958 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0984692 water bottle
0.0750244 pop bottle, soda bottle
0.0602022 dock, dockage, docking facility| resnet18_v1b_0.89|0.3873618 liner, ocean liner
0.1239402 container ship, containership, container vessel
0.1021361 fireboat
0.0560824 dock, dockage, docking facility
0.0553757 submarine, pigboat, sub, U-boat| +resnet18_v1b_custom |0.0047132 spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish
0.0041157 triceratops
0.0040319 zebra
0.0039745 African chameleon, Chamaeleo chamaeleon
0.0039639 African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus| resnet34_v1 |0.5147043 liner, ocean liner
0.0504066 submarine, pigboat, sub, U-boat
0.0452913 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0438951 sandbar, sand bar
0.0422814 fireboat| resnet34_v2 |0.4267518 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.2477035 container ship, containership, container vessel
0.0763714 liner, ocean liner
0.0696334 dock, dockage, docking facility
0.0447001 beacon, lighthouse, beacon light, pharos| resnet34_v1b |0.3476998 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.1918807 liner, ocean liner
0.1346360 drilling platform, offshore rig
0.1013912 container ship, containership, container vessel
0.0379062 aircraft carrier, carrier, flattop, attack aircraft carrier| @@ -257,6 +264,8 @@ resnet50_v1 |0.4411839 liner, ocean liner
0.0861827 container ship, container resnet50_v1_int8 |0.3967755 liner, ocean liner
0.0905244 container ship, containership, container vessel
0.0735052 dock, dockage, docking facility
0.0657126 speedboat
0.0458605 breakwater, groin, groyne, mole, bulwark, seawall, jetty| resnet50_v2 |0.4814195 container ship, containership, container vessel
0.1134715 liner, ocean liner
0.0718216 drilling platform, offshore rig
0.0571725 dock, dockage, docking facility
0.0450075 lifeboat| resnet50_v1b |0.3147515 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.1076186 beacon, lighthouse, beacon light, pharos
0.0938480 liner, ocean liner
0.0601708 container ship, containership, container vessel
0.0513268 catamaran| +resnet50_v1b_custom |0.0048008 Madagascar cat, ring-tailed lemur, Lemur catta
0.0047634 rock python, rock snake, Python sebae
0.0046949 house finch, linnet, Carpodacus mexicanus
0.0046650 sea slug, nudibranch
0.0044506 Tibetan mastiff| +resnet50_v1b_gn|0.2332796 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.1691528 liner, ocean liner
0.1512439 beacon, lighthouse, beacon light, pharos
0.0507854 dock, dockage, docking facility
0.0476710 wreck| resnet50_v1c |0.1537983 liner, ocean liner
0.1531646 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.1442180 catamaran
0.0900431 container ship, containership, container vessel
0.0479482 dock, dockage, docking facility| resnet50_v1d |0.2062457 lifeboat
0.0950574 beacon, lighthouse, beacon light, pharos
0.0893152 liner, ocean liner
0.0759133 breakwater, groin, groyne, mole, bulwark, seawall, jetty
0.0445664 submarine, pigboat, sub, U-boat| resnet50_v1d_0.11|0.1494648 container ship, containership, container vessel
0.0635204 pirate, pirate ship
0.0557595 drilling platform, offshore rig
0.0280262 liner, ocean liner
0.0257161 lifeboat| From e8b7d55e28316025530757a1d92646aed56c8d2a Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Sun, 26 Feb 2023 15:25:42 +0300 Subject: [PATCH 34/63] Fix formatting --- results/mxnet_models_checklist.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/results/mxnet_models_checklist.md b/results/mxnet_models_checklist.md index 66c7be7c4..4402db64a 100644 --- a/results/mxnet_models_checklist.md +++ b/results/mxnet_models_checklist.md @@ -184,7 +184,7 @@ yolo3_mobilenet1.0_coco|+|-| yolo3_mobilenet1.0_custom|+|-| yolo3_mobilenet1.0_voc|+|-| -# Semantic segmentation +### Semantic segmentation Model | Availability in OMZ (2023.02.24)| Availability in the validation table | -|-|-| From 62e574b377ce1f4954be04f559058176ad63e72e Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Fri, 3 Mar 2023 14:53:06 +0300 Subject: [PATCH 35/63] Fix requirements for smoke test + GPU -> NVIDIA GPU --- .github/workflows/smoke_test.yml | 3 +-- requirements_frameworks.txt | 2 ++ src/inference/inference_mxnet.py | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 requirements_frameworks.txt diff --git a/.github/workflows/smoke_test.yml b/.github/workflows/smoke_test.yml index c7213c252..5fd532c2e 100644 --- a/.github/workflows/smoke_test.yml +++ b/.github/workflows/smoke_test.yml @@ -27,8 +27,7 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install -r requirements.txt - python -m pip install openvino-dev[caffe,mxnet,tensorflow2]==2022.2.0 - python -m pip install gluoncv + python -m pip install -r requirements_frameworks.txt - name: Run smoke test for inference benchmark run: cd test/smoke_test && ./run_bench_smoke_test.sh diff --git a/requirements_frameworks.txt b/requirements_frameworks.txt new file mode 100644 index 000000000..dda1ff2e8 --- /dev/null +++ b/requirements_frameworks.txt @@ -0,0 +1,2 @@ +openvino-dev[caffe,mxnet,tensorflow2]==2022.3.0 +gluoncv \ No newline at end of file diff --git a/src/inference/inference_mxnet.py b/src/inference/inference_mxnet.py index 0f024e9c7..d9a4cfc37 100644 --- a/src/inference/inference_mxnet.py +++ b/src/inference/inference_mxnet.py @@ -105,7 +105,7 @@ def cli_argument_parser(): type=bool, dest='raw_output') parser.add_argument('-d', '--device', - help='Specify the target device to infer on CPU or GPU (CPU by default)', + help='Specify the target device to infer on CPU or NVIDIA GPU (CPU by default)', default='CPU', type=str, dest='device') @@ -120,7 +120,7 @@ def get_device_to_infer(device): if device == 'CPU': log.info(f'Inference will be executed on {device}') return mxnet.cpu() - elif device == 'GPU': + elif device == 'NVIDIA GPU': log.info(f'Inference will be executed on {device}') return mxnet.gpu() else: From f8cd8994a2cfcd5a8b2c9caf3d143fcf86850e28 Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Sat, 4 Mar 2023 14:30:45 +0300 Subject: [PATCH 36/63] Print output when number of iterations equals 1 --- src/inference/inference_mxnet.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/inference/inference_mxnet.py b/src/inference/inference_mxnet.py index d9a4cfc37..14c38a486 100644 --- a/src/inference/inference_mxnet.py +++ b/src/inference/inference_mxnet.py @@ -258,8 +258,9 @@ def main(): average_time, latency, fps = process_result(args.batch_size, inference_time) if not args.raw_output: - # print_topk_predictions should be implemented as io.process_output(result, log) - print_topk_predictions(result, args.number_top, args.labels) + if args.number_iter == 1: + # print_topk_predictions should be implemented as io.process_output(result, log) + print_topk_predictions(result, args.number_top, args.labels) result_output(average_time, fps, latency) else: raw_result_output(average_time, fps, latency) From 1b67c039a4c6a0490464a3d841bf8376285f88e3 Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Sat, 4 Mar 2023 14:49:21 +0300 Subject: [PATCH 37/63] Add notes to the validation results --- .../validation_results_mxnet_gluon_modelzoo.md | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/results/validation/validation_results_mxnet_gluon_modelzoo.md b/results/validation/validation_results_mxnet_gluon_modelzoo.md index 768bea6cf..8de5ffa00 100644 --- a/results/validation/validation_results_mxnet_gluon_modelzoo.md +++ b/results/validation/validation_results_mxnet_gluon_modelzoo.md @@ -5,6 +5,15 @@ Complete information about the supported classification models is available [here][gluon_modelzoo_classification]. +Notes: + +- For all classification models input shape BxCxWxH, where + B is a batch size, C is an image number of channels, + W is an image width, H is an image height. + W=H=224 except inceptionv3, for this model W=H=299. +- Values of mean and standard deviation parameters used + for model validation are represented for each image. + ### Test image #1 Data source: [ImageNet][imagenet] @@ -14,7 +23,7 @@ Image resolution: 709 x 510 Mean: [0.485, 0.456, 0.406] Standard deviation: [0.229, 0.224, 0.225] - +
@@ -115,7 +124,7 @@ senet_154 |0.6984364 Granny Smith
0.0078498 syringe
0.0034595 tennis ball< Data source: [ImageNet][imagenet] Image resolution: 500 x 500 - +
@@ -216,7 +225,7 @@ senet_154 |0.8654007 junco, snowbird
0.0118391 water ouzel, dipper
0.00415 Data source: [ImageNet][imagenet] Image resolution: 333 x 500 - +
From f6385364af615bcf46f5f5d99a9cd203270a67ba Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Sat, 4 Mar 2023 15:22:59 +0300 Subject: [PATCH 38/63] Remove space --- results/mxnet_models_checklist.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/results/mxnet_models_checklist.md b/results/mxnet_models_checklist.md index 4402db64a..65df33944 100644 --- a/results/mxnet_models_checklist.md +++ b/results/mxnet_models_checklist.md @@ -34,13 +34,13 @@ mobilenetv2_0.75|+|+| mobilenetv2_1.0|+|+| mobilenetv3_large|+|+| mobilenetv3_small|+|+| -residualattentionnet128|+|Error ( Parameter 'residualattentionmodel0_hybridsequential0_conv0_weight' has not been initialized)| -residualattentionnet164|+|Error ( Parameter 'residualattentionmodel0_hybridsequential0_conv0_weight' has not been initialized)| -residualattentionnet200|+|Error ( Parameter 'residualattentionmodel0_hybridsequential0_conv0_weight' has not been initialized)| -residualattentionnet236|+|Error ( Parameter 'residualattentionmodel0_hybridsequential0_conv0_weight' has not been initialized)| -residualattentionnet452|+|Error ( Parameter 'residualattentionmodel0_hybridsequential0_conv0_weight' has not been initialized)| -residualattentionnet56|+|Error ( Parameter 'residualattentionmodel0_hybridsequential0_conv0_weight' has not been initialized)| -residualattentionnet92|+|Error ( Parameter 'residualattentionmodel0_hybridsequential0_conv0_weight' has not been initialized)| +residualattentionnet128|+|Error (Parameter 'residualattentionmodel0_hybridsequential0_conv0_weight' has not been initialized)| +residualattentionnet164|+|Error (Parameter 'residualattentionmodel0_hybridsequential0_conv0_weight' has not been initialized)| +residualattentionnet200|+|Error (Parameter 'residualattentionmodel0_hybridsequential0_conv0_weight' has not been initialized)| +residualattentionnet236|+|Error (Parameter 'residualattentionmodel0_hybridsequential0_conv0_weight' has not been initialized)| +residualattentionnet452|+|Error (Parameter 'residualattentionmodel0_hybridsequential0_conv0_weight' has not been initialized)| +residualattentionnet56|+|Error (Parameter 'residualattentionmodel0_hybridsequential0_conv0_weight' has not been initialized)| +residualattentionnet92|+|Error (Parameter 'residualattentionmodel0_hybridsequential0_conv0_weight' has not been initialized)| resnest101|+|+| resnest14|+|+| resnest200|+|+| From dfd5fd4374863beb18b4b5d063fe3e83f1e39b82 Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Thu, 9 Mar 2023 18:20:36 +0300 Subject: [PATCH 39/63] Fix preparing output for classification task --- src/inference/inference_mxnet.py | 39 ++++++++++++++++++++------------ src/inference/io_adapter.py | 10 ++++++-- 2 files changed, 32 insertions(+), 17 deletions(-) diff --git a/src/inference/inference_mxnet.py b/src/inference/inference_mxnet.py index 14c38a486..b64b949d2 100644 --- a/src/inference/inference_mxnet.py +++ b/src/inference/inference_mxnet.py @@ -73,6 +73,12 @@ def cli_argument_parser(): type=int, nargs=3, dest='channel_swap') + parser.add_argument('--output_names', + help='Name of the output tensor', + default=None, + type=str, + nargs='+', + dest='output_names') parser.add_argument('-b', '--batch_size', help='Size of the processed pack', default=1, @@ -168,19 +174,6 @@ def create_dict_for_modelwrapper(args): return dictionary -def print_topk_predictions(predictions, k, file_labels): - categories = np.array(json.load(open(file_labels, 'r'))) - log.info(f'Top-{k} results:') - for prediction_idx in range(len(predictions)): - log.info(f'Result for image {prediction_idx}') - top_pred = predictions.topk(k=k)[prediction_idx].asnumpy() - for index in top_pred: - idx = int(index) - probability = predictions[prediction_idx][idx] - category = categories[idx] - log.info('\t{0:.7f} {1}'.format(probability.asscalar(), category)) - - def inference_mxnet(net, num_iterations, get_slice, input_name, k=5, file_labels='image_net_labels.json'): predictions = None @@ -221,6 +214,14 @@ def raw_result_output(average_time, fps, latency): print('{0:.3f},{1:.3f},{2:.3f}'.format(average_time, fps, latency)) +def prepare_output(result, output_names, task): + if (output_names is None) or len(output_names) == 0: + raise ValueError('The number of output tensors does not match the number of corresponding output names') + if task == 'classification': + return {output_names[0]: result.asnumpy()} + else: + raise ValueError(f'Unsupported task {task} to print inference results') + def main(): log.basicConfig( format='[ %(levelname)s ] %(message)s', @@ -259,8 +260,16 @@ def main(): if not args.raw_output: if args.number_iter == 1: - # print_topk_predictions should be implemented as io.process_output(result, log) - print_topk_predictions(result, args.number_top, args.labels) + try: + log.info('Converting output tensor to print results') + result = prepare_output(result, args.output_names, args.task) + + log.info('Inference results') + io.process_output(result, log) + except Exception as ex: + log.warning('Error when printing inference results. {0}'.format(str(ex))) + + log.info('Performance results') result_output(average_time, fps, latency) else: raw_result_output(average_time, fps, latency) diff --git a/src/inference/io_adapter.py b/src/inference/io_adapter.py index c8081aa30..96e787253 100644 --- a/src/inference/io_adapter.py +++ b/src/inference/io_adapter.py @@ -3,6 +3,8 @@ import cv2 import numpy as np +import json +import pathlib class IOAdapter(metaclass=abc.ABCMeta): @@ -237,8 +239,12 @@ def process_output(self, result, log): log.info('Top {0} results:'.format(self._number_top)) if not self._labels: self._labels = os.path.join(os.path.dirname(__file__), 'labels/image_net_synset.txt') - with open(self._labels, 'r') as f: - labels_map = [line.strip() for line in f] + file_extension = pathlib.Path(self._labels).suffix + if file_extension == '.json': + labels_map = np.array(json.load(open(self._labels, 'r'))).tolist() + else: + with open(self._labels, 'r') as f: + labels_map = [line.strip() for line in f] for batch, probs in enumerate(result): probs = np.squeeze(probs) top_ind = np.argsort(probs)[-self._number_top:][::-1] # noqa: PLE1130 From 00bad663aea56c48b686032117dbbf058d88608d Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Thu, 9 Mar 2023 18:29:21 +0300 Subject: [PATCH 40/63] Fix coding style --- src/inference/inference_mxnet.py | 3 +-- src/inference/io_adapter.py | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/inference/inference_mxnet.py b/src/inference/inference_mxnet.py index b64b949d2..ecd9493bf 100644 --- a/src/inference/inference_mxnet.py +++ b/src/inference/inference_mxnet.py @@ -2,9 +2,7 @@ import logging as log import sys from time import time -import json import warnings -import numpy as np import mxnet import gluoncv @@ -222,6 +220,7 @@ def prepare_output(result, output_names, task): else: raise ValueError(f'Unsupported task {task} to print inference results') + def main(): log.basicConfig( format='[ %(levelname)s ] %(message)s', diff --git a/src/inference/io_adapter.py b/src/inference/io_adapter.py index 96e787253..96119b500 100644 --- a/src/inference/io_adapter.py +++ b/src/inference/io_adapter.py @@ -1,10 +1,10 @@ import abc import os -import cv2 -import numpy as np import json import pathlib +import cv2 +import numpy as np class IOAdapter(metaclass=abc.ABCMeta): From 2e5bb26f9118e9aa6800897cfa3382ee65f91d34 Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Sat, 18 Mar 2023 14:09:08 +0300 Subject: [PATCH 41/63] Fix import + code cleanings + support saving model --- src/inference/inference_mxnet.py | 81 ++++++++++++++++++++++---------- src/inference/io_adapter.py | 4 +- 2 files changed, 57 insertions(+), 28 deletions(-) diff --git a/src/inference/inference_mxnet.py b/src/inference/inference_mxnet.py index ecd9493bf..18a1a639a 100644 --- a/src/inference/inference_mxnet.py +++ b/src/inference/inference_mxnet.py @@ -1,5 +1,6 @@ import argparse import logging as log +import os import sys from time import time import warnings @@ -25,17 +26,17 @@ def cli_argument_parser(): type=str, dest='model_params') parser.add_argument('-mn', '--model_name', - help='Model name to download using Gluon package.', + help='Model name to download using GluonCV package.', type=str, dest='model_name') parser.add_argument('-i', '--input', - help='Path to data', + help='Path to data.', required=True, type=str, nargs='+', dest='input') parser.add_argument('-in', '--input_name', - help='Input name', + help='Input name.', default='data', type=str, dest='input_name') @@ -43,76 +44,91 @@ def cli_argument_parser(): help='Input shape BxWxHxC, B is a batch size,' 'W is an input tensor width,' 'H is an input tensor height,' - 'C is an input tensor number of channels', + 'C is an input tensor number of channels.', required=True, type=int, nargs=4, dest='input_shape') + parser.add_argument('--norm', + help='Flag to normalize input images' + '(use --mean and --std arguments to set' + 'required normalization parameters).', + action='store_false', + dest='norm') parser.add_argument('--mean', - help='Parameter mean', + help='Mean values.', default=[0, 0, 0], type=float, nargs=3, dest='mean') parser.add_argument('--std', - help='Parameter standard deviation', + help='Standard deviation values.', default=[1., 1., 1.], type=float, nargs=3, dest='std') - parser.add_argument('--norm', - help='Flag to normalize input images', - default=True, - type=bool, - dest='norm') parser.add_argument('--channel_swap', - help='Parameter channel swap (WxHxC to CxWxH by default)', + help='Parameter of channel swap (WxHxC to CxWxH by default).', default=[2, 0, 1], type=int, nargs=3, dest='channel_swap') parser.add_argument('--output_names', - help='Name of the output tensor', + help='Name of the output tensors.', default=None, type=str, nargs='+', dest='output_names') parser.add_argument('-b', '--batch_size', - help='Size of the processed pack', + help='Batch size.', default=1, type=int, dest='batch_size') parser.add_argument('-l', '--labels', - help='Labels mapping file', + help='Labels mapping file.', default='image_net_labels.json', type=str, dest='labels') parser.add_argument('-nt', '--number_top', - help='Number of top results', + help='Number of top results.', default=5, type=int, dest='number_top') parser.add_argument('-t', '--task', - help='Output processing method. Default: without postprocess', - choices=['classification'], - default='classification', + help='Task type determines the type of output processing ' + 'method. Available values: feedforward - without' + 'postprocessing (by default), classification - output' + 'is a vector of probabilities.', + choices=['feedforward', 'classification'], + default='feedforward', type=str, dest='task') parser.add_argument('-ni', '--number_iter', - help='Number of inference iterations', + help='Number of inference iterations.', default=1, type=int, dest='number_iter') parser.add_argument('--raw_output', - help='Raw output without logs', + help='Raw output without logs.', default=False, type=bool, dest='raw_output') parser.add_argument('-d', '--device', - help='Specify the target device to infer on CPU or NVIDIA GPU (CPU by default)', + help='Specify the target device to infer on CPU or ' + 'NVIDIA GPU (CPU by default)', default='CPU', type=str, dest='device') + parser.add_argument('-s', '--save_model', + help='Flag to indicate whether the model should be saved' + '(it may be required for GluonCV-models)', + action='store_true', + dest='save_model') + parser.add_argument('-p', '--path_save_model', + help='Path to save model', + default=None, + type=str, + dest='path_save_model') args = parser.parse_args() @@ -141,10 +157,20 @@ def load_network_gluon(model_json, model_params, context, input_name): return deserialized_net -def load_network_gluon_model_zoo(model_name, context): - log.info(f'Loading network \"{model_name}\"') +def load_network_gluon_model_zoo(model_name, context, save_model, path_save_model): + log.info(f'Loading network \"{model_name}\" from GluonCV model zoo') net = gluoncv.model_zoo.get_model(model_name, pretrained=True, ctx=context) + if save_model == True: + log.info(f'Saving model \"{model_name}\" to \"{path_save_model}\"') + if path_save_model is None: + path_save_model = os.getcwd() + path_save_model = os.path.join(path_save_model, model_name) + if os.path.exists(path_save_model) == False: + os.mkdir(path_save_model) + gluoncv.utils.export_block(os.path.join(path_save_model, model_name), net, + preprocess=None, layout='CHW', ctx=context) + log.info(f'Info about the network:\n{net}') log.info('Hybridizing model to accelerate inference') @@ -213,6 +239,8 @@ def raw_result_output(average_time, fps, latency): def prepare_output(result, output_names, task): + if task == 'feedforward': + return {} if (output_names is None) or len(output_names) == 0: raise ValueError('The number of output tensors does not match the number of corresponding output names') if task == 'classification': @@ -238,14 +266,15 @@ def main(): if ((args.model_name is not None) and (args.model_json is None) and (args.model_params is None)): - net = load_network_gluon_model_zoo(args.model_name, context) + net = load_network_gluon_model_zoo(args.model_name, context, + args.save_model, args.path_save_model) elif (args.model_json is not None) and (args.model_params is not None): net = load_network_gluon(args.model_json, args.model_params, context, args.input_name) else: raise ValueError('Incorrect arguments.') - log.info(f'Shape for input layer {args.input_name}: {args.input_shape}') + log.info(f'Shape of the input layer {args.input_name}: {args.input_shape}') log.info(f'Preparing input data {args.input}') io.prepare_input(net, args.input) diff --git a/src/inference/io_adapter.py b/src/inference/io_adapter.py index 96119b500..f3db536b1 100644 --- a/src/inference/io_adapter.py +++ b/src/inference/io_adapter.py @@ -2,7 +2,7 @@ import os import json -import pathlib +from pathlib import Path import cv2 import numpy as np @@ -239,7 +239,7 @@ def process_output(self, result, log): log.info('Top {0} results:'.format(self._number_top)) if not self._labels: self._labels = os.path.join(os.path.dirname(__file__), 'labels/image_net_synset.txt') - file_extension = pathlib.Path(self._labels).suffix + file_extension = Path(self._labels).suffix if file_extension == '.json': labels_map = np.array(json.load(open(self._labels, 'r'))).tolist() else: From b6a9e0743d40356eb31855de1647d0ae338665f8 Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Sat, 18 Mar 2023 14:22:19 +0300 Subject: [PATCH 42/63] Fix readme --- src/inference/README.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/inference/README.md b/src/inference/README.md index 232bfe31f..70f558af3 100644 --- a/src/inference/README.md +++ b/src/inference/README.md @@ -504,6 +504,18 @@ inference_mxnet.py - `-ni / --number_iter` - количество прямых проходов по сети. По умолчанию выполняется один проход по сети. - `--raw_output` - работа скрипта без логов. По умолчанию не установлен. +- `-s / --save_model` - флаг, который определяет необходимость сохранения + модели, загруженной средствами GluonCV API из репозитория моделей. + По умолчанию модель не сохраняется. Наличие данной опции обусловлено + необходимостью последующего запуска компонента Accuracy Checker + для проверки качества работы модели. +- `-p / --path_save_model` - путь для сохранения файлов модели. + В процессе сохранения внутри указанной директории создается + вложенная директория с названием модели ``. + Формируется два файла: `-0000.params` - бинарный файл + с обученными параметрами модели, `-symbol.json` - архитектура + модели в формате .json. По умолчанию модель сохраняется в текущей + директории. #### Примеры запуска @@ -516,7 +528,8 @@ python inference_mxnet.py --model_name \ --input_shape \ --mean \ --std \ - --batch_size + --batch_size \ + --save_model --path_save_model ``` **Запуск вывода для модели, которая загружается из файлов** From d7ea596d5a66cb6eb5f6846404293834e50a2483 Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Sat, 18 Mar 2023 14:58:25 +0300 Subject: [PATCH 43/63] Fix smoke test --- src/benchmark/frameworks/mxnet/mxnet_process.py | 5 +++-- src/inference/inference_mxnet.py | 2 +- test/smoke_test/smoke_config.xml | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/benchmark/frameworks/mxnet/mxnet_process.py b/src/benchmark/frameworks/mxnet/mxnet_process.py index 9510a402d..8ef6b8f26 100644 --- a/src/benchmark/frameworks/mxnet/mxnet_process.py +++ b/src/benchmark/frameworks/mxnet/mxnet_process.py @@ -49,8 +49,9 @@ def _fill_command_line(self): common_params, '--input_name', input_name) normalize = self._test.dep_parameters.normalize - common_params = MXNetProcess._add_optional_argument_to_cmd_line( - common_params, '--norm', normalize) + if normalize == 'True': + common_params = MXNetProcess._add_optional_argument_to_cmd_line( + common_params, '--norm', '') mean = self._test.dep_parameters.mean common_params = MXNetProcess._add_optional_argument_to_cmd_line( diff --git a/src/inference/inference_mxnet.py b/src/inference/inference_mxnet.py index 18a1a639a..60776095c 100644 --- a/src/inference/inference_mxnet.py +++ b/src/inference/inference_mxnet.py @@ -53,7 +53,7 @@ def cli_argument_parser(): help='Flag to normalize input images' '(use --mean and --std arguments to set' 'required normalization parameters).', - action='store_false', + action='store_true', dest='norm') parser.add_argument('--mean', help='Mean values.', diff --git a/test/smoke_test/smoke_config.xml b/test/smoke_test/smoke_config.xml index 63956b532..40e14785f 100644 --- a/test/smoke_test/smoke_config.xml +++ b/test/smoke_test/smoke_config.xml @@ -146,7 +146,7 @@ 1 3 224 224 True 0.485 0.456 0.406 - 0.229 0.224 0.225 + 0.229 0.224 0.225 From 5255732f467e88648c450b9ab841d19fe0fb21a8 Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Sat, 18 Mar 2023 15:04:56 +0300 Subject: [PATCH 44/63] Fix coding style + update readme --- src/inference/README.md | 3 +-- src/inference/inference_mxnet.py | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/inference/README.md b/src/inference/README.md index 70f558af3..9a379365e 100644 --- a/src/inference/README.md +++ b/src/inference/README.md @@ -526,8 +526,7 @@ python inference_mxnet.py --model_name \ --input \ --input_name \ --input_shape \ - --mean \ - --std \ + --norm --mean --std \ --batch_size \ --save_model --path_save_model ``` diff --git a/src/inference/inference_mxnet.py b/src/inference/inference_mxnet.py index 60776095c..3ead05e1c 100644 --- a/src/inference/inference_mxnet.py +++ b/src/inference/inference_mxnet.py @@ -161,12 +161,12 @@ def load_network_gluon_model_zoo(model_name, context, save_model, path_save_mode log.info(f'Loading network \"{model_name}\" from GluonCV model zoo') net = gluoncv.model_zoo.get_model(model_name, pretrained=True, ctx=context) - if save_model == True: + if save_model is True: log.info(f'Saving model \"{model_name}\" to \"{path_save_model}\"') if path_save_model is None: path_save_model = os.getcwd() path_save_model = os.path.join(path_save_model, model_name) - if os.path.exists(path_save_model) == False: + if not os.path.exists(path_save_model): os.mkdir(path_save_model) gluoncv.utils.export_block(os.path.join(path_save_model, model_name), net, preprocess=None, layout='CHW', ctx=context) From a10dcd00498420a9108604c4d35a6dad22668b75 Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Sun, 19 Mar 2023 10:17:20 +0300 Subject: [PATCH 45/63] Print traceback --- src/inference/inference_mxnet.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/inference/inference_mxnet.py b/src/inference/inference_mxnet.py index 3ead05e1c..27684fe05 100644 --- a/src/inference/inference_mxnet.py +++ b/src/inference/inference_mxnet.py @@ -2,6 +2,7 @@ import logging as log import os import sys +import traceback from time import time import warnings @@ -301,8 +302,8 @@ def main(): result_output(average_time, fps, latency) else: raw_result_output(average_time, fps, latency) - except Exception as ex: - log.error(str(ex)) + except Exception: + log.error(traceback.format_exc()) sys.exit(1) From 5dcc5c5b518c8610ed885cc770496d69c8c907ec Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Fri, 28 Apr 2023 15:02:34 +0300 Subject: [PATCH 46/63] Revert --- src/benchmark/README.md | 1 - src/benchmark/frameworks/framework_wrapper_registry.py | 1 - src/benchmark/frameworks/known_frameworks.py | 1 - src/benchmark/inference_benchmark.py | 8 ++------ 4 files changed, 2 insertions(+), 9 deletions(-) diff --git a/src/benchmark/README.md b/src/benchmark/README.md index 438347ae7..9fa53bfec 100644 --- a/src/benchmark/README.md +++ b/src/benchmark/README.md @@ -252,4 +252,3 @@ pip install openvino_dev[mxnet,caffe,caffe2,onnx,pytorch,tensorflow2]== Date: Fri, 28 Apr 2023 15:55:48 +0300 Subject: [PATCH 47/63] Add hybrid flag --- .../frameworks/mxnet/mxnet_parameters_parser.py | 11 ++++++++++- src/benchmark/frameworks/mxnet/mxnet_process.py | 5 +++++ src/benchmark/frameworks/mxnet/mxnet_test.py | 1 + .../benchmark_configuration_file_template.xml | 1 + src/inference/inference_mxnet.py | 14 ++++++++++---- test/smoke_test/smoke_config.xml | 1 + 6 files changed, 28 insertions(+), 5 deletions(-) diff --git a/src/benchmark/frameworks/mxnet/mxnet_parameters_parser.py b/src/benchmark/frameworks/mxnet/mxnet_parameters_parser.py index ad6dbbc3d..69658e7ed 100644 --- a/src/benchmark/frameworks/mxnet/mxnet_parameters_parser.py +++ b/src/benchmark/frameworks/mxnet/mxnet_parameters_parser.py @@ -7,6 +7,7 @@ def parse_parameters(self, curr_test): CONFIG_FRAMEWORK_DEPENDENT_TAG = 'FrameworkDependent' CONFIG_FRAMEWORK_DEPENDENT_INPUT_NAME_TAG = 'InputName' CONFIG_FRAMEWORK_DEPENDENT_INPUT_SHAPE_TAG = 'InputShape' + CONFIG_FRAMEWORK_DEPENDENT_HYBRIDIZE_TAG='Hybridize' CONFIG_FRAMEWORK_DEPENDENT_NORMALIZE_TAG = 'Normalize' CONFIG_FRAMEWORK_DEPENDENT_MEAN_TAG = 'Mean' CONFIG_FRAMEWORK_DEPENDENT_STD_TAG = 'Std' @@ -18,6 +19,9 @@ def parse_parameters(self, curr_test): CONFIG_FRAMEWORK_DEPENDENT_INPUT_NAME_TAG)[0].firstChild _input_shape = dep_parameters_tag.getElementsByTagName( CONFIG_FRAMEWORK_DEPENDENT_INPUT_SHAPE_TAG)[0].firstChild + _hybridize = dep_parameters_tag.getElementsByTagName( + CONFIG_FRAMEWORK_DEPENDENT_HYBRIDIZE_TAG)[0].firstChild + print(_hybridize.data) _normalize = dep_parameters_tag.getElementsByTagName( CONFIG_FRAMEWORK_DEPENDENT_NORMALIZE_TAG)[0].firstChild _mean = dep_parameters_tag.getElementsByTagName( @@ -30,6 +34,7 @@ def parse_parameters(self, curr_test): return MXNetParameters( input_name=_input_name.data if _input_name else None, input_shape=_input_shape.data if _input_shape else None, + hybridize=_hybridize.data if _hybridize else None, normalize=_normalize.data if _normalize else None, mean=_mean.data if _mean else None, std=_std.data if _std else None, @@ -38,9 +43,11 @@ def parse_parameters(self, curr_test): class MXNetParameters(FrameworkParameters): - def __init__(self, input_name, input_shape, normalize, mean, std, channel_swap): + def __init__(self, input_name, input_shape, hybridize, normalize, + mean, std, channel_swap): self.input_name = None self.input_shape = None + self.hybridize = None self.normalize = None self.mean = None self.std = None @@ -50,6 +57,8 @@ def __init__(self, input_name, input_shape, normalize, mean, std, channel_swap): self.input_name = input_name if self._parameter_not_is_none(input_shape): self.input_shape = input_shape + if self._parameter_not_is_none(hybridize): + self.hybridize = hybridize if self._parameter_not_is_none(normalize): self.normalize = normalize if self._parameter_not_is_none(mean): diff --git a/src/benchmark/frameworks/mxnet/mxnet_process.py b/src/benchmark/frameworks/mxnet/mxnet_process.py index bae5a28c8..9fd0ed197 100644 --- a/src/benchmark/frameworks/mxnet/mxnet_process.py +++ b/src/benchmark/frameworks/mxnet/mxnet_process.py @@ -48,6 +48,11 @@ def _fill_command_line(self): common_params = MXNetProcess._add_optional_argument_to_cmd_line( common_params, '--input_name', input_name) + hybridize = self._test.dep_parameters.hybridize + if hybridize == 'True': + common_params = MXNetProcess._add_flag_to_cmd_line( + common_params, '--hybrid') + normalize = self._test.dep_parameters.normalize if normalize == 'True': common_params = MXNetProcess._add_flag_to_cmd_line( diff --git a/src/benchmark/frameworks/mxnet/mxnet_test.py b/src/benchmark/frameworks/mxnet/mxnet_test.py index 53ff48db2..e217c8139 100644 --- a/src/benchmark/frameworks/mxnet/mxnet_test.py +++ b/src/benchmark/frameworks/mxnet/mxnet_test.py @@ -13,6 +13,7 @@ def get_report(self): parameters.update({'Iteration count': self.indep_parameters.iteration}) parameters.update({'Input name': self.dep_parameters.input_name}) parameters.update({'Input shape': self.dep_parameters.input_shape}) + parameters.update({'Hybridization flag': self.dep_parameters.hybridize}) parameters.update({'Normalization flag': self.dep_parameters.normalize}) parameters.update({'Mean': self.dep_parameters.mean}) parameters.update({'Standard deviation': self.dep_parameters.std}) diff --git a/src/configs/benchmark_configuration_file_template.xml b/src/configs/benchmark_configuration_file_template.xml index c01392a37..c532d8b54 100644 --- a/src/configs/benchmark_configuration_file_template.xml +++ b/src/configs/benchmark_configuration_file_template.xml @@ -175,6 +175,7 @@ + diff --git a/src/inference/inference_mxnet.py b/src/inference/inference_mxnet.py index afd73cbed..9f9cdbfcc 100644 --- a/src/inference/inference_mxnet.py +++ b/src/inference/inference_mxnet.py @@ -30,6 +30,11 @@ def cli_argument_parser(): help='Model name to download using GluonCV package.', type=str, dest='model_name') + parser.add_argument('--hybrid', + help='Flag to enable symbolic computations.' + 'Default value is false.', + action='store_true', + dest='hybrid') parser.add_argument('-i', '--input', help='Path to data.', required=True, @@ -158,7 +163,7 @@ def load_network_gluon(model_json, model_params, context, input_name): return deserialized_net -def load_network_gluon_model_zoo(model_name, context, save_model, path_save_model): +def load_network_gluon_model_zoo(model_name, hybrid, context, save_model, path_save_model): log.info(f'Loading network \"{model_name}\" from GluonCV model zoo') net = gluoncv.model_zoo.get_model(model_name, pretrained=True, ctx=context) @@ -174,8 +179,9 @@ def load_network_gluon_model_zoo(model_name, context, save_model, path_save_mode log.info(f'Info about the network:\n{net}') - log.info('Hybridizing model to accelerate inference') - net.hybridize() + log.info(f'Hybridizing model to accelerate inference: {hybrid}') + if hybrid is True: + net.hybridize() return net @@ -269,7 +275,7 @@ def main(): if ((args.model_name is not None) and (args.model_json is None) and (args.model_params is None)): - net = load_network_gluon_model_zoo(args.model_name, context, + net = load_network_gluon_model_zoo(args.model_name, args.hybrid, context, args.save_model, args.path_save_model) elif (args.model_json is not None) and (args.model_params is not None): net = load_network_gluon(args.model_json, args.model_params, context, diff --git a/test/smoke_test/smoke_config.xml b/test/smoke_test/smoke_config.xml index db3c8963f..a9323935e 100644 --- a/test/smoke_test/smoke_config.xml +++ b/test/smoke_test/smoke_config.xml @@ -144,6 +144,7 @@ data 1 3 224 224 + True True 0.485 0.456 0.406 0.229 0.224 0.225 From 1a35f3cc3711837cadadeb2bd8b77ea2a311ad40 Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Fri, 28 Apr 2023 16:01:15 +0300 Subject: [PATCH 48/63] Fix codeing style --- src/benchmark/frameworks/mxnet/mxnet_parameters_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/benchmark/frameworks/mxnet/mxnet_parameters_parser.py b/src/benchmark/frameworks/mxnet/mxnet_parameters_parser.py index 69658e7ed..283a47c7c 100644 --- a/src/benchmark/frameworks/mxnet/mxnet_parameters_parser.py +++ b/src/benchmark/frameworks/mxnet/mxnet_parameters_parser.py @@ -7,7 +7,7 @@ def parse_parameters(self, curr_test): CONFIG_FRAMEWORK_DEPENDENT_TAG = 'FrameworkDependent' CONFIG_FRAMEWORK_DEPENDENT_INPUT_NAME_TAG = 'InputName' CONFIG_FRAMEWORK_DEPENDENT_INPUT_SHAPE_TAG = 'InputShape' - CONFIG_FRAMEWORK_DEPENDENT_HYBRIDIZE_TAG='Hybridize' + CONFIG_FRAMEWORK_DEPENDENT_HYBRIDIZE_TAG = 'Hybridize' CONFIG_FRAMEWORK_DEPENDENT_NORMALIZE_TAG = 'Normalize' CONFIG_FRAMEWORK_DEPENDENT_MEAN_TAG = 'Mean' CONFIG_FRAMEWORK_DEPENDENT_STD_TAG = 'Std' From e516ea763d39a39e2c3f21c2261fbbca19a96584 Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Mon, 1 May 2023 07:34:17 +0300 Subject: [PATCH 49/63] Remove input_name and input_shape from report parameters + replace first dim in input_shape by batch_size --- src/benchmark/frameworks/mxnet/mxnet_test.py | 2 -- src/inference/inference_mxnet.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/benchmark/frameworks/mxnet/mxnet_test.py b/src/benchmark/frameworks/mxnet/mxnet_test.py index e217c8139..a0424563c 100644 --- a/src/benchmark/frameworks/mxnet/mxnet_test.py +++ b/src/benchmark/frameworks/mxnet/mxnet_test.py @@ -11,8 +11,6 @@ def get_report(self): parameters = OrderedDict() parameters.update({'Device': self.indep_parameters.device}) parameters.update({'Iteration count': self.indep_parameters.iteration}) - parameters.update({'Input name': self.dep_parameters.input_name}) - parameters.update({'Input shape': self.dep_parameters.input_shape}) parameters.update({'Hybridization flag': self.dep_parameters.hybridize}) parameters.update({'Normalization flag': self.dep_parameters.normalize}) parameters.update({'Mean': self.dep_parameters.mean}) diff --git a/src/inference/inference_mxnet.py b/src/inference/inference_mxnet.py index 9f9cdbfcc..ce7a93f04 100644 --- a/src/inference/inference_mxnet.py +++ b/src/inference/inference_mxnet.py @@ -200,7 +200,7 @@ def create_dict_for_transformer(args): def create_dict_for_modelwrapper(args): dictionary = { 'input_name': args.input_name, - 'input_shape': args.input_shape, + 'input_shape': [args.batch_size] + args.input_shape[1:4], } return dictionary From 55fca5a2cc7c4464b5c10c7f0c33e1bda76fcd72 Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Mon, 1 May 2023 07:44:14 +0300 Subject: [PATCH 50/63] Fix converter (add parameters head) --- src/csv2xlsx/benchmark_table_creator.py | 195 ++++++++++++++++-------- src/csv2xlsx/table_creator.py | 4 + 2 files changed, 137 insertions(+), 62 deletions(-) diff --git a/src/csv2xlsx/benchmark_table_creator.py b/src/csv2xlsx/benchmark_table_creator.py index 3be222e54..042ae6625 100644 --- a/src/csv2xlsx/benchmark_table_creator.py +++ b/src/csv2xlsx/benchmark_table_creator.py @@ -97,7 +97,7 @@ def _get_devices(self): for inference_framework in machine_inference_frameworks: framework_devices = [] for key, value in self._data_dictionary[self._KEY_PARAMETERS].items(): - pattern = re.compile(r'[.]*Device:[ ]*(?P[^,]+)[,]+[.]*') + pattern = re.compile(r'[.]*Device:[ ]*(?P[^,]+)[,]+[ ]*(?P[\w\W\d\.,\[\];:]+)') matcher = re.match(pattern, value) device_name = matcher.group('device_name') if (self._data_dictionary[self._KEY_INFRASTRUCTURE][key] == machine @@ -124,7 +124,7 @@ def _get_precisions(self): for device in framework_devices: device_precisions = [] for key, value in self._data_dictionary[self._KEY_PARAMETERS].items(): - pattern = re.compile(r'[.]*Device:[ ]*(?P[^,]+)[,]+[.]*') + pattern = re.compile(r'[.]*Device:[ ]*(?P[^,]+)[,]+[ ]*(?P[\w\W\d\.,\[\];:]+)') matcher = re.match(pattern, value) device_name = matcher.group('device_name') if (self._data_dictionary[self._KEY_INFRASTRUCTURE][key] == machine @@ -157,9 +157,10 @@ def _get_execution_modes(self): for precision in device_precisions: device_precision_modes = [] for key, value in self._data_dictionary[self._KEY_PARAMETERS].items(): - pattern = re.compile(r'[.]*Device:[ ]*(?P[^,]+)[,]+[.]*') + pattern = re.compile(r'[.]*Device:[ ]*(?P[^,]+)[,]+[ ]*(?P[\w\W\d\.,\[\];:]+)') matcher = re.match(pattern, value) device_name = matcher.group('device_name') + parameters = matcher.group('parameters') if (self._data_dictionary[self._KEY_INFRASTRUCTURE][key] == machine and self._data_dictionary[self._KEY_INFERENCE_FRAMEWORK][key] == inference_framework and device_name == device @@ -175,10 +176,52 @@ def _get_execution_modes(self): logging.info(f'FINISH: _get_execution_modes(). {self._execution_modes}') + def _get_execution_parameters(self): + logging.info('START: _get_execution_parameters()') + + self._execution_parameters = [] + for idx in range(len(self._infrastructure)): + machine = self._infrastructure[idx] + machine_frameworks = self._inference_frameworks[idx] + machine_execparams = [] + for idx2 in range(len(machine_frameworks)): + framework = machine_frameworks[idx2] + machine_framework_devices = self._devices[idx][idx2] + machine_framework_execparams = [] + for idx3 in range(len(machine_framework_devices)): + device = machine_framework_devices[idx3] + machine_framework_device_precisions = self._precisions[idx][idx2][idx3] + framework_device_execparams = [] + for idx4 in range(len(machine_framework_device_precisions)): + precision = machine_framework_device_precisions[idx4] + execution_modes = self._execution_modes[idx][idx2][idx3][idx4] + framework_device_precision_execparams = [] + for mode in execution_modes: + framework_device_precision_mode_execparams = [] + for key, value in self._data_dictionary[self._KEY_PARAMETERS].items(): + pattern = re.compile(r'[.]*Device:[ ]*(?P[^,]+)[,]+[ ]*(?P[\w\W\d\.,\[\];:]+)') + matcher = re.match(pattern, value) + device_name = matcher.group('device_name') + parameters = matcher.group('parameters') + if (self._data_dictionary[self._KEY_INFRASTRUCTURE][key] == machine + and self._data_dictionary[self._KEY_INFERENCE_FRAMEWORK][key] == framework + and device_name == device + and self._data_dictionary[self._KEY_PRECISION][key] == precision + and self._data_dictionary[self._KEY_EXECUTION_MODE][key] == mode + and parameters not in framework_device_precision_mode_execparams): + framework_device_precision_mode_execparams.append(parameters) + framework_device_precision_execparams.append(framework_device_precision_mode_execparams) + framework_device_execparams.append(framework_device_precision_execparams) + machine_framework_execparams.append(framework_device_execparams) + machine_execparams.append(machine_framework_execparams) + self._execution_parameters.append(machine_execparams) + + logging.info(f'FINISH: _get_execution_parameters(). {self._execution_parameters}') + def _fill_horizontal_title(self): logging.info('START: _fill_horizontal_title()') - rel_row_idx = 4 + rel_row_idx = 5 rel_col_idx = 5 self._col_indeces = [] @@ -187,85 +230,109 @@ def _fill_horizontal_title(self): col_idx = rel_col_idx num_cols = 0 machine = self._infrastructure[idx] - machine_inference_frameworks = self._inference_frameworks[idx] - col_idx3 = col_idx + machine_frameworks = self._inference_frameworks[idx] + col_idx4 = col_idx col_indeces2 = [] - for idx2 in range(len(machine_inference_frameworks)): - machine_framework = machine_inference_frameworks[idx2] + for idx2 in range(len(machine_frameworks)): + machine_framework = machine_frameworks[idx2] machine_framework_devices = self._devices[idx][idx2] col_idx2 = col_idx + col_idx3 = col_idx num_cols2 = 0 + num_cols3 = 0 col_indeces3 = [] for idx3 in range(len(machine_framework_devices)): machine_framework_device = machine_framework_devices[idx3] - framework_device_precisions = self._precisions[idx][idx2][idx3] + machine_framework_device_precisions = self._precisions[idx][idx2][idx3] col_idx1 = col_idx num_cols1 = 0 col_indeces4 = [] - for idx4 in range(len(framework_device_precisions)): - framework_device_precision = framework_device_precisions[idx4] - framework_device_precision_modes = self._execution_modes[idx][idx2][idx3][idx4] + for idx4 in range(len(machine_framework_device_precisions)): + machine_framework_device_precision = machine_framework_device_precisions[idx4] + machine_framework_device_precision_modes = self._execution_modes[idx][idx2][idx3][idx4] col_indeces5 = [] - for idx5 in range(len(framework_device_precision_modes)): - framework_device_precision_mode = framework_device_precision_modes[idx5] - self._sheet.write(row_idx, col_idx + idx5, - framework_device_precision_mode, - self._cell_format_title1) - col_indeces5.append(col_idx + idx5) - k = len(framework_device_precision_modes) - if k > 1: - self._sheet.merge_range(row_idx - 1, col_idx1, - row_idx - 1, col_idx1 + k - 1, - framework_device_precision, + for idx5 in range(len(machine_framework_device_precision_modes)): + machine_framework_device_precision_mode = machine_framework_device_precision_modes[idx5] + machine_framework_device_precision_mode_execparams = self._execution_parameters[idx][idx2][idx3][idx4][idx5] + col_indeces6 = [] + for idx6 in range(len(machine_framework_device_precision_mode_execparams)): + execparams = machine_framework_device_precision_mode_execparams[idx6] + self._sheet.write(row_idx, col_idx + idx6, + execparams, + self._cell_format_title1) + col_indeces6.append(col_idx + idx6) + k = len(machine_framework_device_precision_mode_execparams) + if k > 1: + self._sheet.merge_range(row_idx - 1, col_idx1, + row_idx - 1, col_idx1 + k - 1, + machine_framework_device_precision_mode, + self._cell_format_title1) + elif k == 1: + self._sheet.write(row_idx - 1, col_idx1, + machine_framework_device_precision_mode, + self._cell_format_title1) + else: + msg = 'Incorrect number of execution modes' + logging.error(msg) + raise ValueError(msg) + col_idx += k + col_idx1 += k + num_cols1 += k + num_cols2 += k + num_cols3 += k + num_cols += k + col_indeces5.append(col_indeces6) + if num_cols1 > 1: + self._sheet.merge_range(row_idx - 2, col_idx2, + row_idx - 2, col_idx2 + num_cols1 - 1, + machine_framework_device_precision, self._cell_format_title1) - elif k == 1: - self._sheet.write(row_idx - 1, col_idx1, - framework_device_precision, + elif num_cols1 == 1: + self._sheet.write(row_idx - 2, col_idx2, + machine_framework_device_precision, self._cell_format_title1) else: - msg = 'Incorrect number of device precision modes' + msg = 'Incorrect number of precisions' logging.error(msg) raise ValueError(msg) - col_idx += k - col_idx1 += k - num_cols1 += k - num_cols2 += k - num_cols += k + col_idx2 += num_cols1 col_indeces4.append(col_indeces5) - if num_cols1 > 1: - self._sheet.merge_range(row_idx - 2, col_idx2, - row_idx - 2, col_idx2 + num_cols1 - 1, + if num_cols2 > 1: + self._sheet.merge_range(row_idx - 3, col_idx3, + row_idx - 3, col_idx3 + num_cols2 - 1, machine_framework_device, self._cell_format_title1) - elif num_cols1 == 1: - self._sheet.write(row_idx - 2, col_idx2, + elif num_cols2 == 1: + self._sheet.write(row_idx - 3, col_idx3, machine_framework_device, self._cell_format_title1) else: msg = 'Incorrect number of devices' logging.error(msg) raise ValueError(msg) - col_idx2 += num_cols1 + col_idx3 += num_cols2 col_indeces3.append(col_indeces4) - if num_cols2 > 1: - self._sheet.merge_range(row_idx - 3, col_idx3, - row_idx - 3, col_idx3 + num_cols2 - 1, - machine_framework, self._cell_format_title1) - elif num_cols2 == 1: - self._sheet.write(row_idx - 3, col_idx3, - machine_framework, self._cell_format_title1) + if num_cols3 > 1: + self._sheet.merge_range(row_idx - 4, col_idx4, + row_idx - 4, col_idx4 + num_cols3 - 1, + machine_framework, + self._cell_format_title1) + elif num_cols3 == 1: + self._sheet.write(row_idx - 4, col_idx4 + num_cols3 - 1, + machine_framework, + self._cell_format_title1) else: msg = 'Incorrect number of frameworks' logging.error(msg) raise ValueError(msg) - col_idx3 += num_cols2 + col_idx4 += num_cols3 col_indeces2.append(col_indeces3) if num_cols > 1: - self._sheet.merge_range(row_idx - 4, rel_col_idx, - row_idx - 4, rel_col_idx + num_cols - 1, + self._sheet.merge_range(row_idx - 5, rel_col_idx, + row_idx - 5, rel_col_idx + num_cols - 1, machine, self._cell_format_title1) elif num_cols == 1: - self._sheet.write(row_idx - 4, rel_col_idx, + self._sheet.write(row_idx - 5, rel_col_idx, machine, self._cell_format_title1) else: msg = 'Incorrect number of machines' @@ -286,33 +353,34 @@ def create_table_header(self): self._init_xlsx_parameters() # Freeze title panes - self._sheet.freeze_panes(5, 5) + self._sheet.freeze_panes(6, 5) # Write horizontal title (first cells before infrastructure) - self._sheet.merge_range('A1:A5', self._KEY_TASK_TYPE, self._cell_format_title1) + self._sheet.merge_range('A1:A6', self._KEY_TASK_TYPE, self._cell_format_title1) col_width = XlsxTable._get_column_width( self._data_dictionary[self._KEY_TOPOLOGY_NAME], self._cell_format) self._sheet.set_column(1, 1, col_width) - self._sheet.merge_range('B1:B5', self._KEY_TOPOLOGY_NAME, self._cell_format_title1) + self._sheet.merge_range('B1:B6', self._KEY_TOPOLOGY_NAME, self._cell_format_title1) col_width = XlsxTable._get_column_width( self._data_dictionary[self._KEY_TRAIN_FRAMEWORK], self._cell_format) self._sheet.set_column(2, 2, col_width) - self._sheet.merge_range('C1:C5', self._KEY_TRAIN_FRAMEWORK, self._cell_format_title1) + self._sheet.merge_range('C1:C6', self._KEY_TRAIN_FRAMEWORK, self._cell_format_title1) self._add_new_line(self._data_dictionary[self._KEY_BLOB_SIZE]) col_width = XlsxTable._get_column_width( self._data_dictionary[self._KEY_BLOB_SIZE], self._cell_format) self._sheet.set_column(3, 3, col_width) - self._sheet.merge_range('D1:D5', self._KEY_BLOB_SIZE, self._cell_format_title1) - self._sheet.merge_range('E1:E5', self._KEY_BATCH_SIZE, self._cell_format_title1) + self._sheet.merge_range('D1:D6', self._KEY_BLOB_SIZE, self._cell_format_title1) + self._sheet.merge_range('E1:E6', self._KEY_BATCH_SIZE, self._cell_format_title1) self._get_infrastructure() self._get_inference_frameworks() self._get_devices() self._get_precisions() self._get_execution_modes() + self._get_execution_parameters() # Write horizontal title (cells corresponding infrastructure) self._fill_horizontal_title() @@ -324,15 +392,18 @@ def _find_column_idx(self, value): self._infrastructure) idx2 = self._find_inference_framework_idx(value[self._KEY_INFERENCE_FRAMEWORK], self._inference_frameworks[idx1]) - pattern = re.compile(r'[.]*Device:[ ]*(?P[^,]+)[,]+[.]*') + pattern = re.compile(r'[.]*Device:[ ]*(?P[^,]+)[,]+[ ]*(?P[\w\W\d\.,\[\];:]+)') matcher = re.match(pattern, value[self._KEY_PARAMETERS]) device_name = matcher.group('device_name') + parameters = matcher.group('parameters') idx3 = self._find_device_idx(device_name, self._devices[idx1][idx2]) idx4 = self._find_precision_idx(value[self._KEY_PRECISION], self._precisions[idx1][idx2][idx3]) idx5 = self._find_execution_mode_idx(value[self._KEY_EXECUTION_MODE], self._execution_modes[idx1][idx2][idx3][idx4]) - return self._col_indeces[idx1][idx2][idx3][idx4][idx5] + idx6 = self._find_execparams_idx(parameters, + self._execution_parameters[idx1][idx2][idx3][idx4][idx5]) + return self._col_indeces[idx1][idx2][idx3][idx4][idx5][idx6] def _find_row_records(self, task_type, topology_name, train_framework, blob_size, batch_size, experiments, @@ -407,7 +478,7 @@ def _get_records_group(self, task_records, topology_name, train_framework, def write_test_results(self): logging.info('START: write_test_results()') - row_idx = 5 + row_idx = 6 for task_type, task_records in self._table_records.items(): # loop by tasks if len(task_records) <= 0: continue @@ -476,15 +547,15 @@ def beautify_table(self): rel_col_idx = 5 # task type, topology, framework, blob sizes, batch size rel_row_idx = 0 - num_header_rows = 5 # infrastructure, framework, device, precision, mode - col_depth = 3 + num_header_rows = 6 # infrastructure, framework, device, precision, mode, parameters + col_depth = 4 self._draw_bold_bolder(0, 0, num_header_rows, rel_col_idx) self._draw_bold_bolder(num_header_rows - 1, 0, self._full_num_rows - num_header_rows + 1, rel_col_idx) for idx in range(len(self._infrastructure)): - execution_modes = list(deepflatten(self._execution_modes[idx], depth=col_depth)) - num_cols = len(execution_modes) + execution_parameters = list(deepflatten(self._execution_parameters[idx], depth=col_depth)) + num_cols = len(execution_parameters) self._draw_bold_bolder(rel_row_idx, rel_col_idx, num_header_rows, num_cols) self._draw_bold_bolder(rel_row_idx + num_header_rows - 1, rel_col_idx, self._full_num_rows - num_header_rows + 1, num_cols) diff --git a/src/csv2xlsx/table_creator.py b/src/csv2xlsx/table_creator.py index 46b3bf116..316ede0e5 100644 --- a/src/csv2xlsx/table_creator.py +++ b/src/csv2xlsx/table_creator.py @@ -117,6 +117,10 @@ def _find_execution_mode_idx(self, execution_mode, return self._find_idx(execution_mode, available_execution_modes, 'Unknown execution mode') + def _find_execparams_idx(self, parameters, available_parameters): + return self._find_idx(parameters, available_parameters, + 'Unknown param') + @abc.abstractmethod def _find_column_idx(self, value): pass From 0681e5d2328ff589a9fcf67054e2ad8846116191 Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Mon, 1 May 2023 14:42:37 +0300 Subject: [PATCH 51/63] Fix coding style --- src/csv2xlsx/benchmark_table_creator.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/csv2xlsx/benchmark_table_creator.py b/src/csv2xlsx/benchmark_table_creator.py index 042ae6625..355c64035 100644 --- a/src/csv2xlsx/benchmark_table_creator.py +++ b/src/csv2xlsx/benchmark_table_creator.py @@ -97,7 +97,8 @@ def _get_devices(self): for inference_framework in machine_inference_frameworks: framework_devices = [] for key, value in self._data_dictionary[self._KEY_PARAMETERS].items(): - pattern = re.compile(r'[.]*Device:[ ]*(?P[^,]+)[,]+[ ]*(?P[\w\W\d\.,\[\];:]+)') + pattern = re.compile(r'[.]*Device:[ ]*(?P[^,]+)[,]+[ ]*' + r'(?P[\w\W\d\.,\[\];:]+)') matcher = re.match(pattern, value) device_name = matcher.group('device_name') if (self._data_dictionary[self._KEY_INFRASTRUCTURE][key] == machine @@ -124,7 +125,8 @@ def _get_precisions(self): for device in framework_devices: device_precisions = [] for key, value in self._data_dictionary[self._KEY_PARAMETERS].items(): - pattern = re.compile(r'[.]*Device:[ ]*(?P[^,]+)[,]+[ ]*(?P[\w\W\d\.,\[\];:]+)') + pattern = re.compile(r'[.]*Device:[ ]*(?P[^,]+)[,]+[ ]*' + r'(?P[\w\W\d\.,\[\];:]+)') matcher = re.match(pattern, value) device_name = matcher.group('device_name') if (self._data_dictionary[self._KEY_INFRASTRUCTURE][key] == machine @@ -157,10 +159,10 @@ def _get_execution_modes(self): for precision in device_precisions: device_precision_modes = [] for key, value in self._data_dictionary[self._KEY_PARAMETERS].items(): - pattern = re.compile(r'[.]*Device:[ ]*(?P[^,]+)[,]+[ ]*(?P[\w\W\d\.,\[\];:]+)') + pattern = re.compile(r'[.]*Device:[ ]*(?P[^,]+)[,]+[ ]*' + r'(?P[\w\W\d\.,\[\];:]+)') matcher = re.match(pattern, value) device_name = matcher.group('device_name') - parameters = matcher.group('parameters') if (self._data_dictionary[self._KEY_INFRASTRUCTURE][key] == machine and self._data_dictionary[self._KEY_INFERENCE_FRAMEWORK][key] == inference_framework and device_name == device @@ -199,7 +201,8 @@ def _get_execution_parameters(self): for mode in execution_modes: framework_device_precision_mode_execparams = [] for key, value in self._data_dictionary[self._KEY_PARAMETERS].items(): - pattern = re.compile(r'[.]*Device:[ ]*(?P[^,]+)[,]+[ ]*(?P[\w\W\d\.,\[\];:]+)') + pattern = re.compile(r'[.]*Device:[ ]*(?P[^,]+)[,]+[ ]*' + r'(?P[\w\W\d\.,\[\];:]+)') matcher = re.match(pattern, value) device_name = matcher.group('device_name') parameters = matcher.group('parameters') @@ -253,7 +256,8 @@ def _fill_horizontal_title(self): col_indeces5 = [] for idx5 in range(len(machine_framework_device_precision_modes)): machine_framework_device_precision_mode = machine_framework_device_precision_modes[idx5] - machine_framework_device_precision_mode_execparams = self._execution_parameters[idx][idx2][idx3][idx4][idx5] + machine_framework_device_precision_mode_execparams = \ + self._execution_parameters[idx][idx2][idx3][idx4][idx5] col_indeces6 = [] for idx6 in range(len(machine_framework_device_precision_mode_execparams)): execparams = machine_framework_device_precision_mode_execparams[idx6] From ee0899bbbdf64e7f6650698321d24c46280715c3 Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Mon, 1 May 2023 14:49:05 +0300 Subject: [PATCH 52/63] Fix coding style --- src/csv2xlsx/benchmark_table_creator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/csv2xlsx/benchmark_table_creator.py b/src/csv2xlsx/benchmark_table_creator.py index 355c64035..bd0e81b1f 100644 --- a/src/csv2xlsx/benchmark_table_creator.py +++ b/src/csv2xlsx/benchmark_table_creator.py @@ -256,8 +256,8 @@ def _fill_horizontal_title(self): col_indeces5 = [] for idx5 in range(len(machine_framework_device_precision_modes)): machine_framework_device_precision_mode = machine_framework_device_precision_modes[idx5] - machine_framework_device_precision_mode_execparams = \ - self._execution_parameters[idx][idx2][idx3][idx4][idx5] + machine_framework_device_precision_mode_execparams = ( + self._execution_parameters[idx][idx2][idx3][idx4][idx5]) col_indeces6 = [] for idx6 in range(len(machine_framework_device_precision_mode_execparams)): execparams = machine_framework_device_precision_mode_execparams[idx6] From ced99b86fc7b94fe510c62c6b6655d1f3a797daa Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Tue, 2 May 2023 15:05:23 +0300 Subject: [PATCH 53/63] Fix table beautication --- src/csv2xlsx/benchmark_table_creator.py | 8 +- src/csv2xlsx/table_creator.py | 109 +++++++++++++++--------- 2 files changed, 74 insertions(+), 43 deletions(-) diff --git a/src/csv2xlsx/benchmark_table_creator.py b/src/csv2xlsx/benchmark_table_creator.py index bd0e81b1f..8077383cf 100644 --- a/src/csv2xlsx/benchmark_table_creator.py +++ b/src/csv2xlsx/benchmark_table_creator.py @@ -554,15 +554,15 @@ def beautify_table(self): num_header_rows = 6 # infrastructure, framework, device, precision, mode, parameters col_depth = 4 self._draw_bold_bolder(0, 0, num_header_rows, rel_col_idx) - self._draw_bold_bolder(num_header_rows - 1, 0, - self._full_num_rows - num_header_rows + 1, + self._draw_bold_bolder(num_header_rows, 0, + self._full_num_rows - num_header_rows, rel_col_idx) for idx in range(len(self._infrastructure)): execution_parameters = list(deepflatten(self._execution_parameters[idx], depth=col_depth)) num_cols = len(execution_parameters) self._draw_bold_bolder(rel_row_idx, rel_col_idx, num_header_rows, num_cols) - self._draw_bold_bolder(rel_row_idx + num_header_rows - 1, rel_col_idx, - self._full_num_rows - num_header_rows + 1, num_cols) + self._draw_bold_bolder(rel_row_idx + num_header_rows, rel_col_idx, + self._full_num_rows - num_header_rows, num_cols) rel_col_idx += num_cols logging.info('FINISH: beautify_table()') diff --git a/src/csv2xlsx/table_creator.py b/src/csv2xlsx/table_creator.py index 316ede0e5..bda78685e 100644 --- a/src/csv2xlsx/table_creator.py +++ b/src/csv2xlsx/table_creator.py @@ -126,43 +126,11 @@ def _find_column_idx(self, value): pass def _draw_bold_bolder(self, rel_row_idx, rel_col_idx, num_rows, num_cols): - # top left corner - self._sheet.conditional_format(rel_row_idx, rel_col_idx, - rel_row_idx, rel_col_idx, - {'type': 'formula', 'criteria': 'True', - 'format': self._book.add_format({'top': 5, - 'bottom': 1, - 'left': 5, - 'right': 1})}) - # bottom left corner - self._sheet.conditional_format(rel_row_idx + num_rows - 1, rel_col_idx, - rel_row_idx + num_rows - 1, rel_col_idx, - {'type': 'formula', 'criteria': 'True', - 'format': self._book.add_format({'top': 1, - 'bottom': 5, - 'left': 5, - 'right': 1})}) - # top right corner - self._sheet.conditional_format(rel_row_idx, rel_col_idx + num_cols - 1, - rel_row_idx, rel_col_idx + num_cols - 1, - {'type': 'formula', 'criteria': 'True', - 'format': self._book.add_format({'top': 5, - 'bottom': 1, - 'left': 1, - 'right': 5})}) - # bottom right corner - self._sheet.conditional_format(rel_row_idx + num_rows - 1, rel_col_idx + num_cols - 1, - rel_row_idx + num_rows - 1, rel_col_idx + num_cols - 1, - {'type': 'formula', 'criteria': 'True', - 'format': self._book.add_format({'top': 1, - 'bottom': 5, - 'left': 1, - 'right': 5})}) if (num_cols > 2): # top self._sheet.conditional_format(rel_row_idx, rel_col_idx + 1, rel_row_idx, rel_col_idx + num_cols - 2, - {'type': 'formula', 'criteria': 'True', + {'type': 'no_blanks', 'format': self._book.add_format({'top': 5, 'bottom': 1, 'left': 1, @@ -170,28 +138,91 @@ def _draw_bold_bolder(self, rel_row_idx, rel_col_idx, num_rows, num_cols): # bottom self._sheet.conditional_format(rel_row_idx + num_rows - 1, rel_col_idx + 1, rel_row_idx + num_rows - 1, rel_col_idx + num_cols - 2, - {'type': 'formula', 'criteria': 'True', + {'type': 'no_blanks', 'format': self._book.add_format({'top': 1, 'bottom': 5, 'left': 1, 'right': 1})}) if (num_rows > 2): + # right + self._sheet.conditional_format(rel_row_idx + 1, rel_col_idx + num_cols - 1, + rel_row_idx + num_rows - 2, rel_col_idx + num_cols - 1, + {'type': 'no_blanks', + 'format': self._book.add_format({'top': 1, + 'bottom': 1, + 'left': 1, + 'right': 5})}) # left self._sheet.conditional_format(rel_row_idx + 1, rel_col_idx, rel_row_idx + num_rows - 2, rel_col_idx, - {'type': 'formula', 'criteria': 'True', + {'type': 'no_blanks', 'format': self._book.add_format({'top': 1, 'bottom': 1, 'left': 5, 'right': 1})}) - # right - self._sheet.conditional_format(rel_row_idx + 1, rel_col_idx + num_cols - 1, - rel_row_idx + num_rows - 2, rel_col_idx + num_cols - 1, - {'type': 'formula', 'criteria': 'True', + if num_cols == 1: + self._sheet.conditional_format(rel_row_idx, rel_col_idx, + rel_row_idx, rel_col_idx, + {'type': 'no_blanks', + 'format': self._book.add_format({'top': 5, + 'bottom': 1, + 'left': 5, + 'right': 5})}) + self._sheet.conditional_format(rel_row_idx + num_rows - 1, rel_col_idx, + rel_row_idx + num_rows - 1, rel_col_idx, + {'type': 'no_blanks', + 'format': self._book.add_format({'top': 1, + 'bottom': 5, + 'left': 5, + 'right': 5})}) + if num_rows == 1: + self._sheet.conditional_format(rel_row_idx, rel_col_idx, + rel_row_idx, rel_col_idx, + {'type': 'no_blanks', + 'format': self._book.add_format({'top': 5, + 'bottom': 5, + 'left': 5, + 'right': 1})}) + self._sheet.conditional_format(rel_row_idx, rel_col_idx + num_cols - 1, + rel_row_idx, rel_col_idx + num_cols - 1, + {'type': 'no_blanks', + 'format': self._book.add_format({'top': 5, + 'bottom': 5, + 'left': 1, + 'right': 5})}) + if num_rows > 1 and num_cols > 1: + # top left corner + self._sheet.conditional_format(rel_row_idx, rel_col_idx, + rel_row_idx, rel_col_idx, + {'type': 'no_blanks', + 'format': self._book.add_format({'top': 5, + 'bottom': 1, + 'left': 5, + 'right': 1})}) + # bottom left corner + self._sheet.conditional_format(rel_row_idx + num_rows - 1, rel_col_idx, + rel_row_idx + num_rows - 1, rel_col_idx, + {'type': 'no_blanks', 'format': self._book.add_format({'top': 1, + 'bottom': 5, + 'left': 5, + 'right': 1})}) + # top right corner + self._sheet.conditional_format(rel_row_idx, rel_col_idx + num_cols - 1, + rel_row_idx, rel_col_idx + num_cols - 1, + {'type': 'no_blanks', + 'format': self._book.add_format({'top': 5, 'bottom': 1, 'left': 1, 'right': 5})}) + # bottom right corner + self._sheet.conditional_format(rel_row_idx + num_rows - 1, rel_col_idx + num_cols - 1, + rel_row_idx + num_rows - 1, rel_col_idx + num_cols - 1, + {'type': 'no_blanks', + 'format': self._book.add_format({'top': 1, + 'bottom': 5, + 'left': 1, + 'right': 5})}) @abc.abstractmethod def read_csv_table(self): From 8090fad4ff09e887335b86aa8aa0f9f87a7c7613 Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Sat, 13 May 2023 15:58:10 +0300 Subject: [PATCH 54/63] Fix converter multiframework data --- src/csv2xlsx/benchmark_table_creator.py | 105 ++++++++++++------------ 1 file changed, 53 insertions(+), 52 deletions(-) diff --git a/src/csv2xlsx/benchmark_table_creator.py b/src/csv2xlsx/benchmark_table_creator.py index 8077383cf..d166c7800 100644 --- a/src/csv2xlsx/benchmark_table_creator.py +++ b/src/csv2xlsx/benchmark_table_creator.py @@ -229,84 +229,88 @@ def _fill_horizontal_title(self): self._col_indeces = [] for idx in range(len(self._infrastructure)): - row_idx = rel_row_idx - col_idx = rel_col_idx - num_cols = 0 machine = self._infrastructure[idx] machine_frameworks = self._inference_frameworks[idx] - col_idx4 = col_idx + + row_idx = rel_row_idx + + col_idx2 = rel_col_idx + num_cols2 = 0 col_indeces2 = [] for idx2 in range(len(machine_frameworks)): machine_framework = machine_frameworks[idx2] machine_framework_devices = self._devices[idx][idx2] - col_idx2 = col_idx - col_idx3 = col_idx - num_cols2 = 0 + + col_idx3 = col_idx2 num_cols3 = 0 col_indeces3 = [] for idx3 in range(len(machine_framework_devices)): machine_framework_device = machine_framework_devices[idx3] machine_framework_device_precisions = self._precisions[idx][idx2][idx3] - col_idx1 = col_idx - num_cols1 = 0 + + col_idx4 = col_idx3 + num_cols4 = 0 col_indeces4 = [] for idx4 in range(len(machine_framework_device_precisions)): machine_framework_device_precision = machine_framework_device_precisions[idx4] machine_framework_device_precision_modes = self._execution_modes[idx][idx2][idx3][idx4] + + col_idx5 = col_idx4 + num_cols5 = 0 col_indeces5 = [] for idx5 in range(len(machine_framework_device_precision_modes)): machine_framework_device_precision_mode = machine_framework_device_precision_modes[idx5] machine_framework_device_precision_mode_execparams = ( self._execution_parameters[idx][idx2][idx3][idx4][idx5]) + + col_idx6 = col_idx5 + num_cols6 = len(machine_framework_device_precision_mode_execparams) col_indeces6 = [] - for idx6 in range(len(machine_framework_device_precision_mode_execparams)): + for idx6 in range(num_cols6): execparams = machine_framework_device_precision_mode_execparams[idx6] - self._sheet.write(row_idx, col_idx + idx6, + self._sheet.write(row_idx, col_idx6 + idx6, execparams, self._cell_format_title1) - col_indeces6.append(col_idx + idx6) - k = len(machine_framework_device_precision_mode_execparams) - if k > 1: - self._sheet.merge_range(row_idx - 1, col_idx1, - row_idx - 1, col_idx1 + k - 1, + col_indeces6.append(col_idx6 + idx6) + + if num_cols6 > 1: + self._sheet.merge_range(row_idx - 1, col_idx5, + row_idx - 1, col_idx5 + num_cols6 - 1, machine_framework_device_precision_mode, self._cell_format_title1) - elif k == 1: - self._sheet.write(row_idx - 1, col_idx1, + elif num_cols6 == 1: + self._sheet.write(row_idx - 1, col_idx5, machine_framework_device_precision_mode, self._cell_format_title1) else: msg = 'Incorrect number of execution modes' logging.error(msg) raise ValueError(msg) - col_idx += k - col_idx1 += k - num_cols1 += k - num_cols2 += k - num_cols3 += k - num_cols += k + col_idx5 += num_cols6 + num_cols5 += num_cols6 col_indeces5.append(col_indeces6) - if num_cols1 > 1: - self._sheet.merge_range(row_idx - 2, col_idx2, - row_idx - 2, col_idx2 + num_cols1 - 1, + if num_cols5 > 1: + self._sheet.merge_range(row_idx - 2, col_idx4, + row_idx - 2, col_idx4 + num_cols5 - 1, machine_framework_device_precision, self._cell_format_title1) - elif num_cols1 == 1: - self._sheet.write(row_idx - 2, col_idx2, + elif num_cols5 == 1: + self._sheet.write(row_idx - 2, col_idx4, machine_framework_device_precision, self._cell_format_title1) else: msg = 'Incorrect number of precisions' logging.error(msg) raise ValueError(msg) - col_idx2 += num_cols1 + col_idx4 += num_cols5 + num_cols4 += num_cols5 col_indeces4.append(col_indeces5) - if num_cols2 > 1: + if num_cols4 > 1: self._sheet.merge_range(row_idx - 3, col_idx3, - row_idx - 3, col_idx3 + num_cols2 - 1, + row_idx - 3, col_idx3 + num_cols4 - 1, machine_framework_device, self._cell_format_title1) - elif num_cols2 == 1: + elif num_cols4 == 1: self._sheet.write(row_idx - 3, col_idx3, machine_framework_device, self._cell_format_title1) @@ -314,37 +318,40 @@ def _fill_horizontal_title(self): msg = 'Incorrect number of devices' logging.error(msg) raise ValueError(msg) - col_idx3 += num_cols2 + col_idx3 += num_cols4 + num_cols3 += num_cols4 col_indeces3.append(col_indeces4) if num_cols3 > 1: - self._sheet.merge_range(row_idx - 4, col_idx4, - row_idx - 4, col_idx4 + num_cols3 - 1, + self._sheet.merge_range(row_idx - 4, col_idx2, + row_idx - 4, col_idx2 + num_cols3 - 1, machine_framework, self._cell_format_title1) elif num_cols3 == 1: - self._sheet.write(row_idx - 4, col_idx4 + num_cols3 - 1, + self._sheet.write(row_idx - 4, col_idx2 + num_cols3 - 1, machine_framework, self._cell_format_title1) else: msg = 'Incorrect number of frameworks' logging.error(msg) raise ValueError(msg) - col_idx4 += num_cols3 + col_idx2 += num_cols3 + num_cols2 += num_cols3 col_indeces2.append(col_indeces3) - if num_cols > 1: + if num_cols2 > 1: self._sheet.merge_range(row_idx - 5, rel_col_idx, - row_idx - 5, rel_col_idx + num_cols - 1, + row_idx - 5, rel_col_idx + num_cols2 - 1, machine, self._cell_format_title1) - elif num_cols == 1: + elif num_cols2 == 1: self._sheet.write(row_idx - 5, rel_col_idx, machine, self._cell_format_title1) else: msg = 'Incorrect number of machines' logging.error(msg) raise ValueError(msg) - rel_col_idx += num_cols + rel_col_idx += num_cols2 self._col_indeces.append(col_indeces2) + logging.info(self._col_indeces) logging.info('FINISH: _fill_horizontal_title()') def _add_new_line(self, values, sep=','): @@ -467,14 +474,13 @@ def create_table_rows(self): logging.info('FINISH: create_table_rows()') def _get_records_group(self, task_records, topology_name, train_framework, - blob_size, processed_records_idxs): + processed_records_idxs): records_group = [] for idx in range(len(task_records)): record = task_records[idx] if (idx not in processed_records_idxs and record[self._KEY_TOPOLOGY_NAME] == topology_name - and record[self._KEY_TRAIN_FRAMEWORK] == train_framework - and record[self._KEY_BLOB_SIZE] == blob_size): + and record[self._KEY_TRAIN_FRAMEWORK] == train_framework): processed_records_idxs.append(idx) records_group.append(record) return records_group @@ -496,11 +502,9 @@ def write_test_results(self): for record in task_records: # searching for records for the same topologies topology_name = record[self._KEY_TOPOLOGY_NAME] train_framework = record[self._KEY_TRAIN_FRAMEWORK] - blob_size = record[self._KEY_BLOB_SIZE] records_group = self._get_records_group(task_records, topology_name, train_framework, - blob_size, processed_records_idxs) topology_num_records = len(records_group) if topology_num_records == 0: @@ -514,15 +518,12 @@ def write_test_results(self): row_idx + topology_num_records - 1, 2, train_framework, self._cell_format_title2) - self._sheet.merge_range(row_idx, 3, - row_idx + topology_num_records - 1, - 3, blob_size, - self._cell_format) else: self._sheet.write(row_idx, 1, topology_name, self._cell_format_title2) self._sheet.write(row_idx, 2, train_framework, self._cell_format_title2) - self._sheet.write(row_idx, 3, blob_size, self._cell_format) for topology_record in records_group: + self._sheet.write(row_idx, 3, topology_record[self._KEY_BLOB_SIZE], + self._cell_format_title2) self._sheet.write(row_idx, 4, topology_record[self._KEY_BATCH_SIZE], self._cell_format_title2) for key, value in topology_record[self._KEY_FPS].items(): From 68626188c66cda8c1083eaa8fe1fd0af8006fb15 Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Sun, 14 May 2023 15:54:03 +0300 Subject: [PATCH 55/63] Code cleanings --- src/inference/inference_mxnet.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/inference/inference_mxnet.py b/src/inference/inference_mxnet.py index ce7a93f04..fa31148d5 100644 --- a/src/inference/inference_mxnet.py +++ b/src/inference/inference_mxnet.py @@ -205,8 +205,7 @@ def create_dict_for_modelwrapper(args): return dictionary -def inference_mxnet(net, num_iterations, get_slice, input_name, - k=5, file_labels='image_net_labels.json'): +def inference_mxnet(net, num_iterations, get_slice, input_name): predictions = None time_infer = [] slice_input = None From e3929f4027f354a7a7477584383cf3a3c9a6a442 Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Sun, 14 May 2023 16:14:11 +0300 Subject: [PATCH 56/63] Move common functions to postprocessing_data.py --- src/inference/inference_mxnet.py | 25 ++++--------------------- src/inference/postprocessing_data.py | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/src/inference/inference_mxnet.py b/src/inference/inference_mxnet.py index fa31148d5..aa114820c 100644 --- a/src/inference/inference_mxnet.py +++ b/src/inference/inference_mxnet.py @@ -228,24 +228,6 @@ def inference_mxnet(net, num_iterations, get_slice, input_name): return predictions, time_infer -def process_result(batch_size, inference_time): - inference_time = pp.three_sigma_rule(inference_time) - average_time = pp.calculate_average_time(inference_time) - latency = pp.calculate_latency(inference_time) - fps = pp.calculate_fps(batch_size, latency) - return average_time, latency, fps - - -def result_output(average_time, fps, latency): - log.info('Average time of single pass : {0:.3f}'.format(average_time)) - log.info('FPS : {0:.3f}'.format(fps)) - log.info('Latency : {0:.3f}'.format(latency)) - - -def raw_result_output(average_time, fps, latency): - print('{0:.3f},{1:.3f},{2:.3f}'.format(average_time, fps, latency)) - - def prepare_output(result, output_names, task): if task == 'feedforward': return {} @@ -292,7 +274,8 @@ def main(): io.get_slice_input, args.input_name) log.info('Computing performance metrics') - average_time, latency, fps = process_result(args.batch_size, inference_time) + average_time, latency, fps = pp.calculate_performance_metrics(args.batch_size, + inference_time) if not args.raw_output: if args.number_iter == 1: @@ -306,9 +289,9 @@ def main(): log.warning('Error when printing inference results. {0}'.format(str(ex))) log.info('Performance results') - result_output(average_time, fps, latency) + pp.log_performance_metrics(log, average_time, fps, latency) else: - raw_result_output(average_time, fps, latency) + pp.print_performance_metrics(average_time, fps, latency) except Exception: log.error(traceback.format_exc()) sys.exit(1) diff --git a/src/inference/postprocessing_data.py b/src/inference/postprocessing_data.py index ea4cc23da..46d27eb89 100644 --- a/src/inference/postprocessing_data.py +++ b/src/inference/postprocessing_data.py @@ -36,3 +36,21 @@ def calculate_fps(pictures, time): if time == 0: return -1 return pictures / time + + +def calculate_performance_metrics(batch_size, inference_time): + inference_time = three_sigma_rule(inference_time) + average_time = calculate_average_time(inference_time) + latency = calculate_latency(inference_time) + fps = calculate_fps(batch_size, latency) + return average_time, latency, fps + + +def log_performance_metrics(log, average_time, fps, latency): + log.info('Average time of single pass : {0:.3f}'.format(average_time)) + log.info('FPS : {0:.3f}'.format(fps)) + log.info('Latency : {0:.3f}'.format(latency)) + + +def print_performance_metrics(average_time, fps, latency): + print('{0:.3f},{1:.3f},{2:.3f}'.format(average_time, fps, latency)) From be00d151d4807061b4927def9321fda4c9d6a131 Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Sun, 14 May 2023 16:24:50 +0300 Subject: [PATCH 57/63] Move common functions for async mode to postprocessing_data.py --- src/inference/inference_async_mode.py | 26 +++++--------------------- src/inference/inference_mxnet.py | 8 ++++---- src/inference/postprocessing_data.py | 22 +++++++++++++++++++--- 3 files changed, 28 insertions(+), 28 deletions(-) diff --git a/src/inference/inference_async_mode.py b/src/inference/inference_async_mode.py index b1f67daad..40d935c6a 100644 --- a/src/inference/inference_async_mode.py +++ b/src/inference/inference_async_mode.py @@ -179,22 +179,6 @@ def infer_async(compiled_model, number_iter, num_request, get_slice): return result, inference_time -def process_result(inference_time, batch_size, iteration_count): - average_time = inference_time / iteration_count - fps = pp.calculate_fps(batch_size * iteration_count, inference_time) - - return average_time, fps - - -def result_output(average_time, fps, log): - log.info('Average time of single pass : {0:.3f}'.format(average_time)) - log.info('FPS : {0:.3f}'.format(fps)) - - -def raw_result_output(average_time, fps): - print('{0:.3f},{1:.3f}'.format(average_time, fps)) - - def main(): log.basicConfig( format='[ %(levelname)s ] %(message)s', @@ -225,11 +209,9 @@ def main(): utils.reshape_input(model, args.batch_size) log.info('Prepare input data') - io.prepare_input(model, args.input) log.info('Create executable network') - compiled_model = utils.compile_model(core, model, args.device, args.priority) log.info('Runtime parameters') @@ -244,13 +226,15 @@ def main(): log.info(f'Starting inference ({args.number_iter} iterations) with {args.requests} requests on {args.device}') result, time = infer_async(compiled_model, args.number_iter, args.requests, io.get_slice_input) - average_time, fps = process_result(time, args.batch_size, args.number_iter) + average_time, fps = pp.calculate_performance_metrics_async_mode(time, + args.batch_size, + args.number_iter) if not args.raw_output: io.process_output(result, log) - result_output(average_time, fps, log) + pp.log_performance_metrics_async_mode(log, average_time, fps) else: - raw_result_output(average_time, fps) + pp.print_performance_metrics_async_mode(average_time, fps) del model del compiled_model del core diff --git a/src/inference/inference_mxnet.py b/src/inference/inference_mxnet.py index aa114820c..1ffebe3d8 100644 --- a/src/inference/inference_mxnet.py +++ b/src/inference/inference_mxnet.py @@ -274,8 +274,8 @@ def main(): io.get_slice_input, args.input_name) log.info('Computing performance metrics') - average_time, latency, fps = pp.calculate_performance_metrics(args.batch_size, - inference_time) + average_time, latency, fps = pp.calculate_performance_metrics_sync_mode(args.batch_size, + inference_time) if not args.raw_output: if args.number_iter == 1: @@ -289,9 +289,9 @@ def main(): log.warning('Error when printing inference results. {0}'.format(str(ex))) log.info('Performance results') - pp.log_performance_metrics(log, average_time, fps, latency) + pp.log_performance_metrics_sync_mode(log, average_time, fps, latency) else: - pp.print_performance_metrics(average_time, fps, latency) + pp.print_performance_metrics_sync_mode(average_time, fps, latency) except Exception: log.error(traceback.format_exc()) sys.exit(1) diff --git a/src/inference/postprocessing_data.py b/src/inference/postprocessing_data.py index 46d27eb89..391fc7b2c 100644 --- a/src/inference/postprocessing_data.py +++ b/src/inference/postprocessing_data.py @@ -38,7 +38,7 @@ def calculate_fps(pictures, time): return pictures / time -def calculate_performance_metrics(batch_size, inference_time): +def calculate_performance_metrics_sync_mode(batch_size, inference_time): inference_time = three_sigma_rule(inference_time) average_time = calculate_average_time(inference_time) latency = calculate_latency(inference_time) @@ -46,11 +46,27 @@ def calculate_performance_metrics(batch_size, inference_time): return average_time, latency, fps -def log_performance_metrics(log, average_time, fps, latency): +def log_performance_metrics_sync_mode(log, average_time, fps, latency): log.info('Average time of single pass : {0:.3f}'.format(average_time)) log.info('FPS : {0:.3f}'.format(fps)) log.info('Latency : {0:.3f}'.format(latency)) -def print_performance_metrics(average_time, fps, latency): +def print_performance_metrics_sync_mode(average_time, fps, latency): print('{0:.3f},{1:.3f},{2:.3f}'.format(average_time, fps, latency)) + + +def calculate_performance_metrics_async_mode(inference_time, batch_size, iteration_count): + average_time = inference_time / iteration_count + fps = calculate_fps(batch_size * iteration_count, inference_time) + + return average_time, fps + + +def log_performance_metrics_async_mode(log, average_time, fps): + log.info('Average time of single pass : {0:.3f}'.format(average_time)) + log.info('FPS : {0:.3f}'.format(fps)) + + +def print_performance_metrics_async_mode(average_time, fps): + print('{0:.3f},{1:.3f}'.format(average_time, fps)) From d0aac8ef8acbcc399fae2684494f7366729076bd Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Mon, 15 May 2023 07:12:37 +0300 Subject: [PATCH 58/63] Move computing and printing performance metrics --- src/inference/inference_caffe.py | 45 +++++++++-------------- src/inference/inference_opencv.py | 26 ++----------- src/inference/inference_pytorch.py | 25 ++----------- src/inference/inference_sync_mode.py | 40 ++++++++------------ src/inference/inference_tensorflow.py | 38 +++++++------------ src/inference/inference_tensorflowlite.py | 26 ++----------- src/inference/postprocessing_data.py | 12 +++--- 7 files changed, 66 insertions(+), 146 deletions(-) diff --git a/src/inference/inference_caffe.py b/src/inference/inference_caffe.py index fd20d13a2..e83199fd0 100644 --- a/src/inference/inference_caffe.py +++ b/src/inference/inference_caffe.py @@ -162,24 +162,6 @@ def inference_caffe(net, number_iter, get_slice): return result, time_infer -def process_result(batch_size, inference_time): - inference_time = pp.three_sigma_rule(inference_time) - average_time = pp.calculate_average_time(inference_time) - latency = pp.calculate_latency(inference_time) - fps = pp.calculate_fps(batch_size, latency) - return average_time, latency, fps - - -def result_output(average_time, fps, latency, log): - log.info('Average time of single pass : {0:.3f}'.format(average_time)) - log.info('FPS : {0:.3f}'.format(fps)) - log.info('Latency : {0:.3f}'.format(latency)) - - -def raw_result_output(average_time, fps, latency): - print('{0:.3f},{1:.3f},{2:.3f}'.format(average_time, fps, latency)) - - def create_dict_for_transformer(args): dictionary = { 'channel_swap': args.channel_swap, @@ -206,28 +188,37 @@ def main(): set_device_to_infer(args.device) log.info('The device has been assigned: {0}'.format(args.device)) - log.info('Loading network files:\n\t {0}\n\t {1}'.format(args.model_prototxt, args.model_caffemodel)) - + log.info('Loading network files:\n\t {0}\n\t {1}'.format( + args.model_prototxt, args.model_caffemodel)) net = load_network(args.model_prototxt, args.model_caffemodel) net = network_input_reshape(net, args.batch_size) - input_shapes = get_input_shape(model_wrapper, net) + input_shapes = get_input_shape(model_wrapper, net) for layer in input_shapes: log.info('Shape for input layer {0}: {1}'.format(layer, input_shapes[layer])) - log.info('Prepare input data') + log.info('Prepare input data') io.prepare_input(net, args.input) log.info(f'Starting inference ({args.number_iter} iterations)') - result, inference_time = inference_caffe(net, args.number_iter, io.get_slice_input) - time, latency, fps = process_result(args.batch_size, inference_time) + + log.info('Computing performance metrics') + average_time, latency, fps = pp.calculate_performance_metrics_sync_mode(args.batch_size, + inference_time) if not args.raw_output: - io.process_output(result, log) - result_output(time, fps, latency, log) + if args.number_iter == 1: + try: + log.info('Inference results') + io.process_output(result, log) + except Exception as ex: + log.warning('Error when printing inference results. {0}'.format(str(ex))) + + log.info('Performance results') + pp.log_performance_metrics_sync_mode(log, average_time, fps, latency) else: - raw_result_output(time, fps, latency) + pp.print_performance_metrics_sync_mode(average_time, fps, latency) except Exception: log.error(traceback.format_exc()) sys.exit(1) diff --git a/src/inference/inference_opencv.py b/src/inference/inference_opencv.py index fcb99f828..d41f1abee 100644 --- a/src/inference/inference_opencv.py +++ b/src/inference/inference_opencv.py @@ -202,15 +202,6 @@ def inference_opencv(net, input_name, output_names, number_iter, get_slice): return result, time_infer -def process_result(batch_size, inference_time): - inference_time = pp.three_sigma_rule(inference_time) - average_time = pp.calculate_average_time(inference_time) - latency = pp.calculate_latency(inference_time) - fps = pp.calculate_fps(batch_size, latency) - - return average_time, latency, fps - - def prepare_output(result, output_names, task, args): if task == 'feedforward': return {} @@ -222,16 +213,6 @@ def prepare_output(result, output_names, task, args): raise ValueError(f'Unsupported task {task} to print inference results') -def result_output(average_time, fps, latency, log): - log.info('Average time of single pass : {0:.3f}'.format(average_time)) - log.info('FPS : {0:.3f}'.format(fps)) - log.info('Latency : {0:.3f}'.format(latency)) - - -def raw_result_output(average_time, fps, latency): - print('{0:.3f},{1:.3f},{2:.3f}'.format(average_time, fps, latency)) - - def create_dict_for_transformer(args): dictionary = { 'scalefactor': 1 / args.inv_scale_factor, @@ -292,7 +273,8 @@ def main(): net, args.input_name, args.output_names, args.number_iter, io.get_slice_input) log.info('Computing performance metrics') - average_time, latency, fps = process_result(args.batch_size, inference_time) + average_time, latency, fps = pp.calculate_performance_metrics_sync_mode(args.batch_size, + inference_time) if not args.raw_output: if args.number_iter == 1: @@ -306,9 +288,9 @@ def main(): log.warning('Error when printing inference results. {0}'.format(str(ex))) log.info('Performance results') - result_output(average_time, fps, latency, log) + pp.log_performance_metrics_sync_mode(log, average_time, fps, latency) else: - raw_result_output(average_time, fps, latency) + pp.print_performance_metrics_sync_mode(average_time, fps, latency) except Exception: log.error(traceback.format_exc()) sys.exit(1) diff --git a/src/inference/inference_pytorch.py b/src/inference/inference_pytorch.py index 50c5b036f..54ae6b21f 100644 --- a/src/inference/inference_pytorch.py +++ b/src/inference/inference_pytorch.py @@ -210,24 +210,6 @@ def inference_pytorch(model, num_iterations, get_slice, input_name, inference_mo return predictions, time_infer -def process_result(batch_size, inference_time): - inference_time = pp.three_sigma_rule(inference_time) - average_time = pp.calculate_average_time(inference_time) - latency = pp.calculate_latency(inference_time) - fps = pp.calculate_fps(batch_size, latency) - return average_time, latency, fps - - -def result_output(average_time, fps, latency): - log.info('Average time of single pass : {0:.3f}'.format(average_time)) - log.info('FPS : {0:.3f}'.format(fps)) - log.info('Latency : {0:.3f}'.format(latency)) - - -def raw_result_output(average_time, fps, latency): - print('{0:.3f},{1:.3f},{2:.3f}'.format(average_time, fps, latency)) - - def prepare_output(result, output_names, task): if task == 'feedforward': return {} @@ -271,7 +253,8 @@ def main(): io.get_slice_input, args.input_name, args.inference_mode) log.info('Computing performance metrics') - average_time, latency, fps = process_result(args.batch_size, inference_time) + average_time, latency, fps = pp.calculate_performance_metrics_sync_mode(args.batch_size, + inference_time) if not args.raw_output: if args.number_iter == 1: @@ -285,9 +268,9 @@ def main(): log.warning('Error when printing inference results. {0}'.format(str(ex))) log.info('Performance results') - result_output(average_time, fps, latency) + pp.log_performance_metrics_sync_mode(log, average_time, fps, latency) else: - raw_result_output(average_time, fps, latency) + pp.print_performance_metrics_sync_mode(average_time, fps, latency) except Exception: log.error(traceback.format_exc()) sys.exit(1) diff --git a/src/inference/inference_sync_mode.py b/src/inference/inference_sync_mode.py index b5c051cba..7bcab2051 100644 --- a/src/inference/inference_sync_mode.py +++ b/src/inference/inference_sync_mode.py @@ -156,26 +156,6 @@ def infer_sync(compiled_model, number_iter, get_slice): return result, time_infer -def process_result(inference_time, batch_size, min_infer_time): - correct_time = pp.delete_incorrect_time(inference_time, min_infer_time) - correct_time = pp.three_sigma_rule(correct_time) - average_time = pp.calculate_average_time(correct_time) - latency = pp.calculate_latency(correct_time) - fps = pp.calculate_fps(batch_size, latency) - - return average_time, latency, fps - - -def result_output(average_time, fps, latency, log): - log.info('Average time of single pass : {0:.3f}'.format(average_time)) - log.info('FPS : {0:.3f}'.format(fps)) - log.info('Latency : {0:.3f}'.format(latency)) - - -def raw_result_output(average_time, fps, latency): - print('{0:.3f},{1:.3f},{2:.3f}'.format(average_time, fps, latency)) - - def main(): log.basicConfig( format='[ %(levelname)s ] %(message)s', @@ -215,15 +195,25 @@ def main(): compiled_model = utils.compile_model(core, model, args.device, args.priority) log.info(f'Starting inference ({args.number_iter} iterations) on {args.device}') - result, time = infer_sync(compiled_model, args.number_iter, io.get_slice_input) - average_time, latency, fps = process_result(time, args.batch_size, args.mininfer) + + log.info('Computing performance metrics') + average_time, latency, fps = pp.calculate_performance_metrics_sync_mode(time, + args.batch_size, + args.mininfer) if not args.raw_output: - io.process_output(result, log) - result_output(average_time, fps, latency, log) + if args.number_iter == 1: + try: + log.info('Inference results') + io.process_output(result, log) + except Exception as ex: + log.warning('Error when printing inference results. {0}'.format(str(ex))) + + log.info('Performance results') + pp.log_performance_metrics_sync_mode(log, average_time, fps, latency) else: - raw_result_output(average_time, fps, latency) + pp.print_performance_metrics_sync_mode(average_time, fps, latency) del model del compiled_model del core diff --git a/src/inference/inference_tensorflow.py b/src/inference/inference_tensorflow.py index e496137cb..88d76d024 100644 --- a/src/inference/inference_tensorflow.py +++ b/src/inference/inference_tensorflow.py @@ -246,25 +246,6 @@ def create_config_for_inference(num_intra_threads, num_inter_threads): return config -def process_result(batch_size, inference_time): - inference_time = pp.three_sigma_rule(inference_time) - average_time = pp.calculate_average_time(inference_time) - latency = pp.calculate_latency(inference_time) - fps = pp.calculate_fps(batch_size, latency) - - return average_time, latency, fps - - -def result_output(average_time, fps, latency, log): - log.info('Average time of single pass : {0:.3f}'.format(average_time)) - log.info('FPS : {0:.3f}'.format(fps)) - log.info('Latency : {0:.3f}'.format(latency)) - - -def raw_result_output(average_time, fps, latency): - print('{0:.3f},{1:.3f},{2:.3f}'.format(average_time, fps, latency)) - - def create_dict_for_transformer(args): dictionary = {'channel_swap': args.channel_swap, 'mean': args.mean, 'input_scale': args.input_scale} @@ -301,15 +282,24 @@ def main(): config) log.info('Computing performance metrics') - average_time, latency, fps = process_result(args.batch_size, inference_time) + average_time, latency, fps = pp.calculate_performance_metrics_sync_mode(args.batch_size, + inference_time) if not args.raw_output: if args.number_iter == 1: - result = prepare_output(result, outputs_names, args.task) - io.process_output(result, log) - result_output(average_time, fps, latency, log) + try: + log.info('Converting output tensor to print results') + result = prepare_output(result, outputs_names, args.task) + + log.info('Inference results') + io.process_output(result, log) + except Exception as ex: + log.warning('Error when printing inference results. {0}'.format(str(ex))) + + log.info('Performance results') + pp.log_performance_metrics_sync_mode(log, average_time, fps, latency) else: - raw_result_output(average_time, fps, latency) + pp.print_performance_metrics_sync_mode(average_time, fps, latency) except Exception as ex: log.error(str(ex)) sys.exit(1) diff --git a/src/inference/inference_tensorflowlite.py b/src/inference/inference_tensorflowlite.py index 75acaae33..be8d4076d 100644 --- a/src/inference/inference_tensorflowlite.py +++ b/src/inference/inference_tensorflowlite.py @@ -199,25 +199,6 @@ def reshape_model_input(io_model_wrapper, model, log): model.resize_tensor_input(model_input['index'], shape) -def process_result(batch_size, inference_time): - inference_time = pp.three_sigma_rule(inference_time) - average_time = pp.calculate_average_time(inference_time) - latency = pp.calculate_latency(inference_time) - fps = pp.calculate_fps(batch_size, latency) - - return average_time, latency, fps - - -def result_output(average_time, fps, latency, log): - log.info(f'Average time of single pass : {average_time:.3f}') - log.info(f'FPS : {fps:.3f}') - log.info(f'Latency : {latency:.3f}') - - -def raw_result_output(average_time, fps, latency): - print(f'{average_time:.3f},{fps:.3f},{latency:.3f}') - - def create_dict_for_transformer(args): dictionary = {} for name in args.input_names: @@ -317,7 +298,8 @@ def main(): log.info(f'Starting inference ({args.number_iter} iterations)') result, inference_time = inference_tflite(interpreter, args.number_iter, io.get_slice_input) - time, latency, fps = process_result(args.batch_size, inference_time) + average_time, latency, fps = pp.calculate_performance_metrics_sync_mode(args.batch_size, + inference_time) if not args.raw_output: if args.number_iter == 1: try: @@ -330,9 +312,9 @@ def main(): log.warning('Error when printing inference results. {0}'.format(str(ex))) log.info('Performance results') - result_output(time, fps, latency, log) + pp.log_performance_metrics_sync_mode(log, average_time, fps, latency) else: - raw_result_output(time, fps, latency) + pp.print_performance_metrics_sync_mode(average_time, fps, latency) except Exception: log.error(traceback.format_exc()) sys.exit(1) diff --git a/src/inference/postprocessing_data.py b/src/inference/postprocessing_data.py index 391fc7b2c..1eb809a28 100644 --- a/src/inference/postprocessing_data.py +++ b/src/inference/postprocessing_data.py @@ -38,7 +38,9 @@ def calculate_fps(pictures, time): return pictures / time -def calculate_performance_metrics_sync_mode(batch_size, inference_time): +def calculate_performance_metrics_sync_mode(batch_size, inference_time, + min_infer_time = 0.0): + inference_time = delete_incorrect_time(inference_time, min_infer_time) inference_time = three_sigma_rule(inference_time) average_time = calculate_average_time(inference_time) latency = calculate_latency(inference_time) @@ -47,13 +49,13 @@ def calculate_performance_metrics_sync_mode(batch_size, inference_time): def log_performance_metrics_sync_mode(log, average_time, fps, latency): - log.info('Average time of single pass : {0:.3f}'.format(average_time)) - log.info('FPS : {0:.3f}'.format(fps)) - log.info('Latency : {0:.3f}'.format(latency)) + log.info(f'Average time of single pass : {average_time:.3f}') + log.info(f'FPS : {fps:.3f}') + log.info(f'Latency : {latency:.3f}') def print_performance_metrics_sync_mode(average_time, fps, latency): - print('{0:.3f},{1:.3f},{2:.3f}'.format(average_time, fps, latency)) + print(f'{average_time:.3f},{fps:.3f},{latency:.3f}') def calculate_performance_metrics_async_mode(inference_time, batch_size, iteration_count): From 82b2f5c1564a77760071fbffec6cb3973f5d8946 Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Mon, 15 May 2023 07:26:35 +0300 Subject: [PATCH 59/63] Fix sync mode + fix coding style --- src/inference/inference_sync_mode.py | 6 +++--- src/inference/postprocessing_data.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/inference/inference_sync_mode.py b/src/inference/inference_sync_mode.py index 7bcab2051..a1831840f 100644 --- a/src/inference/inference_sync_mode.py +++ b/src/inference/inference_sync_mode.py @@ -195,11 +195,11 @@ def main(): compiled_model = utils.compile_model(core, model, args.device, args.priority) log.info(f'Starting inference ({args.number_iter} iterations) on {args.device}') - result, time = infer_sync(compiled_model, args.number_iter, io.get_slice_input) + result, inference_time = infer_sync(compiled_model, args.number_iter, io.get_slice_input) log.info('Computing performance metrics') - average_time, latency, fps = pp.calculate_performance_metrics_sync_mode(time, - args.batch_size, + average_time, latency, fps = pp.calculate_performance_metrics_sync_mode(args.batch_size, + inference_time, args.mininfer) if not args.raw_output: diff --git a/src/inference/postprocessing_data.py b/src/inference/postprocessing_data.py index 1eb809a28..856ba453f 100644 --- a/src/inference/postprocessing_data.py +++ b/src/inference/postprocessing_data.py @@ -39,7 +39,7 @@ def calculate_fps(pictures, time): def calculate_performance_metrics_sync_mode(batch_size, inference_time, - min_infer_time = 0.0): + min_infer_time=0.0): inference_time = delete_incorrect_time(inference_time, min_infer_time) inference_time = three_sigma_rule(inference_time) average_time = calculate_average_time(inference_time) From 536f85a887a5d17c50da4c45d29428fb2f60b00c Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Mon, 15 May 2023 07:32:09 +0300 Subject: [PATCH 60/63] Code cleanings --- src/inference/inference_sync_mode.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/inference/inference_sync_mode.py b/src/inference/inference_sync_mode.py index a1831840f..6a8319ffe 100644 --- a/src/inference/inference_sync_mode.py +++ b/src/inference/inference_sync_mode.py @@ -187,11 +187,9 @@ def main(): utils.reshape_input(model, args.batch_size) log.info('Prepare input data') - io.prepare_input(model, args.input) log.info('Create executable network') - compiled_model = utils.compile_model(core, model, args.device, args.priority) log.info(f'Starting inference ({args.number_iter} iterations) on {args.device}') From 4821c1cdf3f9fb5872fa93a7ec7e9bdf109c78ac Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Mon, 15 May 2023 15:05:26 +0300 Subject: [PATCH 61/63] Remove unreported parameters --- src/benchmark/frameworks/config_parser/test_reporter.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/benchmark/frameworks/config_parser/test_reporter.py b/src/benchmark/frameworks/config_parser/test_reporter.py index 72a918ed9..a167021f1 100644 --- a/src/benchmark/frameworks/config_parser/test_reporter.py +++ b/src/benchmark/frameworks/config_parser/test_reporter.py @@ -65,8 +65,6 @@ def prepare_framework_params(self): match_parameter_description['mean'] = 'Mean' match_parameter_description['input_scale'] = 'Scale' match_parameter_description['layout'] = 'Layout' - # duplicate because pytorch launcher does not match common template. To be fixed - match_parameter_description['output_name'] = 'Output name' match_parameter_description['hybridize'] = 'Hybridization flag' match_parameter_description['normalize'] = 'Normalization flag' match_parameter_description['std'] = 'Standard deviation' From 15a0cf1db768fcf51a62efcb07b532d0ff406e7f Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Mon, 15 May 2023 17:45:28 +0300 Subject: [PATCH 62/63] Code cleanings --- src/inference/inference_async_mode.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/inference/inference_async_mode.py b/src/inference/inference_async_mode.py index 40d935c6a..db7b5cbb8 100644 --- a/src/inference/inference_async_mode.py +++ b/src/inference/inference_async_mode.py @@ -226,12 +226,21 @@ def main(): log.info(f'Starting inference ({args.number_iter} iterations) with {args.requests} requests on {args.device}') result, time = infer_async(compiled_model, args.number_iter, args.requests, io.get_slice_input) + + log.info('Computing performance metrics') average_time, fps = pp.calculate_performance_metrics_async_mode(time, args.batch_size, args.number_iter) if not args.raw_output: - io.process_output(result, log) + if args.number_iter == 1: + try: + log.info('Inference results') + io.process_output(result, log) + except Exception as ex: + log.warning('Error when printing inference results. {0}'.format(str(ex))) + + log.info('Performance results') pp.log_performance_metrics_async_mode(log, average_time, fps) else: pp.print_performance_metrics_async_mode(average_time, fps) From 4c0c472710bfb675f9ea74662048605a99ffd64d Mon Sep 17 00:00:00 2001 From: valentina-kustikova Date: Mon, 15 May 2023 17:51:28 +0300 Subject: [PATCH 63/63] Fix report parameters --- src/benchmark/frameworks/config_parser/test_reporter.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/benchmark/frameworks/config_parser/test_reporter.py b/src/benchmark/frameworks/config_parser/test_reporter.py index a167021f1..7f16c7022 100644 --- a/src/benchmark/frameworks/config_parser/test_reporter.py +++ b/src/benchmark/frameworks/config_parser/test_reporter.py @@ -62,15 +62,7 @@ def prepare_framework_params(self): match_parameter_description['num_inter_threads'] = 'Inter threads' match_parameter_description['num_intra_threads'] = 'Intra threads' - match_parameter_description['mean'] = 'Mean' - match_parameter_description['input_scale'] = 'Scale' - match_parameter_description['layout'] = 'Layout' match_parameter_description['hybridize'] = 'Hybridization flag' - match_parameter_description['normalize'] = 'Normalization flag' - match_parameter_description['std'] = 'Standard deviation' - match_parameter_description['channel_swap'] = 'Channel swap' - match_parameter_description['swapRB'] = 'Channel swap' - match_parameter_description['crop'] = 'Crop' match_parameter_description['model_type'] = 'Model type' match_parameter_description['inference_mode'] = 'Inference mode'