From 8240e3e627451da38a9c73aa710b8c08d981e300 Mon Sep 17 00:00:00 2001 From: Matthew Liu <96169489+TheMattBin@users.noreply.github.com> Date: Sat, 14 Sep 2024 12:09:38 +0800 Subject: [PATCH 1/7] added onnx export for deta --- docs/source/exporters/onnx/overview.mdx | 1 + optimum/exporters/onnx/model_configs.py | 11 +++++++++++ optimum/exporters/tasks.py | 5 +++++ 3 files changed, 17 insertions(+) diff --git a/docs/source/exporters/onnx/overview.mdx b/docs/source/exporters/onnx/overview.mdx index 747e1396fb4..136afb8f976 100644 --- a/docs/source/exporters/onnx/overview.mdx +++ b/docs/source/exporters/onnx/overview.mdx @@ -37,6 +37,7 @@ Supported architectures from [🤗 Transformers](https://huggingface.co/docs/tra - Deberta - Deberta-v2 - Deit +- Deta - Detr - DistilBert - Donut-Swin diff --git a/optimum/exporters/onnx/model_configs.py b/optimum/exporters/onnx/model_configs.py index d4b15b2968b..03f943d71e4 100644 --- a/optimum/exporters/onnx/model_configs.py +++ b/optimum/exporters/onnx/model_configs.py @@ -790,6 +790,17 @@ class ResNetOnnxConfig(ViTOnnxConfig): ATOL_FOR_VALIDATION = 1e-3 DEFAULT_ONNX_OPSET = 11 +class DetaOnnxConfig(ViTOnnxConfig): + # OPSET=16 required. Otherwise we get the following error: + # torch.onnx.errors.UnsupportedOperatorError: Exporting the operator 'aten::grid_sampler' to ONNX opset version 12 is not supported. Support for this operator was added in version 16, try exporting with this version. + DEFAULT_ONNX_OPSET = 16 + DUMMY_INPUT_GENERATOR_CLASSES = (RTDetrDummyInputGenerator, ) + ATOL_FOR_VALIDATION = 1e-3 + + @property + def inputs(self) -> Dict[str, Dict[int, str]]: + return {"pixel_values": {0: "batch_size"}} + class DetrOnnxConfig(ViTOnnxConfig): DEFAULT_ONNX_OPSET = 12 diff --git a/optimum/exporters/tasks.py b/optimum/exporters/tasks.py index 97053040879..6361e4c4f21 100644 --- a/optimum/exporters/tasks.py +++ b/optimum/exporters/tasks.py @@ -568,6 +568,11 @@ class TasksManager: "masked-im", onnx="DeiTOnnxConfig", ), + "deta": supported_tasks_mapping( + "feature-extraction", + "object-detection", + onnx="DetaOnnxConfig" + ), "detr": supported_tasks_mapping( "feature-extraction", "object-detection", From 61972d0b5490925527e7aba5d5d506f82f62b9a9 Mon Sep 17 00:00:00 2001 From: Matthew Liu <96169489+TheMattBin@users.noreply.github.com> Date: Sat, 14 Sep 2024 12:36:52 +0800 Subject: [PATCH 2/7] added test for deta onnx --- tests/exporters/exporters_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/exporters/exporters_utils.py b/tests/exporters/exporters_utils.py index a55c7a124df..9e6b15f7c6e 100644 --- a/tests/exporters/exporters_utils.py +++ b/tests/exporters/exporters_utils.py @@ -220,6 +220,7 @@ "deberta": "hf-internal-testing/tiny-random-DebertaModel", # Not using microsoft/deberta-base because it takes too much time for testing. "deberta-v2": "hf-internal-testing/tiny-random-DebertaV2Model", # Not using microsoft/deberta-v2-xlarge because it takes too much time for testing. "deit": "facebook/deit-small-patch16-224", + "deta": "jozhang97/deta-resnet-50", "detr": "hf-internal-testing/tiny-random-detr", # Not using facebook/detr-resnet-50 because it takes too much time for testing. "distilbert": "distilbert-base-cased", "electra": "google/electra-base-generator", From 958030cf79767ee0669b4c4eaf9026b8d9f9a9dd Mon Sep 17 00:00:00 2001 From: Matthew Liu <96169489+TheMattBin@users.noreply.github.com> Date: Sat, 14 Sep 2024 12:38:00 +0800 Subject: [PATCH 3/7] Update normalized_config.py --- optimum/utils/normalized_config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/optimum/utils/normalized_config.py b/optimum/utils/normalized_config.py index 81207b76496..530d7a5295c 100644 --- a/optimum/utils/normalized_config.py +++ b/optimum/utils/normalized_config.py @@ -203,6 +203,7 @@ class NormalizedConfigManager: 'convnextv2', 'data2vec-text', 'data2vec-vision', + 'deta', 'detr', 'flaubert', 'groupvit', From 3a5121cbde8db5f79619eef3eeb998a43f4ac1dc Mon Sep 17 00:00:00 2001 From: Matthew Liu <96169489+TheMattBin@users.noreply.github.com> Date: Sat, 14 Sep 2024 14:19:17 +0800 Subject: [PATCH 4/7] Update model_configs.py --- optimum/exporters/onnx/model_configs.py | 34 +++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/optimum/exporters/onnx/model_configs.py b/optimum/exporters/onnx/model_configs.py index 03f943d71e4..c9fabccdb9c 100644 --- a/optimum/exporters/onnx/model_configs.py +++ b/optimum/exporters/onnx/model_configs.py @@ -790,6 +790,40 @@ class ResNetOnnxConfig(ViTOnnxConfig): ATOL_FOR_VALIDATION = 1e-3 DEFAULT_ONNX_OPSET = 11 +class DetaDummyInputGenerator(DummyVisionInputGenerator): + def __init__( + self, + task: str, + normalized_config: NormalizedVisionConfig, + batch_size: int = DEFAULT_DUMMY_SHAPES["batch_size"], + num_channels: int = DEFAULT_DUMMY_SHAPES["num_channels"], + width: int = DEFAULT_DUMMY_SHAPES["width"], + height: int = DEFAULT_DUMMY_SHAPES["height"], + **kwargs, + ): + super().__init__( + task=task, + normalized_config=normalized_config, + batch_size=batch_size, + num_channels=num_channels, + width=width, + height=height, + **kwargs, + ) + + from transformers.onnx.utils import get_preprocessor + + preprocessor = get_preprocessor(normalized_config._name_or_path) + if preprocessor is not None and hasattr(preprocessor, "crop_size"): + self.height = preprocessor.crop_size.get("height", self.height) + self.width = preprocessor.crop_size.get("width", self.width) + + def generate(self, input_name: str, framework: str = "pt", int_dtype: str = "int64", float_dtype: str = "fp32"): + input_ = super().generate( + input_name=input_name, framework=framework, int_dtype=int_dtype, float_dtype=float_dtype + ) + return input_ + class DetaOnnxConfig(ViTOnnxConfig): # OPSET=16 required. Otherwise we get the following error: # torch.onnx.errors.UnsupportedOperatorError: Exporting the operator 'aten::grid_sampler' to ONNX opset version 12 is not supported. Support for this operator was added in version 16, try exporting with this version. From 8673a060a4c80379fd55dcf1892b8ea18091adca Mon Sep 17 00:00:00 2001 From: Matthew Liu <96169489+TheMattBin@users.noreply.github.com> Date: Sat, 14 Sep 2024 14:48:21 +0800 Subject: [PATCH 5/7] Update model_configs.py --- optimum/exporters/onnx/model_configs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optimum/exporters/onnx/model_configs.py b/optimum/exporters/onnx/model_configs.py index c9fabccdb9c..2857e080551 100644 --- a/optimum/exporters/onnx/model_configs.py +++ b/optimum/exporters/onnx/model_configs.py @@ -828,7 +828,7 @@ class DetaOnnxConfig(ViTOnnxConfig): # OPSET=16 required. Otherwise we get the following error: # torch.onnx.errors.UnsupportedOperatorError: Exporting the operator 'aten::grid_sampler' to ONNX opset version 12 is not supported. Support for this operator was added in version 16, try exporting with this version. DEFAULT_ONNX_OPSET = 16 - DUMMY_INPUT_GENERATOR_CLASSES = (RTDetrDummyInputGenerator, ) + DUMMY_INPUT_GENERATOR_CLASSES = (DetaDummyInputGenerator, ) ATOL_FOR_VALIDATION = 1e-3 @property From 436a6986800113dbed8bc7f6528f75239aad316c Mon Sep 17 00:00:00 2001 From: Matthew Liu <96169489+TheMattBin@users.noreply.github.com> Date: Sat, 14 Sep 2024 15:32:58 +0800 Subject: [PATCH 6/7] Update model_configs.py --- optimum/exporters/onnx/model_configs.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/optimum/exporters/onnx/model_configs.py b/optimum/exporters/onnx/model_configs.py index 2857e080551..aa2517f5f3d 100644 --- a/optimum/exporters/onnx/model_configs.py +++ b/optimum/exporters/onnx/model_configs.py @@ -833,7 +833,10 @@ class DetaOnnxConfig(ViTOnnxConfig): @property def inputs(self) -> Dict[str, Dict[int, str]]: - return {"pixel_values": {0: "batch_size"}} + return { + "pixel_values": {0: "batch_size"}, + 'pixel_mask': {0: "batch_size"} + } class DetrOnnxConfig(ViTOnnxConfig): From 3336fbfe403e2af2e91aeac5ee25501a5cc6007d Mon Sep 17 00:00:00 2001 From: Matthew Liu <96169489+TheMattBin@users.noreply.github.com> Date: Sun, 22 Sep 2024 15:08:15 +0800 Subject: [PATCH 7/7] changed dummy width and height that inputs are inline after processor --- optimum/exporters/onnx/model_configs.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/optimum/exporters/onnx/model_configs.py b/optimum/exporters/onnx/model_configs.py index aa2517f5f3d..37d192b8a8e 100644 --- a/optimum/exporters/onnx/model_configs.py +++ b/optimum/exporters/onnx/model_configs.py @@ -811,12 +811,14 @@ def __init__( **kwargs, ) - from transformers.onnx.utils import get_preprocessor - - preprocessor = get_preprocessor(normalized_config._name_or_path) - if preprocessor is not None and hasattr(preprocessor, "crop_size"): - self.height = preprocessor.crop_size.get("height", self.height) - self.width = preprocessor.crop_size.get("width", self.width) + # from transformers.onnx.utils import get_preprocessor + + # preprocessor = get_preprocessor(normalized_config._name_or_path) + # if preprocessor is not None and hasattr(preprocessor, "crop_size"): + # self.height = preprocessor.crop_size.get("height", self.height) + # self.width = preprocessor.crop_size.get("width", self.width) + self.width = 1066 + self.height = 800 def generate(self, input_name: str, framework: str = "pt", int_dtype: str = "int64", float_dtype: str = "fp32"): input_ = super().generate(