Update RVC2 and RVC4 benchmark scripts to work with the dai Benchmark Nodes

.github/workflows/modelconverter_test.yaml

@@ -51,7 +51,7 @@ jobs:
         cache: pip
 
     - name: Install dependencies
-      run: pip install -e .[dev]
+      run: pip install -e .[dev] --extra-index-url https://artifacts.luxonis.com/artifactory/luxonis-python-release-local/
 
     - name: Authenticate to Google Cloud
       id: google-auth

.github/workflows/unittests.yaml

@@ -25,7 +25,7 @@ jobs:
         cache: pip
 
     - name: Install package
-      run: python -m pip install -e .[dev]
+      run: python -m pip install -e .[dev] --extra-index-url https://artifacts.luxonis.com/artifactory/luxonis-python-release-local/
 
     - name: Run Unit Tests
       env:

README.md

@@ -52,6 +52,13 @@ pip install modelconv
 
 Run `modelconverter --help` to see the available commands and options.
 
+> \[!NOTE\]
+> To use the [benchmarking feature](#benchmarking), the `depthai v3` package must be installed. While the `depthai v3` is not yet released on PyPI, you can install it with the following command:
+>
+> ```bash
+> pip install -r requirements-bench.txt --extra-index-url https://artifacts.luxonis.com/artifactory/luxonis-python-release-local/
+> ```
+
 ## Configuration
 
 There are two main ways to execute configure the conversion process:
@@ -437,3 +444,6 @@ modelconverter benchmark rvc3 --model-path <path_to_model.xml>
 
 The command prints a table with the benchmark results to the console and
 optionally saves the results to a `.csv` file.
+
+> \[!NOTE\]
+> For **RVC2** and **RVC4**: The `--model-path` can be a path to a local .blob file, a NN Archive file (.tar.xz), or a name of a model slug from [Luxonis HubAI](https://hub.luxonis.com/ai). To access models from different teams in Luxonis HubAI, remember to update the HUBAI_API_KEY environment variable respectively.

modelconverter/__main__.py

@@ -175,10 +175,12 @@ def benchmark(
 
     **RVC2**
 
-    - `--repetitions`: The number of repetitions to perform. Default: `1`
+    - `--repetitions`: The number of repetitions to perform. Default: `10`
 
     - `--num-threads`: The number of threads to use for inference. Default: `2`
 
+    - `--num-messages`: The number of messages to measure for each report. Default: `50`
+
     ---
 
     **RVC3**
@@ -191,8 +193,18 @@ def benchmark(
 
     - `--profile`: The SNPE profile to use for inference. Default: `"default"`
 
+    - `--runtime`: The SNPE runtime to use for inference (dsp or cpu). Default: `"dsp"`
+
     - `--num-images`: The number of images to use for inference. Default: `1000`
 
+    - `--dai-benchmark`: Whether to run the benchmark using the DAI V3. If False the SNPE tools are used. Default: `True`
+
+    - `--repetitions`: The number of repetitions to perform (dai-benchmark only). Default: `10`
+
+    - `--num-threads`: The number of threads to use for inference (dai-benchmark only). Default: `1`
+
+    - `--num-messages`: The number of messages to measure for each report (dai-benchmark only). Default: `50`
+
     ---
     """
 
@@ -203,6 +215,13 @@ def benchmark(
             key = key[2:].replace("-", "_")
         else:
             raise typer.BadParameter(f"Unknown argument: {key}")
+        if key == "dai_benchmark":
+            value = value.capitalize()
+            if value not in ["True", "False"]:
+                raise typer.BadParameter(
+                    "dai_benchmark must be either True or False"
+                )
+            value = value == "True"
         kwargs[key] = value
     Benchmark = get_benchmark(target)
     benchmark = Benchmark(str(model_path))

modelconverter/packages/base_benchmark.py

@@ -1,3 +1,4 @@
+import re
 from abc import ABC, abstractmethod
 from collections import namedtuple
 from logging import getLogger
@@ -7,7 +8,7 @@
 import pandas as pd
 from typing_extensions import TypeAlias
 
-from modelconverter.utils import resolve_path
+from modelconverter.utils import is_hubai_available, resolve_path
 
 logger = getLogger(__name__)
 
@@ -23,14 +24,36 @@
 
 
 class Benchmark(ABC):
+    VALID_EXTENSIONS = (".tar.xz", ".blob", ".dlc")
+    HUB_MODEL_PATTERN = re.compile(r"^(?:([^/]+)/)?([^:]+):(.+)$")
+
     def __init__(
         self,
         model_path: str,
         dataset_path: Optional[Path] = None,
     ):
-        self.model_path = resolve_path(model_path, Path.cwd())
+        if any(model_path.endswith(ext) for ext in self.VALID_EXTENSIONS):
+            self.model_path = resolve_path(model_path, Path.cwd())
+            self.model_name = self.model_path.stem
+        else:
+            hub_match = self.HUB_MODEL_PATTERN.match(model_path)
+            if not hub_match:
+                raise ValueError(
+                    "Invalid 'model-path' format. Expected either:\n"
+                    "- Model file path: path/to/model.blob, path/to/model.dlc or path/to/model.tar.xz\n"
+                    "- HubAI model slug: [team_name/]model_name:variant"
+                )
+            team_name, model_name, model_variant = hub_match.groups()
+            if is_hubai_available(model_name, model_variant):
+                self.model_path = model_path
+                self.model_name = model_name
+            else:
+                raise ValueError(
+                    f"Model {team_name+'/' if team_name else ''}{model_name}:{model_variant} not found in HubAI."
+                )
+
         self.dataset_path = dataset_path
-        self.model_name = self.model_path.stem
+
         self.header = [
             *self.default_configuration.keys(),
             "fps",
@@ -64,7 +87,13 @@ def print_results(
             title=f"Benchmark Results for [yellow]{self.model_name}",
             box=box.ROUNDED,
         )
-        for field in self.header:
+
+        updated_header = [
+            *results[0][0].keys(),
+            "fps",
+            "latency (ms)",
+        ]
+        for field in updated_header:
             table.add_column(f"[cyan]{field}")
         for configuration, result in results:
             fps_color = (
@@ -74,17 +103,22 @@ def print_results(
                 if result.fps < 5
                 else "green"
             )
-            latency_color = (
-                "yellow"
-                if 50 < result.latency < 100
-                else "red"
-                if result.latency > 100
-                else "green"
-            )
+            if isinstance(result.latency, str):
+                latency_color = "orange3"
+            else:
+                latency_color = (
+                    "yellow"
+                    if 50 < result.latency < 100
+                    else "red"
+                    if result.latency > 100
+                    else "green"
+                )
             table.add_row(
                 *map(lambda x: f"[magenta]{x}", configuration.values()),
                 f"[{fps_color}]{result.fps:.2f}",
-                f"[{latency_color}]{result.latency:.5f}",
+                f"[{latency_color}]{result.latency}"
+                if isinstance(result.latency, str)
+                else f"[{latency_color}]{result.latency:.5f}",
             )
         console = Console()
         console.print(table)