Local & Docker mode loading via config using Triton API (#744)

* Intial changes. Unit tests passing

* Fix for model-repository error

* Second attempt at fixing model_repo issue

* Checking for existence of model repo (because analyze doesn't have to provide one)

* Fixes for C_API mode

* Fixing codeql issue

* Fixing representation to be unique

* Fixing PA printing to keep config variant

* Fixing metrics interval

* Correctly using json

* Fixing codeql

* Fixes based on review

* Fixing typo

* Making variant_name optional

* remove csv

* Fixing bls bug

model_analyzer/config/generate/base_model_config_generator.py

@@ -70,6 +70,7 @@ def __init__(
         self._base_model = model
         self._base_model_name = model.model_name()
         self._remote_mode = config.triton_launch_mode == "remote"
+        self._c_api_mode = config.triton_launch_mode == "c_api"
         self._cpu_only = model.cpu_only()
         self._default_only = default_only
         self._early_exit_enable = early_exit_enable
@@ -154,7 +155,7 @@ def _get_last_results_max_throughput(self) -> Optional[float]:
 
     def _make_remote_model_config_variant(self) -> ModelConfigVariant:
         if not self._config.reload_model_disable:
-            self._client.load_model(self._base_model_name)
+            self._client.load_model(model_name=self._base_model_name)
         model_config = ModelConfig.create_from_triton_api(
             self._client, self._base_model_name, self._config.client_max_retries
         )
@@ -171,13 +172,15 @@ def _make_direct_mode_model_config_variant(
             param_combo=param_combo,
             model=self._base_model,
             model_variant_name_manager=self._model_variant_name_manager,
+            c_api_mode=self._c_api_mode,
         )
 
     @staticmethod
     def make_model_config_variant(
         param_combo: dict,
         model: ModelProfileSpec,
         model_variant_name_manager: ModelVariantNameManager,
+        c_api_mode: bool,
     ) -> ModelConfigVariant:
         """
         Loads the base model config from the model repository, and then applies the
@@ -189,6 +192,7 @@ def make_model_config_variant(
             dict of key:value pairs to apply to the model config
         model: ModelProfileSpec
         model_variant_name_manager: ModelVariantNameManager
+        c_api_mode: Set to true if mode is c_api
         """
         logger_str: List[str] = []
         model_name = model.model_name()
@@ -211,7 +215,7 @@ def make_model_config_variant(
             logger.info(str)
         logger.info("")
 
-        model_config_dict["name"] = variant_name
+        model_config_dict["name"] = variant_name if c_api_mode else model_name
         model_config = ModelConfig.create_from_dictionary(model_config_dict)
         model_config.set_cpu_only(model.cpu_only())
 
@@ -222,6 +226,7 @@ def make_ensemble_model_config_variant(
         model: ModelProfileSpec,
         ensemble_composing_model_config_variants: List[ModelConfigVariant],
         model_variant_name_manager: ModelVariantNameManager,
+        c_api_mode: bool,
         param_combo: Dict = {},
     ) -> ModelConfigVariant:
         """
@@ -235,6 +240,7 @@ def make_ensemble_model_config_variant(
         ensemble_composing_model_config_variants: List of ModelConfigVariants
             The list of composing model ModelConfigs
         model_variant_name_manager: ModelVariantNameManager
+        c_api_mode: Set to true if mode is c_api
 
         """
         logger_str: List[str] = []
@@ -243,12 +249,8 @@ def make_ensemble_model_config_variant(
             model, param_combo, logger_str
         )
 
-        ensemble_config_dicts = [
-            composing_model_config_variant.model_config.to_dict()
-            for composing_model_config_variant in ensemble_composing_model_config_variants
-        ]
         ensemble_key = ModelVariantNameManager.make_ensemble_composing_model_key(
-            ensemble_config_dicts
+            ensemble_composing_model_config_variants
         )
 
         (
@@ -265,7 +267,7 @@ def make_ensemble_model_config_variant(
         for str in logger_str:
             logger.info(str)
 
-        model_config_dict["name"] = variant_name
+        model_config_dict["name"] = variant_name if c_api_mode else model_name
         model_config = ModelConfig.create_from_dictionary(model_config_dict)
 
         return ModelConfigVariant(model_config, variant_name)

model_analyzer/config/generate/model_run_config_generator.py

@@ -100,7 +100,7 @@ def get_configs(self) -> Generator[ModelRunConfig, None, None]:
         for model_config_variant in self._mcg.get_configs():
             self._pacg = PerfAnalyzerConfigGenerator(
                 self._config,
-                model_config_variant.variant_name,
+                model_config_variant.model_config.get_field("name"),
                 self._model_pa_flags,
                 self._model_parameters,
                 self._pacg_early_exit_enable,

model_analyzer/config/generate/model_variant_name_manager.py

@@ -18,6 +18,7 @@
 from typing import Dict, List, Tuple
 
 from model_analyzer.constants import DEFAULT_CONFIG_PARAMS
+from model_analyzer.triton.model.model_config_variant import ModelConfigVariant
 
 
 class ModelVariantNameManager:
@@ -45,12 +46,9 @@ def from_dict(
 
     @staticmethod
     def make_ensemble_composing_model_key(
-        ensemble_config_dicts: List[Dict],
+        ensemble_model_config_variants: List[ModelConfigVariant],
     ) -> Dict[str, str]:
-        ensemble_names = [
-            ensemble_config_dict["name"]
-            for ensemble_config_dict in ensemble_config_dicts
-        ]
+        ensemble_names = [emcv.variant_name for emcv in ensemble_model_config_variants]
         ensemble_key = ",".join(ensemble_names)
 
         return {"key": ensemble_key}

model_analyzer/config/generate/quick_run_config_generator.py

@@ -91,6 +91,8 @@ def __init__(
 
         self._triton_env = BruteRunConfigGenerator.determine_triton_server_env(models)
 
+        self._c_api_mode = config.triton_launch_mode == "c_api"
+
         # This tracks measured results for all coordinates
         self._coordinate_data = CoordinateData()
 
@@ -425,6 +427,7 @@ def _get_next_ensemble_model_config_variant(
                 ensemble_composing_model_config_variants=composing_config_variants,
                 model_variant_name_manager=self._model_variant_name_manager,
                 param_combo=param_combo,
+                c_api_mode=self._c_api_mode,
             )
         )
 
@@ -471,6 +474,7 @@ def _get_next_model_config_variant(
             param_combo=param_combo,
             model=model,
             model_variant_name_manager=self._model_variant_name_manager,
+            c_api_mode=self._c_api_mode,
         )
 
         return model_config_variant
@@ -483,7 +487,7 @@ def _create_next_model_run_config(
         composing_model_config_variants: List[ModelConfigVariant],
     ) -> ModelRunConfig:
         perf_analyzer_config = self._get_next_perf_analyzer_config(
-            model_config_variant.variant_name, model, model_index
+            model.model_name(), model, model_index
         )
         model_run_config = ModelRunConfig(
             model.model_name(), model_config_variant, perf_analyzer_config
@@ -624,6 +628,7 @@ def _create_default_ensemble_model_run_config(
             model=model,
             ensemble_composing_model_config_variants=default_composing_model_config_variants,
             model_variant_name_manager=self._model_variant_name_manager,
+            c_api_mode=self._c_api_mode,
         )
 
         default_perf_analyzer_config = self._create_default_perf_analyzer_config(
@@ -652,6 +657,7 @@ def _create_default_composing_model_config_variants(
                     param_combo={},
                     model=composing_model,
                     model_variant_name_manager=self._model_variant_name_manager,
+                    c_api_mode=self._c_api_mode,
                 )
             )
 
@@ -665,6 +671,7 @@ def _create_default_model_run_config(
                 param_combo={},
                 model=model,
                 model_variant_name_manager=self._model_variant_name_manager,
+                c_api_mode=self._c_api_mode,
             )
         )
 

model_analyzer/config/run/model_run_config.py

@@ -145,7 +145,8 @@ def representation(self) -> str:
         Returns a representation string for the ModelRunConfig that can be used
         as a key to uniquely identify it
         """
-        repr = self.perf_config().representation()
+        repr = self.model_variant_name()
+        repr += " " + self.perf_config().representation()
 
         if self._composing_config_variants:
             repr += " " + (",").join(self.get_composing_config_names())  # type: ignore

model_analyzer/perf_analyzer/perf_config.py

@@ -207,7 +207,7 @@ def update_config_from_profile_config(self, model_name, profile_config):
 
             params.update({"protocol": profile_config.client_protocol, "url": url})
 
-            metrics_interval = (
+            metrics_interval = int(
                 profile_config.monitoring_interval * SECONDS_TO_MILLISECONDS_MULTIPLIER
             )
             params.update(

model_analyzer/record/metrics_manager.py

@@ -387,14 +387,9 @@ def _load_model_variants(self, run_config):
 
             # Composing configs for BLS models are not automatically loaded by the top-level model
             if mrc.is_bls_model():
-                for composing_config_variant in mrc.composing_configs():
-                    original_composing_config = (
-                        BaseModelConfigGenerator.create_original_config_from_variant(
-                            composing_config_variant
-                        )
-                    )
+                for composing_config_variant in mrc.composing_config_variants():
                     if not self._load_model_variant(
-                        variant_config=original_composing_config
+                        variant_config=composing_config_variant
                     ):
                         return False
 
@@ -423,13 +418,23 @@ def _do_load_model_variant(self, variant_config):
             log_file=self._server.log_file(),
         )
 
+        model_name = variant_config.model_config.get_field("name")
         variant_name = variant_config.variant_name
-        if self._client.load_model(model_name=variant_name) == -1:
+        config_str = variant_config.model_config.get_config_str()
+        if (
+            self._client.load_model(
+                model_name=model_name,
+                variant_name=variant_name,
+                config_str=config_str,
+            )
+            == -1
+        ):
             return False
 
         if (
             self._client.wait_for_model_ready(
-                model_name=variant_name, num_retries=self._config.client_max_retries
+                model_name=variant_config.model_config.get_field("name"),
+                num_retries=self._config.client_max_retries,
             )
             == -1
         ):
@@ -710,16 +715,15 @@ def _get_triton_metrics_gpus(self):
         return triton_gpus
 
     def _print_run_config_info(self, run_config):
-        for perf_config in [
-            mrc.perf_config() for mrc in run_config.model_run_configs()
-        ]:
+        for model_run_config in run_config.model_run_configs():
+            perf_config = model_run_config.perf_config()
             if perf_config["request-rate-range"]:
                 logger.info(
-                    f"Profiling {perf_config['model-name']}: client batch size={perf_config['batch-size']}, request-rate-range={perf_config['request-rate-range']}"
+                    f"Profiling {model_run_config.model_variant_name()}: client batch size={perf_config['batch-size']}, request-rate-range={perf_config['request-rate-range']}"
                 )
             else:
                 logger.info(
-                    f"Profiling {perf_config['model-name']}: client batch size={perf_config['batch-size']}, concurrency={perf_config['concurrency-range']}"
+                    f"Profiling {model_run_config.model_variant_name()}: client batch size={perf_config['batch-size']}, concurrency={perf_config['concurrency-range']}"
                 )
 
         # Vertical spacing when running multiple models at a time

model_analyzer/reports/report_manager.py

@@ -642,19 +642,23 @@ def _create_model_summary_sentence(self, run_config: RunConfig) -> str:
             summary_sentence = (
                 summary_sentence
                 + "<LI> "
-                + self._create_summary_config_info(model_run_config.model_config())
+                + self._create_summary_config_info(
+                    model_run_config.model_config_variant()
+                )
                 + " </LI>"
             )
 
         return summary_sentence
 
     def _create_composing_model_summary_sentence(self, run_config: RunConfig) -> str:
         summary_sentence = ""
-        for composing_config in run_config.model_run_configs()[0].composing_configs():
+        for composing_config_variant in run_config.model_run_configs()[
+            0
+        ].composing_config_variants():
             summary_sentence = (
                 summary_sentence
                 + "<LI> "
-                + self._create_summary_config_info(composing_config)
+                + self._create_summary_config_info(composing_config_variant)
                 + " </LI>"
             )
 
@@ -1082,8 +1086,10 @@ def _create_non_gpu_metric_string(self, run_config_measurement, non_gpu_metric):
         else:
             return f"{non_gpu_metrics[0].value()}"
 
-    def _create_summary_config_info(self, model_config):
-        config_info = f"<strong>{model_config.get_field('name')}</strong>: "
+    def _create_summary_config_info(self, model_config_variant):
+        model_config = model_config_variant.model_config
+
+        config_info = f"<strong>{model_config_variant.variant_name}</strong>: "
         config_info = (
             config_info + f"{self._create_instance_group_phrase(model_config)} with a "
         )
@@ -1235,11 +1241,11 @@ def _build_detailed_info(self, model_config_name):
         if run_config.is_ensemble_model():
             sentence = f"<strong>{model_config_name}</strong> is comprised of the following composing models:"
 
-            for composing_config in run_config.composing_configs():
+            for composing_config_variant in run_config.composing_config_variants():
                 sentence = (
                     sentence
                     + "<LI> "
-                    + self._create_summary_config_info(composing_config)
+                    + self._create_summary_config_info(composing_config_variant)
                     + " </LI>"
                 )
 
@@ -1250,11 +1256,11 @@ def _build_detailed_info(self, model_config_name):
         elif run_config.is_bls_model():
             sentence = f"<strong>{model_config_name}</strong> is comprised of the following composing models:"
 
-            for composing_config in run_config.composing_configs():
+            for composing_config_variant in run_config.composing_config_variants():
                 sentence = (
                     sentence
                     + "<LI> "
-                    + self._create_summary_config_info(composing_config)
+                    + self._create_summary_config_info(composing_config_variant)
                     + " </LI>"
                 )
 

model_analyzer/result/result_table_manager.py

@@ -388,10 +388,11 @@ def _tabulate_measurements(self, run_config_result):
 
     def _tabulate_measurements_setup(self, run_config_result):
         if run_config_result.run_config().is_ensemble_model():
+            model_config_variants = (
+                run_config_result.run_config().composing_config_variants()
+            )
             model_configs = run_config_result.run_config().composing_configs()
-            composing_config_names = [
-                model_config.get_field("name") for model_config in model_configs
-            ]
+            composing_config_names = [mcv.variant_name for mcv in model_config_variants]
         else:
             model_configs = [
                 model_run_configs.model_config()

model_analyzer/triton/client/client.py

@@ -74,7 +74,7 @@ def wait_for_server_ready(
             "Could not determine server readiness. " "Number of retries exceeded."
         )
 
-    def load_model(self, model_name):
+    def load_model(self, model_name, variant_name="", config_str=None):
         """
         Request the inference server to load
         a particular model in explicit model
@@ -83,20 +83,28 @@ def load_model(self, model_name):
         Parameters
         ----------
         model_name : str
-            name of the model to load from repository
+            Name of the model
+
+        variant_name: str
+            Name of the model variant
+
+        config_str: str
+            Optional config string used to load the model
 
         Returns
         ------
         int or None
             Returns -1 if the failed.
         """
 
+        variant_name = variant_name if variant_name else model_name
+
         try:
-            self._client.load_model(model_name)
-            logger.debug(f"Model {model_name} loaded")
+            self._client.load_model(model_name, config=config_str)
+            logger.debug(f"Model {variant_name} loaded")
             return None
         except Exception as e:
-            logger.info(f"Model {model_name} load failed: {e}")
+            logger.info(f"Model {variant_name} load failed: {e}")
             return -1
 
     def unload_model(self, model_name):