Add remote mode config sweep to brute search (#751)

* Brute run in remote mode changes

* Fixing L0 test

* Fixes based on review comments

* Fix L0

model_analyzer/config/generate/brute_run_config_generator.py

@@ -102,11 +102,7 @@ def _get_next_config(self) -> Generator[RunConfig, None, None]:
         if not self._skip_default_config:
             yield from self._generate_subset(0, default_only=True)
 
-        if self._should_generate_non_default_configs():
-            yield from self._generate_subset(0, default_only=False)
-
-    def _should_generate_non_default_configs(self) -> bool:
-        return self._config.triton_launch_mode != "remote"
+        yield from self._generate_subset(0, default_only=False)
 
     def _generate_subset(
         self, index: int, default_only: bool

model_analyzer/config/generate/manual_model_config_generator.py

@@ -130,23 +130,7 @@ def _get_next_model_config_variant(self) -> ModelConfigVariant:
 
     def _generate_model_config_variants(self) -> List[List[ModelConfigVariant]]:
         """Generate all model config combinations"""
-        if self._remote_mode:
-            configs = self._generate_remote_mode_model_config_variants()
-        else:
-            configs = self._generate_direct_modes_model_config_variants()
-
-        return configs
-
-    def _generate_remote_mode_model_config_variants(
-        self,
-    ) -> List[List[ModelConfigVariant]]:
-        """Generate model configs for remote mode"""
-        return [[self._make_remote_model_config_variant()]]
 
-    def _generate_direct_modes_model_config_variants(
-        self,
-    ) -> List[List[ModelConfigVariant]]:
-        """Generate model configs for direct (non-remote) modes"""
         model_config_variants = []
         for param_combo in self._non_max_batch_size_param_combos:
             configs_with_max_batch_size = []
@@ -171,9 +155,6 @@ def _determine_max_batch_sizes_and_param_combos(self) -> None:
         """
         Determine self._max_batch_sizes and self._non_max_batch_size_param_combos
         """
-        if self._remote_mode:
-            return
-
         if self._default_only:
             self._non_max_batch_size_param_combos = [DEFAULT_CONFIG_PARAMS]
         else:

model_analyzer/config/generate/model_config_generator_factory.py

@@ -67,11 +67,10 @@ def create_model_config_generator(
         A generator that implements ConfigGeneratorInterface and creates ModelConfigs
         """
 
-        remote_mode = config.triton_launch_mode == "remote"
         search_disabled = config.run_config_search_disable
         model_config_params = model.model_config_parameters()
 
-        if remote_mode or search_disabled or model_config_params:
+        if search_disabled or model_config_params:
             return ManualModelConfigGenerator(
                 config,
                 gpus,

qa/L0_config_search/test.sh

@@ -70,9 +70,6 @@ for launch_mode in $TRITON_LAUNCH_MODES; do
         MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS $MODEL_ANALYZER_PORTS"
 
         if [ $launch_mode == 'remote' ]; then
-            NUM_ROW_OUTPUT_FILE=`echo $config | sed 's/\.yml//'`-param-$launch_mode.txt
-            NUM_MODELS_OUTPUT_FILE=`echo $config | sed 's/\.yml//'`-models-$launch_mode.txt
-
             # For remote launch, set server args and start server
             SERVER=`which tritonserver`
             SERVER_ARGS="--model-repository=$MODEL_REPOSITORY --model-control-mode=explicit --http-port $http_port --grpc-port $grpc_port --metrics-port $metrics_port"
@@ -84,11 +81,10 @@ for launch_mode in $TRITON_LAUNCH_MODES; do
                 cat $SERVER_LOG
                 RET=1
             fi
-        else
-            NUM_MODELS_OUTPUT_FILE=`echo $config | sed 's/\.yml//'`-models.txt
-            NUM_ROW_OUTPUT_FILE=`echo $config | sed 's/\.yml//'`-param.txt
         fi
 
+        NUM_MODELS_OUTPUT_FILE=`echo $config | sed 's/\.yml//'`-models.txt
+        NUM_ROW_OUTPUT_FILE=`echo $config | sed 's/\.yml//'`-param.txt
         TEST_OUTPUT_NUM_ROWS=`cat $NUM_ROW_OUTPUT_FILE`
         TEST_MODELS_NUM=`cat $NUM_MODELS_OUTPUT_FILE`
         MODEL_ANALYZER_SUBCOMMAND="profile"

qa/L0_config_search/test_config_generator.py

@@ -70,7 +70,7 @@ def generate_search_disable(self):
         total_param_count = self._calculate_total_params(
             concurrency_count, instance_count
         )
-        self._write_file(total_param_count, 1, 2, 1, model_config)
+        self._write_file(total_param_count, 2, model_config)
 
     def generate_max_limit_with_model_config(self):
         concurrency_count = 2
@@ -91,7 +91,7 @@ def generate_max_limit_with_model_config(self):
         total_param_count = self._calculate_total_params(
             concurrency_count, instance_count
         )
-        self._write_file(total_param_count, 2, 2, 1, model_config)
+        self._write_file(total_param_count, 2, model_config)
 
     def generate_max_limit(self):
         concurrency_count = 2
@@ -105,7 +105,7 @@ def generate_max_limit(self):
         total_param_count = self._calculate_total_params(
             concurrency_count, instance_count
         )
-        self._write_file(total_param_count, 2, 8, 1, model_config)
+        self._write_file(total_param_count, 8, model_config)
 
     def generate_max_limit_with_param(self):
         concurrency_count = 1  # 1 because concurrency parameter is 1 entry below
@@ -124,7 +124,7 @@ def generate_max_limit_with_param(self):
         total_param_count = self._calculate_total_params(
             concurrency_count, instance_count
         )
-        self._write_file(total_param_count, 1, 6, 1, model_config)
+        self._write_file(total_param_count, 6, model_config)
 
     def generate_max_limit_with_param_and_model_config(self):
         concurrency_count = 1  # 1 because concurrency parameter is 1 entry below
@@ -146,7 +146,7 @@ def generate_max_limit_with_param_and_model_config(self):
         total_param_count = self._calculate_total_params(
             concurrency_count, instance_count
         )
-        self._write_file(total_param_count, 1, 2, 1, model_config)
+        self._write_file(total_param_count, 2, model_config)
 
     def generate_max_limit_with_dynamic_batch_disable(self):
         concurrency_count = 2
@@ -160,7 +160,7 @@ def generate_max_limit_with_dynamic_batch_disable(self):
         total_param_count = self._calculate_total_params(
             concurrency_count, instance_count
         )
-        self._write_file(total_param_count, 2, 4, 1, model_config)
+        self._write_file(total_param_count, 4, model_config)
 
     def _calculate_total_params(
         self, concurrency_count, instance_count, default_config_count=1
@@ -178,19 +178,13 @@ def _calculate_total_params(
     def _write_file(
         self,
         total_param,
-        total_param_remote,
         total_models,
-        total_models_remote,
         model_config,
     ):
         with open(f"./config-{self.test_id}-param.txt", "w") as file:
             file.write(str(total_param))
-        with open(f"./config-{self.test_id}-param-remote.txt", "w") as file:
-            file.write(str(total_param_remote))
         with open(f"./config-{self.test_id}-models.txt", "w") as file:
             file.write(str(total_models))
-        with open(f"./config-{self.test_id}-models-remote.txt", "w") as file:
-            file.write(str(total_models_remote))
         with open(f"./config-{self.test_id}.yml", "w") as file:
             yaml.dump(model_config, file)
 

tests/test_model_config_generator.py

@@ -455,8 +455,6 @@ def test_direct_nonempty_default_config(self):
     def test_remote_yes_params_specified(self):
         """
         Test remote mode with model_config_parameters specified
-
-        It should always return a single empty config in remote mode
         """
 
         # yapf: disable
@@ -473,27 +471,39 @@ def test_remote_yes_params_specified(self):
                             count: [1,2]
             """)
 
-        expected_configs = [{}]
+        expected_configs = [
+            {'instance_group': [{'count': 1, 'kind': 'KIND_GPU'}], 'max_batch_size': 1},
+            {'instance_group': [{'count': 1, 'kind': 'KIND_GPU'}], 'max_batch_size': 4},
+            {'instance_group': [{'count': 1, 'kind': 'KIND_GPU'}], 'max_batch_size': 16},
+            {'instance_group': [{'count': 2, 'kind': 'KIND_GPU'}], 'max_batch_size': 1},
+            {'instance_group': [{'count': 2, 'kind': 'KIND_GPU'}], 'max_batch_size': 4},
+            {'instance_group': [{'count': 2, 'kind': 'KIND_GPU'}], 'max_batch_size': 16}
+        ]
         # yapf: enable
 
         self._run_and_test_model_config_generator(yaml_str, expected_configs)
 
     def test_remote_no_params_specified(self):
         """
         Test remote mode with no model_config_parameters specified
-
-        It should always return a single empty config in remote mode
         """
 
         # yapf: disable
         yaml_str = ("""
             triton_launch_mode: remote
-            run_config_search_max_instance_count: 16
+            run_config_search_max_instance_count: 4
+            run_config_search_min_model_batch_size: 8
+            run_config_search_max_model_batch_size: 8
             profile_models:
                 - my-model
             """)
 
-        expected_configs = [{}]
+        expected_configs = [
+            {'max_batch_size': 8, 'instance_group': [{'count': 1, 'kind': 'KIND_GPU'}],'dynamic_batching': {}},
+            {'max_batch_size': 8, 'instance_group': [{'count': 2, 'kind': 'KIND_GPU'}],'dynamic_batching': {}},
+            {'max_batch_size': 8, 'instance_group': [{'count': 3, 'kind': 'KIND_GPU'}],'dynamic_batching': {}},
+            {'max_batch_size': 8, 'instance_group': [{'count': 4, 'kind': 'KIND_GPU'}],'dynamic_batching': {}}
+        ]
         # yapf: enable
 
         self._run_and_test_model_config_generator(yaml_str, expected_configs)

tests/test_model_manager.py

@@ -257,24 +257,31 @@ def test_manual_concurrency(self):
     def test_remote_mode(self):
         """
         Test remote mode
-
-        In remote mode all model_config_parameters (ie. instance count) are ignored
         """
 
         expected_ranges = [
             {
-                "instances": [None],
-                "kind": [None],
+                "instances": [1, 2],
+                "kind": ["KIND_GPU"],
+                "batching": [0],
+                "batch_sizes": [1],
+                "max_batch_size": [8],
+                "concurrency": [1, 2, 4, 8, 16],
+            },
+            {
+                "instances": [1],
+                "kind": ["KIND_CPU"],
                 "batching": [None],
                 "batch_sizes": [1],
-                "concurrency": [1, 2, 4, 8, 16, 32, 64, 128, 256, 512],
-            }
+                "max_batch_size": [8],
+                "concurrency": [1, 2, 4, 8, 16],
+            },
         ]
 
         yaml_str = """
             profile_models: test_model
-            run_config_search_max_concurrency: 512
-            run_config_search_max_instance_count: 7
+            run_config_search_max_concurrency: 16
+            run_config_search_max_instance_count: 2
             run_config_search_min_model_batch_size: 8
             run_config_search_max_model_batch_size: 8
             run_config_search_disable: False