From 281f6cd9a290a475aa143b63e57c588ec3afc14a Mon Sep 17 00:00:00 2001 From: Brian Raf <92820864+nv-braf@users.noreply.github.com> Date: Thu, 24 Aug 2023 07:35:24 -0700 Subject: [PATCH] Add remote mode config sweep to brute search (#751) * Brute run in remote mode changes * Fixing L0 test * Fixes based on review comments * Fix L0 --- .../generate/brute_run_config_generator.py | 6 +---- .../generate/manual_model_config_generator.py | 19 --------------- .../model_config_generator_factory.py | 3 +-- qa/L0_config_search/test.sh | 8 ++----- qa/L0_config_search/test_config_generator.py | 18 +++++--------- tests/test_model_config_generator.py | 24 +++++++++++++------ tests/test_model_manager.py | 23 +++++++++++------- 7 files changed, 42 insertions(+), 59 deletions(-) diff --git a/model_analyzer/config/generate/brute_run_config_generator.py b/model_analyzer/config/generate/brute_run_config_generator.py index 56bc8cb1a..d226811aa 100755 --- a/model_analyzer/config/generate/brute_run_config_generator.py +++ b/model_analyzer/config/generate/brute_run_config_generator.py @@ -102,11 +102,7 @@ def _get_next_config(self) -> Generator[RunConfig, None, None]: if not self._skip_default_config: yield from self._generate_subset(0, default_only=True) - if self._should_generate_non_default_configs(): - yield from self._generate_subset(0, default_only=False) - - def _should_generate_non_default_configs(self) -> bool: - return self._config.triton_launch_mode != "remote" + yield from self._generate_subset(0, default_only=False) def _generate_subset( self, index: int, default_only: bool diff --git a/model_analyzer/config/generate/manual_model_config_generator.py b/model_analyzer/config/generate/manual_model_config_generator.py index 671ad65f9..a95ae27ab 100755 --- a/model_analyzer/config/generate/manual_model_config_generator.py +++ b/model_analyzer/config/generate/manual_model_config_generator.py @@ -130,23 +130,7 @@ def _get_next_model_config_variant(self) -> ModelConfigVariant: def _generate_model_config_variants(self) -> List[List[ModelConfigVariant]]: """Generate all model config combinations""" - if self._remote_mode: - configs = self._generate_remote_mode_model_config_variants() - else: - configs = self._generate_direct_modes_model_config_variants() - - return configs - - def _generate_remote_mode_model_config_variants( - self, - ) -> List[List[ModelConfigVariant]]: - """Generate model configs for remote mode""" - return [[self._make_remote_model_config_variant()]] - def _generate_direct_modes_model_config_variants( - self, - ) -> List[List[ModelConfigVariant]]: - """Generate model configs for direct (non-remote) modes""" model_config_variants = [] for param_combo in self._non_max_batch_size_param_combos: configs_with_max_batch_size = [] @@ -171,9 +155,6 @@ def _determine_max_batch_sizes_and_param_combos(self) -> None: """ Determine self._max_batch_sizes and self._non_max_batch_size_param_combos """ - if self._remote_mode: - return - if self._default_only: self._non_max_batch_size_param_combos = [DEFAULT_CONFIG_PARAMS] else: diff --git a/model_analyzer/config/generate/model_config_generator_factory.py b/model_analyzer/config/generate/model_config_generator_factory.py index ceeb1259e..b728df548 100755 --- a/model_analyzer/config/generate/model_config_generator_factory.py +++ b/model_analyzer/config/generate/model_config_generator_factory.py @@ -67,11 +67,10 @@ def create_model_config_generator( A generator that implements ConfigGeneratorInterface and creates ModelConfigs """ - remote_mode = config.triton_launch_mode == "remote" search_disabled = config.run_config_search_disable model_config_params = model.model_config_parameters() - if remote_mode or search_disabled or model_config_params: + if search_disabled or model_config_params: return ManualModelConfigGenerator( config, gpus, diff --git a/qa/L0_config_search/test.sh b/qa/L0_config_search/test.sh index 347f31efd..0679b85a8 100755 --- a/qa/L0_config_search/test.sh +++ b/qa/L0_config_search/test.sh @@ -70,9 +70,6 @@ for launch_mode in $TRITON_LAUNCH_MODES; do MODEL_ANALYZER_ARGS="$MODEL_ANALYZER_ARGS $MODEL_ANALYZER_PORTS" if [ $launch_mode == 'remote' ]; then - NUM_ROW_OUTPUT_FILE=`echo $config | sed 's/\.yml//'`-param-$launch_mode.txt - NUM_MODELS_OUTPUT_FILE=`echo $config | sed 's/\.yml//'`-models-$launch_mode.txt - # For remote launch, set server args and start server SERVER=`which tritonserver` SERVER_ARGS="--model-repository=$MODEL_REPOSITORY --model-control-mode=explicit --http-port $http_port --grpc-port $grpc_port --metrics-port $metrics_port" @@ -84,11 +81,10 @@ for launch_mode in $TRITON_LAUNCH_MODES; do cat $SERVER_LOG RET=1 fi - else - NUM_MODELS_OUTPUT_FILE=`echo $config | sed 's/\.yml//'`-models.txt - NUM_ROW_OUTPUT_FILE=`echo $config | sed 's/\.yml//'`-param.txt fi + NUM_MODELS_OUTPUT_FILE=`echo $config | sed 's/\.yml//'`-models.txt + NUM_ROW_OUTPUT_FILE=`echo $config | sed 's/\.yml//'`-param.txt TEST_OUTPUT_NUM_ROWS=`cat $NUM_ROW_OUTPUT_FILE` TEST_MODELS_NUM=`cat $NUM_MODELS_OUTPUT_FILE` MODEL_ANALYZER_SUBCOMMAND="profile" diff --git a/qa/L0_config_search/test_config_generator.py b/qa/L0_config_search/test_config_generator.py index 5ca2d689c..79145d56c 100755 --- a/qa/L0_config_search/test_config_generator.py +++ b/qa/L0_config_search/test_config_generator.py @@ -70,7 +70,7 @@ def generate_search_disable(self): total_param_count = self._calculate_total_params( concurrency_count, instance_count ) - self._write_file(total_param_count, 1, 2, 1, model_config) + self._write_file(total_param_count, 2, model_config) def generate_max_limit_with_model_config(self): concurrency_count = 2 @@ -91,7 +91,7 @@ def generate_max_limit_with_model_config(self): total_param_count = self._calculate_total_params( concurrency_count, instance_count ) - self._write_file(total_param_count, 2, 2, 1, model_config) + self._write_file(total_param_count, 2, model_config) def generate_max_limit(self): concurrency_count = 2 @@ -105,7 +105,7 @@ def generate_max_limit(self): total_param_count = self._calculate_total_params( concurrency_count, instance_count ) - self._write_file(total_param_count, 2, 8, 1, model_config) + self._write_file(total_param_count, 8, model_config) def generate_max_limit_with_param(self): concurrency_count = 1 # 1 because concurrency parameter is 1 entry below @@ -124,7 +124,7 @@ def generate_max_limit_with_param(self): total_param_count = self._calculate_total_params( concurrency_count, instance_count ) - self._write_file(total_param_count, 1, 6, 1, model_config) + self._write_file(total_param_count, 6, model_config) def generate_max_limit_with_param_and_model_config(self): concurrency_count = 1 # 1 because concurrency parameter is 1 entry below @@ -146,7 +146,7 @@ def generate_max_limit_with_param_and_model_config(self): total_param_count = self._calculate_total_params( concurrency_count, instance_count ) - self._write_file(total_param_count, 1, 2, 1, model_config) + self._write_file(total_param_count, 2, model_config) def generate_max_limit_with_dynamic_batch_disable(self): concurrency_count = 2 @@ -160,7 +160,7 @@ def generate_max_limit_with_dynamic_batch_disable(self): total_param_count = self._calculate_total_params( concurrency_count, instance_count ) - self._write_file(total_param_count, 2, 4, 1, model_config) + self._write_file(total_param_count, 4, model_config) def _calculate_total_params( self, concurrency_count, instance_count, default_config_count=1 @@ -178,19 +178,13 @@ def _calculate_total_params( def _write_file( self, total_param, - total_param_remote, total_models, - total_models_remote, model_config, ): with open(f"./config-{self.test_id}-param.txt", "w") as file: file.write(str(total_param)) - with open(f"./config-{self.test_id}-param-remote.txt", "w") as file: - file.write(str(total_param_remote)) with open(f"./config-{self.test_id}-models.txt", "w") as file: file.write(str(total_models)) - with open(f"./config-{self.test_id}-models-remote.txt", "w") as file: - file.write(str(total_models_remote)) with open(f"./config-{self.test_id}.yml", "w") as file: yaml.dump(model_config, file) diff --git a/tests/test_model_config_generator.py b/tests/test_model_config_generator.py index 5a54f8c04..8f7ff3512 100755 --- a/tests/test_model_config_generator.py +++ b/tests/test_model_config_generator.py @@ -455,8 +455,6 @@ def test_direct_nonempty_default_config(self): def test_remote_yes_params_specified(self): """ Test remote mode with model_config_parameters specified - - It should always return a single empty config in remote mode """ # yapf: disable @@ -473,7 +471,14 @@ def test_remote_yes_params_specified(self): count: [1,2] """) - expected_configs = [{}] + expected_configs = [ + {'instance_group': [{'count': 1, 'kind': 'KIND_GPU'}], 'max_batch_size': 1}, + {'instance_group': [{'count': 1, 'kind': 'KIND_GPU'}], 'max_batch_size': 4}, + {'instance_group': [{'count': 1, 'kind': 'KIND_GPU'}], 'max_batch_size': 16}, + {'instance_group': [{'count': 2, 'kind': 'KIND_GPU'}], 'max_batch_size': 1}, + {'instance_group': [{'count': 2, 'kind': 'KIND_GPU'}], 'max_batch_size': 4}, + {'instance_group': [{'count': 2, 'kind': 'KIND_GPU'}], 'max_batch_size': 16} + ] # yapf: enable self._run_and_test_model_config_generator(yaml_str, expected_configs) @@ -481,19 +486,24 @@ def test_remote_yes_params_specified(self): def test_remote_no_params_specified(self): """ Test remote mode with no model_config_parameters specified - - It should always return a single empty config in remote mode """ # yapf: disable yaml_str = (""" triton_launch_mode: remote - run_config_search_max_instance_count: 16 + run_config_search_max_instance_count: 4 + run_config_search_min_model_batch_size: 8 + run_config_search_max_model_batch_size: 8 profile_models: - my-model """) - expected_configs = [{}] + expected_configs = [ + {'max_batch_size': 8, 'instance_group': [{'count': 1, 'kind': 'KIND_GPU'}],'dynamic_batching': {}}, + {'max_batch_size': 8, 'instance_group': [{'count': 2, 'kind': 'KIND_GPU'}],'dynamic_batching': {}}, + {'max_batch_size': 8, 'instance_group': [{'count': 3, 'kind': 'KIND_GPU'}],'dynamic_batching': {}}, + {'max_batch_size': 8, 'instance_group': [{'count': 4, 'kind': 'KIND_GPU'}],'dynamic_batching': {}} + ] # yapf: enable self._run_and_test_model_config_generator(yaml_str, expected_configs) diff --git a/tests/test_model_manager.py b/tests/test_model_manager.py index 3b338326c..0370c6c77 100755 --- a/tests/test_model_manager.py +++ b/tests/test_model_manager.py @@ -257,24 +257,31 @@ def test_manual_concurrency(self): def test_remote_mode(self): """ Test remote mode - - In remote mode all model_config_parameters (ie. instance count) are ignored """ expected_ranges = [ { - "instances": [None], - "kind": [None], + "instances": [1, 2], + "kind": ["KIND_GPU"], + "batching": [0], + "batch_sizes": [1], + "max_batch_size": [8], + "concurrency": [1, 2, 4, 8, 16], + }, + { + "instances": [1], + "kind": ["KIND_CPU"], "batching": [None], "batch_sizes": [1], - "concurrency": [1, 2, 4, 8, 16, 32, 64, 128, 256, 512], - } + "max_batch_size": [8], + "concurrency": [1, 2, 4, 8, 16], + }, ] yaml_str = """ profile_models: test_model - run_config_search_max_concurrency: 512 - run_config_search_max_instance_count: 7 + run_config_search_max_concurrency: 16 + run_config_search_max_instance_count: 2 run_config_search_min_model_batch_size: 8 run_config_search_max_model_batch_size: 8 run_config_search_disable: False