diff --git a/genai-perf/genai_perf/config/generate/perf_analyzer_config.py b/genai-perf/genai_perf/config/generate/perf_analyzer_config.py index 57cf6bea..0e698651 100644 --- a/genai-perf/genai_perf/config/generate/perf_analyzer_config.py +++ b/genai-perf/genai_perf/config/generate/perf_analyzer_config.py @@ -54,6 +54,15 @@ def _set_options_based_on_objective( if parameter.usage == SearchUsage.RUNTIME_PA: self._parameters[name] = parameter.get_value_based_on_category() + ########################################################################### + # Get Accessor Methods + ########################################################################### + def get_parameters(self) -> Dict[str, Any]: + """ + Returns a dictionary of parameters and their values + """ + return self._parameters + ########################################################################### # CLI String Creation Methods ########################################################################### diff --git a/genai-perf/genai_perf/config/run/run_config.py b/genai-perf/genai_perf/config/run/run_config.py index ace9367e..82e0f1f2 100644 --- a/genai-perf/genai_perf/config/run/run_config.py +++ b/genai-perf/genai_perf/config/run/run_config.py @@ -93,6 +93,9 @@ def read_from_checkpoint(cls, run_config_dict: CheckpointObject) -> "RunConfig": ########################################################################### # Get Accessor Methods ########################################################################### + def get_perf_analyzer_parameters(self) -> Dict[str, Any]: + return self.perf_analyzer_config.get_parameters() + def get_all_gpu_metrics(self) -> GpuRecords: return self.measurement.get_all_gpu_metrics() diff --git a/genai-perf/genai_perf/record/types/input_sequence_length.py b/genai-perf/genai_perf/record/types/input_sequence_length.py new file mode 100644 index 00000000..8d7d4065 --- /dev/null +++ b/genai-perf/genai_perf/record/types/input_sequence_length.py @@ -0,0 +1,103 @@ +# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from functools import total_ordering + +from genai_perf.record.record import IncreasingRecord + + +@total_ordering +class InputSequenceLength(IncreasingRecord): + """ + A record for perf_analyzer + metric 'Input Sequence Length' + """ + + tag = "input_sequence_length" + + def __init__(self, value, timestamp=0): + """ + Parameters + ---------- + value : float + The throughput from the perf analyzer output + timestamp : float + Elapsed time from start of program + """ + + super().__init__(value, timestamp) + + @staticmethod + def value_function(): + """ + Returns the total value from a list + + Returns + ------- + Total value of the list + """ + return sum + + @staticmethod + def header(aggregation_tag=False): + """ + Parameters + ---------- + aggregation_tag: bool + An optional tag that may be displayed + as part of the header indicating that + this record has been aggregated using + max, min or average etc. + + Returns + ------- + str + The full name of the + metric. + """ + + return "Input Sequence Length" + + def __eq__(self, other): + """ + Allows checking for + equality between two records + """ + + return self.value() == other.value() + + def __lt__(self, other): + """ + Allows checking if + this record is less than + the other + """ + + return self.value() < other.value() + + def __add__(self, other): + """ + Allows adding two records together + to produce a brand new record. + """ + + return self.__class__(value=(self.value() + other.value())) + + def __sub__(self, other): + """ + Allows subtracting two records together + to produce a brand new record. + """ + + return self.__class__(value=(self.value() - other.value())) diff --git a/genai-perf/genai_perf/record/types/output_sequence_length.py b/genai-perf/genai_perf/record/types/output_sequence_length.py new file mode 100644 index 00000000..f42f0f18 --- /dev/null +++ b/genai-perf/genai_perf/record/types/output_sequence_length.py @@ -0,0 +1,103 @@ +# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from functools import total_ordering + +from genai_perf.record.record import IncreasingRecord + + +@total_ordering +class OutputSequenceLength(IncreasingRecord): + """ + A record for perf_analyzer + metric 'Output Sequence Length' + """ + + tag = "output_sequence_length" + + def __init__(self, value, timestamp=0): + """ + Parameters + ---------- + value : float + The throughput from the perf analyzer output + timestamp : float + Elapsed time from start of program + """ + + super().__init__(value, timestamp) + + @staticmethod + def value_function(): + """ + Returns the total value from a list + + Returns + ------- + Total value of the list + """ + return sum + + @staticmethod + def header(aggregation_tag=False): + """ + Parameters + ---------- + aggregation_tag: bool + An optional tag that may be displayed + as part of the header indicating that + this record has been aggregated using + max, min or average etc. + + Returns + ------- + str + The full name of the + metric. + """ + + return "Output Sequence Length" + + def __eq__(self, other): + """ + Allows checking for + equality between two records + """ + + return self.value() == other.value() + + def __lt__(self, other): + """ + Allows checking if + this record is less than + the other + """ + + return self.value() < other.value() + + def __add__(self, other): + """ + Allows adding two records together + to produce a brand new record. + """ + + return self.__class__(value=(self.value() + other.value())) + + def __sub__(self, other): + """ + Allows subtracting two records together + to produce a brand new record. + """ + + return self.__class__(value=(self.value() - other.value())) diff --git a/genai-perf/genai_perf/record/types/output_token_throughput_per_request.py b/genai-perf/genai_perf/record/types/output_token_throughput_per_request.py new file mode 100644 index 00000000..04065ef3 --- /dev/null +++ b/genai-perf/genai_perf/record/types/output_token_throughput_per_request.py @@ -0,0 +1,103 @@ +# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from functools import total_ordering + +from genai_perf.record.record import IncreasingRecord + + +@total_ordering +class OutputTokenThroughputPerRequest(IncreasingRecord): + """ + A record for perf_analyzer + metric 'Output Token Throughput Per Request' + """ + + tag = "output_token_throughput_per_request" + + def __init__(self, value, timestamp=0): + """ + Parameters + ---------- + value : float + The throughput from the perf analyzer output + timestamp : float + Elapsed time from start of program + """ + + super().__init__(value, timestamp) + + @staticmethod + def value_function(): + """ + Returns the total value from a list + + Returns + ------- + Total value of the list + """ + return sum + + @staticmethod + def header(aggregation_tag=False): + """ + Parameters + ---------- + aggregation_tag: bool + An optional tag that may be displayed + as part of the header indicating that + this record has been aggregated using + max, min or average etc. + + Returns + ------- + str + The full name of the + metric. + """ + + return "Output Token Throughput Per Request (infer/sec)" + + def __eq__(self, other): + """ + Allows checking for + equality between two records + """ + + return self.value() == other.value() + + def __lt__(self, other): + """ + Allows checking if + this record is less than + the other + """ + + return self.value() < other.value() + + def __add__(self, other): + """ + Allows adding two records together + to produce a brand new record. + """ + + return self.__class__(value=(self.value() + other.value())) + + def __sub__(self, other): + """ + Allows subtracting two records together + to produce a brand new record. + """ + + return self.__class__(value=(self.value() - other.value())) diff --git a/genai-perf/tests/test_checkpoint.py b/genai-perf/tests/test_checkpoint.py index 312a22fa..bfb4a694 100644 --- a/genai-perf/tests/test_checkpoint.py +++ b/genai-perf/tests/test_checkpoint.py @@ -17,6 +17,8 @@ from unittest.mock import patch from genai_perf.checkpoint.checkpoint import Checkpoint +from genai_perf.config.generate.search_parameters import SearchParameters +from genai_perf.config.generate.sweep_objective_generator import SweepObjectiveGenerator from genai_perf.config.input.config_command import ConfigCommand from genai_perf.config.run.results import Results from tests.test_utils import create_run_config @@ -27,17 +29,28 @@ class TestCheckpoint(unittest.TestCase): # Setup & Teardown ########################################################################### def setUp(self): - self._results = Results() + self._config = ConfigCommand(model_names=["test_model"]) + self._model_search_parameters = { + "test_model": SearchParameters(self._config.analyze) + } + + self._sweep_obj_gen = SweepObjectiveGenerator( + self._config, self._model_search_parameters + ) - for i in range(10): - run_config_name = "test_run_config_" + str(i) + self._results = Results() + for count, objective in enumerate(self._sweep_obj_gen.get_objectives()): + run_config_name = "test_model_run_config_" + str(count) run_config = create_run_config( run_config_name=run_config_name, + model_objective_parameters=objective, model_name="test_model", - gpu_power=500 + 10 * i, - gpu_utilization=50 - i, - throughput=300 - 10 * i, - latency=100 - 5 * i, + gpu_power=500 + 10 * count, + gpu_utilization=50 - count, + throughput=300 - 10 * count, + latency=100 - 5 * count, + input_seq_length=20 + 10 * count, + output_seq_length=50 + 5 * count, ) self._results.add_run_config(run_config) diff --git a/genai-perf/tests/test_record.py b/genai-perf/tests/test_record.py index f928296b..d2b398f4 100644 --- a/genai-perf/tests/test_record.py +++ b/genai-perf/tests/test_record.py @@ -83,7 +83,10 @@ def setUp(self): record_types[t] for t in [ "perf_throughput", + "input_sequence_length", + "output_sequence_length", "output_token_throughput", + "output_token_throughput_per_request", "gpu_free_memory", "gpu_utilization", "cpu_available_ram", diff --git a/genai-perf/tests/test_utils.py b/genai-perf/tests/test_utils.py index f882e30b..24cf79ad 100644 --- a/genai-perf/tests/test_utils.py +++ b/genai-perf/tests/test_utils.py @@ -24,7 +24,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from typing import Union +from typing import Optional, Union import pytest from genai_perf.config.generate.genai_perf_config import GenAIPerfConfig @@ -35,9 +35,11 @@ from genai_perf.metrics.statistics import Statistics from genai_perf.record.types.gpu_power_usage import GPUPowerUsage from genai_perf.record.types.gpu_utilization import GPUUtilization +from genai_perf.record.types.input_sequence_length import InputSequenceLength +from genai_perf.record.types.output_sequence_length import OutputSequenceLength from genai_perf.record.types.perf_latency_p99 import PerfLatencyP99 from genai_perf.record.types.perf_throughput import PerfThroughput -from genai_perf.types import GpuId, PerfRecords +from genai_perf.types import GpuId, ModelObjectiveParameters, PerfRecords def ns_to_sec(ns: int) -> Union[int, float]: @@ -57,14 +59,21 @@ def check_statistics(s1: Statistics, s2: Statistics) -> None: ########################################################################### # Perf Metrics Constructor ########################################################################### -def create_perf_metrics(throughput: int, latency: int) -> PerfRecords: - throughput_record = PerfThroughput(throughput) - latency_record = PerfLatencyP99(latency) - - perf_metrics = { - PerfThroughput.tag: throughput_record, - PerfLatencyP99.tag: latency_record, - } +def create_perf_metrics( + throughput: Optional[int] = None, + latency: Optional[int] = None, + input_seq_length: Optional[int] = None, + output_seq_length: Optional[int] = None, +) -> PerfRecords: + perf_metrics: PerfRecords = {} + if throughput: + perf_metrics[PerfThroughput.tag] = PerfThroughput(throughput) + if latency: + perf_metrics[PerfLatencyP99.tag] = PerfLatencyP99(latency) + if input_seq_length: + perf_metrics[InputSequenceLength.tag] = InputSequenceLength(input_seq_length) + if output_seq_length: + perf_metrics[OutputSequenceLength.tag] = OutputSequenceLength(output_seq_length) return perf_metrics @@ -94,23 +103,40 @@ def create_run_config_measurement( def create_run_config( run_config_name: str, model_name: str = "test_model", + model_objective_parameters: ModelObjectiveParameters = {}, gpu_power: int = 0, gpu_utilization: int = 0, gpu_id: GpuId = "0", throughput: int = 0, latency: int = 0, + input_seq_length: int = 0, + output_seq_length: int = 0, ) -> RunConfig: config = ConfigCommand([model_name]) - genai_perf_config = GenAIPerfConfig(config=config, model_objective_parameters={}) + genai_perf_config = GenAIPerfConfig( + config=config, model_objective_parameters=model_objective_parameters + ) perf_analyzer_config = PerfAnalyzerConfig( - model_name=model_name, config=config, model_objective_parameters={} + model_name=model_name, + config=config, + model_objective_parameters=model_objective_parameters, ) + rcm = create_run_config_measurement(gpu_power, gpu_utilization, gpu_id) - perf_metrics = create_perf_metrics(throughput, latency) - rcm.add_perf_metrics(model_name, perf_metrics) + + perf_metrics = create_perf_metrics( + throughput=throughput, + latency=latency, + input_seq_length=input_seq_length, + output_seq_length=output_seq_length, + ) + rcm.add_perf_metrics(model_name=model_name, perf_metrics=perf_metrics) run_config = RunConfig( - run_config_name, genai_perf_config, perf_analyzer_config, rcm + name=run_config_name, + genai_perf_config=genai_perf_config, + perf_analyzer_config=perf_analyzer_config, + measurement=rcm, ) return run_config