Adding records for ISL/OSL and testing this in checkpoint creation

genai-perf/genai_perf/config/generate/perf_analyzer_config.py

@@ -54,6 +54,15 @@ def _set_options_based_on_objective(
                 if parameter.usage == SearchUsage.RUNTIME_PA:
                     self._parameters[name] = parameter.get_value_based_on_category()
 
+    ###########################################################################
+    # Get Accessor Methods
+    ###########################################################################
+    def get_parameters(self) -> Dict[str, Any]:
+        """
+        Returns a dictionary of parameters and their values
+        """
+        return self._parameters
+
     ###########################################################################
     # CLI String Creation Methods
     ###########################################################################

genai-perf/genai_perf/config/run/run_config.py

@@ -93,6 +93,9 @@ def read_from_checkpoint(cls, run_config_dict: CheckpointObject) -> "RunConfig":
     ###########################################################################
     # Get Accessor Methods
     ###########################################################################
+    def get_perf_analyzer_parameters(self) -> Dict[str, Any]:
+        return self.perf_analyzer_config.get_parameters()
+
     def get_all_gpu_metrics(self) -> GpuRecords:
         return self.measurement.get_all_gpu_metrics()
 

genai-perf/genai_perf/record/types/input_sequence_length.py

@@ -0,0 +1,103 @@
+# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from functools import total_ordering
+
+from genai_perf.record.record import IncreasingRecord
+
+
+@total_ordering
+class InputSequenceLength(IncreasingRecord):
+    """
+    A record for perf_analyzer
+    metric 'Input Sequence Length'
+    """
+
+    tag = "input_sequence_length"
+
+    def __init__(self, value, timestamp=0):
+        """
+        Parameters
+        ----------
+        value : float
+            The throughput from the perf analyzer output
+        timestamp : float
+            Elapsed time from start of program
+        """
+
+        super().__init__(value, timestamp)
+
+    @staticmethod
+    def value_function():
+        """
+        Returns the total value from a list
+
+        Returns
+        -------
+        Total value of the list
+        """
+        return sum
+
+    @staticmethod
+    def header(aggregation_tag=False):
+        """
+        Parameters
+        ----------
+        aggregation_tag: bool
+            An optional tag that may be displayed
+            as part of the header indicating that
+            this record has been aggregated using
+            max, min or average etc.
+
+        Returns
+        -------
+        str
+            The full name of the
+            metric.
+        """
+
+        return "Input Sequence Length"
+
+    def __eq__(self, other):
+        """
+        Allows checking for
+        equality between two records
+        """
+
+        return self.value() == other.value()
+
+    def __lt__(self, other):
+        """
+        Allows checking if
+        this record is less than
+        the other
+        """
+
+        return self.value() < other.value()
+
+    def __add__(self, other):
+        """
+        Allows adding two records together
+        to produce a brand new record.
+        """
+
+        return self.__class__(value=(self.value() + other.value()))
+
+    def __sub__(self, other):
+        """
+        Allows subtracting two records together
+        to produce a brand new record.
+        """
+
+        return self.__class__(value=(self.value() - other.value()))

genai-perf/genai_perf/record/types/output_sequence_length.py

@@ -0,0 +1,103 @@
+# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from functools import total_ordering
+
+from genai_perf.record.record import IncreasingRecord
+
+
+@total_ordering
+class OutputSequenceLength(IncreasingRecord):
+    """
+    A record for perf_analyzer
+    metric 'Output Sequence Length'
+    """
+
+    tag = "output_sequence_length"
+
+    def __init__(self, value, timestamp=0):
+        """
+        Parameters
+        ----------
+        value : float
+            The throughput from the perf analyzer output
+        timestamp : float
+            Elapsed time from start of program
+        """
+
+        super().__init__(value, timestamp)
+
+    @staticmethod
+    def value_function():
+        """
+        Returns the total value from a list
+
+        Returns
+        -------
+        Total value of the list
+        """
+        return sum
+
+    @staticmethod
+    def header(aggregation_tag=False):
+        """
+        Parameters
+        ----------
+        aggregation_tag: bool
+            An optional tag that may be displayed
+            as part of the header indicating that
+            this record has been aggregated using
+            max, min or average etc.
+
+        Returns
+        -------
+        str
+            The full name of the
+            metric.
+        """
+
+        return "Output Sequence Length"
+
+    def __eq__(self, other):
+        """
+        Allows checking for
+        equality between two records
+        """
+
+        return self.value() == other.value()
+
+    def __lt__(self, other):
+        """
+        Allows checking if
+        this record is less than
+        the other
+        """
+
+        return self.value() < other.value()
+
+    def __add__(self, other):
+        """
+        Allows adding two records together
+        to produce a brand new record.
+        """
+
+        return self.__class__(value=(self.value() + other.value()))
+
+    def __sub__(self, other):
+        """
+        Allows subtracting two records together
+        to produce a brand new record.
+        """
+
+        return self.__class__(value=(self.value() - other.value()))

genai-perf/genai_perf/record/types/output_token_throughput_per_request.py

@@ -0,0 +1,103 @@
+# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from functools import total_ordering
+
+from genai_perf.record.record import IncreasingRecord
+
+
+@total_ordering
+class OutputTokenThroughputPerRequest(IncreasingRecord):
+    """
+    A record for perf_analyzer
+    metric 'Output Token Throughput Per Request'
+    """
+
+    tag = "output_token_throughput_per_request"
+
+    def __init__(self, value, timestamp=0):
+        """
+        Parameters
+        ----------
+        value : float
+            The throughput from the perf analyzer output
+        timestamp : float
+            Elapsed time from start of program
+        """
+
+        super().__init__(value, timestamp)
+
+    @staticmethod
+    def value_function():
+        """
+        Returns the total value from a list
+
+        Returns
+        -------
+        Total value of the list
+        """
+        return sum
+
+    @staticmethod
+    def header(aggregation_tag=False):
+        """
+        Parameters
+        ----------
+        aggregation_tag: bool
+            An optional tag that may be displayed
+            as part of the header indicating that
+            this record has been aggregated using
+            max, min or average etc.
+
+        Returns
+        -------
+        str
+            The full name of the
+            metric.
+        """
+
+        return "Output Token Throughput Per Request (infer/sec)"
+
+    def __eq__(self, other):
+        """
+        Allows checking for
+        equality between two records
+        """
+
+        return self.value() == other.value()
+
+    def __lt__(self, other):
+        """
+        Allows checking if
+        this record is less than
+        the other
+        """
+
+        return self.value() < other.value()
+
+    def __add__(self, other):
+        """
+        Allows adding two records together
+        to produce a brand new record.
+        """
+
+        return self.__class__(value=(self.value() + other.value()))
+
+    def __sub__(self, other):
+        """
+        Allows subtracting two records together
+        to produce a brand new record.
+        """
+
+        return self.__class__(value=(self.value() - other.value()))

genai-perf/tests/test_checkpoint.py

@@ -17,6 +17,8 @@
 from unittest.mock import patch
 
 from genai_perf.checkpoint.checkpoint import Checkpoint
+from genai_perf.config.generate.search_parameters import SearchParameters
+from genai_perf.config.generate.sweep_objective_generator import SweepObjectiveGenerator
 from genai_perf.config.input.config_command import ConfigCommand
 from genai_perf.config.run.results import Results
 from tests.test_utils import create_run_config
@@ -27,17 +29,28 @@ class TestCheckpoint(unittest.TestCase):
     # Setup & Teardown
     ###########################################################################
     def setUp(self):
-        self._results = Results()
+        self._config = ConfigCommand(model_names=["test_model"])
+        self._model_search_parameters = {
+            "test_model": SearchParameters(self._config.analyze)
+        }
+
+        self._sweep_obj_gen = SweepObjectiveGenerator(
+            self._config, self._model_search_parameters
+        )
 
-        for i in range(10):
-            run_config_name = "test_run_config_" + str(i)
+        self._results = Results()
+        for count, objective in enumerate(self._sweep_obj_gen.get_objectives()):
+            run_config_name = "test_model_run_config_" + str(count)
             run_config = create_run_config(
                 run_config_name=run_config_name,
+                model_objective_parameters=objective,
                 model_name="test_model",
-                gpu_power=500 + 10 * i,
-                gpu_utilization=50 - i,
-                throughput=300 - 10 * i,
-                latency=100 - 5 * i,
+                gpu_power=500 + 10 * count,
+                gpu_utilization=50 - count,
+                throughput=300 - 10 * count,
+                latency=100 - 5 * count,
+                input_seq_length=20 + 10 * count,
+                output_seq_length=50 + 5 * count,
             )
             self._results.add_run_config(run_config)
 

genai-perf/tests/test_record.py

@@ -83,7 +83,10 @@ def setUp(self):
             record_types[t]
             for t in [
                 "perf_throughput",
+                "input_sequence_length",
+                "output_sequence_length",
                 "output_token_throughput",
+                "output_token_throughput_per_request",
                 "gpu_free_memory",
                 "gpu_utilization",
                 "cpu_available_ram",