Add testing for DataLoader (#727)

* Add testing for DataLoader in PA
triton-inference-server · Jul 25, 2024 · 442915d · 442915d
1 parent c514dea
commit 442915d
Show file tree

Hide file tree

Showing 3 changed files with 196 additions and 1 deletion.
diff --git a/src/c++/perf_analyzer/perf_utils.h b/src/c++/perf_analyzer/perf_utils.h
@@ -56,7 +56,7 @@ constexpr uint64_t NANOS_PER_MILLIS = 1000000;
 
 // Will use the characters specified here to construct random strings
 std::string const character_set =
-    "abcdefghijklmnaoqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890 .?!";
+    "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890 .?!";
 
 // A boolean flag to mark an interrupt and commencement of early exit
 extern volatile bool early_exit;

diff --git a/src/c++/perf_analyzer/test_dataloader.cc b/src/c++/perf_analyzer/test_dataloader.cc
@@ -28,6 +28,7 @@
 #include "doctest.h"
 #include "mock_data_loader.h"
 
+
 namespace triton { namespace perfanalyzer {
 
 /// Helper class for testing the DataLoader
@@ -104,6 +105,199 @@ TEST_CASE("dataloader: GetTotalSteps")
   CHECK_EQ(dataloader.GetTotalSteps(2), 0);
 }
 
+TEST_CASE("dataloader: ValidateIOExistsInModel")
+{
+  MockDataLoader dataloader;
+  std::shared_ptr<ModelTensorMap> inputs = std::make_shared<ModelTensorMap>();
+  std::shared_ptr<ModelTensorMap> outputs = std::make_shared<ModelTensorMap>();
+  ModelTensor input1 = TestDataLoader::CreateTensor("INPUT1");
+  ModelTensor output1 = TestDataLoader::CreateTensor("OUTPUT1");
+  inputs->insert(std::make_pair(input1.name_, input1));
+  outputs->insert(std::make_pair(output1.name_, output1));
+
+  SUBCASE("Directory does not exist")
+  {
+    std::string data_directory = "non_existent_directory";
+    cb::Error status =
+        dataloader.ValidateIOExistsInModel(inputs, outputs, data_directory);
+    CHECK(
+        status.Message() ==
+        "Error: Directory does not exist or is not a directory: "
+        "non_existent_directory");
+    CHECK(status.Err() == pa::GENERIC_ERROR);
+  }
+
+  SUBCASE("Directory is not a directory")
+  {
+    std::string data_directory = "tmp/test.txt";
+    std::ofstream file(data_directory);
+    cb::Error status =
+        dataloader.ValidateIOExistsInModel(inputs, outputs, data_directory);
+    CHECK(
+        status.Message() ==
+        "Error: Directory does not exist or is not a directory: tmp/test.txt");
+    CHECK(status.Err() == pa::GENERIC_ERROR);
+    std::remove(data_directory.c_str());
+  }
+
+  SUBCASE("Valid directory but no corresponding files")
+  {
+    std::string data_directory = "valid_directory";
+    std::filesystem::create_directory(data_directory);
+    std::ofstream(data_directory + "/invalid_file").close();
+    cb::Error status =
+        dataloader.ValidateIOExistsInModel(inputs, outputs, data_directory);
+    std::filesystem::remove_all(data_directory);
+    CHECK(
+        status.Message() ==
+        "Provided data file 'invalid_file' does not correspond to a valid "
+        "model input or output.");
+    CHECK(status.Err() == pa::GENERIC_ERROR);
+  }
+
+  SUBCASE("Valid directory with corresponding files")
+  {
+    std::string data_directory = "valid_directory";
+    std::filesystem::create_directory(data_directory);
+    std::ofstream(data_directory + "/INPUT1").close();
+    std::ofstream(data_directory + "/OUTPUT1").close();
+    cb::Error status =
+        dataloader.ValidateIOExistsInModel(inputs, outputs, data_directory);
+    std::filesystem::remove_all(data_directory);
+    CHECK(status.Message().empty());
+    CHECK(status.IsOk());
+  }
+
+  SUBCASE("Valid directory with multiple input and output tensors")
+  {
+    ModelTensor input2 = TestDataLoader::CreateTensor("INPUT2");
+    ModelTensor output2 = TestDataLoader::CreateTensor("OUTPUT2");
+
+    inputs->insert(std::make_pair(input2.name_, input2));
+    outputs->insert(std::make_pair(output2.name_, output2));
+
+    std::string data_directory = "valid_directory_multiple";
+    std::filesystem::create_directory(data_directory);
+    std::ofstream(data_directory + "/INPUT1").close();
+    std::ofstream(data_directory + "/INPUT2").close();
+    std::ofstream(data_directory + "/OUTPUT1").close();
+    std::ofstream(data_directory + "/OUTPUT2").close();
+
+    cb::Error status =
+        dataloader.ValidateIOExistsInModel(inputs, outputs, data_directory);
+    std::filesystem::remove_all(data_directory);
+    CHECK(status.Message().empty());
+    CHECK(status.IsOk());
+  }
+}
+
+TEST_CASE("dataloader: ReadDataFromJSON")
+{
+  DataLoader dataloader;
+  std::shared_ptr<ModelTensorMap> inputs = std::make_shared<ModelTensorMap>();
+  std::shared_ptr<ModelTensorMap> outputs = std::make_shared<ModelTensorMap>();
+  ModelTensor input1 = TestDataLoader::CreateTensor("INPUT1");
+  ModelTensor output1 = TestDataLoader::CreateTensor("OUTPUT1");
+
+  inputs->insert(std::make_pair(input1.name_, input1));
+  outputs->insert(std::make_pair(output1.name_, output1));
+
+  SUBCASE("File does not exist")
+  {
+    std::string json_file = "non_existent_file.json";
+    cb::Error status = dataloader.ReadDataFromJSON(inputs, outputs, json_file);
+    CHECK(status.Message() == "failed to open file for reading provided data");
+    CHECK(status.Err() == pa::GENERIC_ERROR);
+  }
+
+  SUBCASE("Valid JSON file")
+  {
+    std::string json_file = "valid_file.json";
+    std::ofstream out(json_file);
+    out << R"({
+                    "data": [
+                      { "INPUT1": [1] },
+                      { "INPUT1": [2] },
+                      { "INPUT1": [3] }
+                    ],
+                    "validation_data": [
+                      { "OUTPUT1": [4] },
+                      { "OUTPUT1": [5] },
+                      { "OUTPUT1": [6] }
+                  ]})";
+    out.close();
+
+    cb::Error status = dataloader.ReadDataFromJSON(inputs, outputs, json_file);
+    std::filesystem::remove(json_file);
+    CHECK(status.Message().empty());
+    CHECK(status.IsOk());
+  }
+
+  SUBCASE("Invalid JSON file")
+  {
+    std::string json_file = "invalid_file.json";
+    std::ofstream out(json_file);
+    out << R"({invalid_json: 1,)";
+    out.close();
+
+    cb::Error status = dataloader.ReadDataFromJSON(inputs, outputs, json_file);
+    std::filesystem::remove(json_file);
+
+    CHECK(
+        status.Message() ==
+        "failed to parse the specified json file for reading provided data");
+    CHECK(status.Err() == pa::GENERIC_ERROR);
+  }
+
+  SUBCASE("Multiple input and output tensors")
+  {
+    ModelTensor input2 = TestDataLoader::CreateTensor("INPUT2");
+    ModelTensor output2 = TestDataLoader::CreateTensor("OUTPUT2");
+
+    inputs->insert(std::make_pair(input2.name_, input2));
+    outputs->insert(std::make_pair(output2.name_, output2));
+
+    std::string json_file = "valid_file_multiple_input_output.json";
+    std::ofstream out(json_file);
+    out << R"({
+                "data": [
+                    {
+                        "INPUT1": [1],
+                        "INPUT2": [4]
+                    },
+                    {
+                        "INPUT1": [2],
+                        "INPUT2": [5]
+                    },
+                    {
+                        "INPUT1": [3],
+                        "INPUT2": [6]
+                    }
+                ],
+                "validation_data": [
+                    {
+                        "OUTPUT1": [4],
+                        "OUTPUT2": [7]
+                    },
+                    {
+                        "OUTPUT1": [5],
+                        "OUTPUT2": [8]
+                    },
+                    {
+                        "OUTPUT1": [6],
+                        "OUTPUT2": [9]
+                    }
+                ]
+              })";
+    out.close();
+
+    cb::Error status = dataloader.ReadDataFromJSON(inputs, outputs, json_file);
+    std::filesystem::remove(json_file);
+    CHECK(status.Message().empty());
+    CHECK(status.IsOk());
+  }
+}
+
 TEST_CASE("dataloader: GetInputData missing data")
 {
   MockDataLoader dataloader;

diff --git a/src/c++/perf_analyzer/test_perf_utils.cc b/src/c++/perf_analyzer/test_perf_utils.cc
@@ -144,6 +144,7 @@ TEST_CASE("perf_utils: ConvertDTypeFromTFS")
         std::make_pair("DT_DOUBLE", "FP64"),
         std::make_pair("DT_INT32", "INT32"),
         std::make_pair("DT_INT16", "INT16"),
+        std::make_pair("DT_UINT16", "UINT16"),
         std::make_pair("DT_INT8", "INT8"),
         std::make_pair("DT_UINT8", "UINT8"),
         std::make_pair("DT_STRING", "BYTES"),