diff --git a/src/c++/library/CMakeLists.txt b/src/c++/library/CMakeLists.txt index 697930f4e..7a62971e5 100644 --- a/src/c++/library/CMakeLists.txt +++ b/src/c++/library/CMakeLists.txt @@ -223,7 +223,6 @@ if(TRITON_ENABLE_CC_GRPC OR TRITON_ENABLE_PERF_ANALYZER) grpcclient_static PRIVATE gRPC::grpc++ PRIVATE gRPC::grpc - PRIVATE triton-common-model-config PUBLIC protobuf::libprotobuf PUBLIC Threads::Threads ) @@ -252,7 +251,6 @@ if(TRITON_ENABLE_CC_GRPC OR TRITON_ENABLE_PERF_ANALYZER) grpcclient PRIVATE gRPC::grpc++ PRIVATE gRPC::grpc - PRIVATE triton-common-model-config PUBLIC protobuf::libprotobuf PUBLIC Threads::Threads ) @@ -378,10 +376,6 @@ if(TRITON_ENABLE_CC_HTTP OR TRITON_ENABLE_PERF_ANALYZER) http-client-library EXCLUDE_FROM_ALL OBJECT ${REQUEST_SRCS} ${REQUEST_HDRS} ) - add_dependencies( - http-client-library - proto-library - ) if (NOT WIN32) set_property( @@ -394,14 +388,12 @@ if(TRITON_ENABLE_CC_HTTP OR TRITON_ENABLE_PERF_ANALYZER) http-client-library PUBLIC triton-common-json # from repo-common - triton-common-model-config ) # libhttpclient_static.a add_library( httpclient_static STATIC $ - $ ) add_library( TritonClient::httpclient_static ALIAS httpclient_static @@ -410,7 +402,6 @@ if(TRITON_ENABLE_CC_HTTP OR TRITON_ENABLE_PERF_ANALYZER) target_link_libraries( httpclient_static PRIVATE triton-common-json - PRIVATE triton-common-model-config PUBLIC CURL::libcurl PUBLIC Threads::Threads ) @@ -426,7 +417,6 @@ if(TRITON_ENABLE_CC_HTTP OR TRITON_ENABLE_PERF_ANALYZER) add_library( httpclient SHARED $ - $ ) add_library( TritonClient::httpclient ALIAS httpclient @@ -444,7 +434,6 @@ if(TRITON_ENABLE_CC_HTTP OR TRITON_ENABLE_PERF_ANALYZER) target_link_libraries( httpclient PRIVATE triton-common-json - PRIVATE triton-common-model-config PUBLIC CURL::libcurl PUBLIC Threads::Threads ) @@ -470,7 +459,6 @@ if(TRITON_ENABLE_CC_HTTP OR TRITON_ENABLE_PERF_ANALYZER) $ $ $ - $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} ) diff --git a/src/c++/library/common.cc b/src/c++/library/common.cc index 566e3d366..27da69888 100644 --- a/src/c++/library/common.cc +++ b/src/c++/library/common.cc @@ -1,4 +1,4 @@ -// Copyright 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions @@ -26,10 +26,6 @@ #include "common.h" -#include - -#include "triton/common/model_config.h" - namespace triton { namespace client { //============================================================================== @@ -236,26 +232,6 @@ InferInput::SetBinaryData(const bool binary_data) return Error::Success; } -Error -InferInput::ValidateData() const -{ - inference::DataType datatype = - triton::common::ProtocolStringToDataType(datatype_); - // String inputs will be checked at core and backend to reduce overhead. - if (datatype == inference::DataType::TYPE_STRING) { - return Error::Success; - } - - int64_t expected_byte_size = triton::common::GetByteSize(datatype, shape_); - if ((int64_t)byte_size_ != expected_byte_size) { - return Error( - "input '" + name_ + "' got unexpected byte size " + - std::to_string(byte_size_) + ", expected " + - std::to_string(expected_byte_size)); - } - return Error::Success; -} - Error InferInput::PrepareForRequest() { diff --git a/src/c++/library/common.h b/src/c++/library/common.h index 64717e628..8d05b966b 100644 --- a/src/c++/library/common.h +++ b/src/c++/library/common.h @@ -1,4 +1,4 @@ -// Copyright 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions @@ -354,10 +354,6 @@ class InferInput { /// \return Error object indicating success or failure. Error SetBinaryData(const bool binary_data); - /// Validate input has data and input shape matches input data. - /// \return Error object indicating success of failure. - Error ValidateData() const; - private: #ifdef TRITON_INFERENCE_SERVER_CLIENT_CLASS friend class TRITON_INFERENCE_SERVER_CLIENT_CLASS; diff --git a/src/c++/library/grpc_client.cc b/src/c++/library/grpc_client.cc index 02a1a888e..c9ee70125 100644 --- a/src/c++/library/grpc_client.cc +++ b/src/c++/library/grpc_client.cc @@ -1,4 +1,4 @@ -// Copyright 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions @@ -1470,13 +1470,7 @@ InferenceServerGrpcClient::PreRunProcessing( int index = 0; infer_request_.mutable_raw_input_contents()->Clear(); - Error err; for (const auto input : inputs) { - err = input->ValidateData(); - if (!err.IsOk()) { - return err; - } - // Add new InferInputTensor submessages only if required, otherwise // reuse the submessages already available. auto grpc_input = (infer_request_.inputs().size() <= index) diff --git a/src/c++/library/http_client.cc b/src/c++/library/http_client.cc index 728e49ab3..a2651f2eb 100644 --- a/src/c++/library/http_client.cc +++ b/src/c++/library/http_client.cc @@ -1,4 +1,4 @@ -// Copyright 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions @@ -2116,11 +2116,6 @@ InferenceServerHttpClient::PreRunProcessing( // Add the buffers holding input tensor data bool all_inputs_are_json{true}; for (const auto this_input : inputs) { - err = this_input->ValidateData(); - if (!err.IsOk()) { - return err; - } - if (this_input->BinaryData()) { all_inputs_are_json = false; } diff --git a/src/c++/tests/CMakeLists.txt b/src/c++/tests/CMakeLists.txt index 40a3f6d61..81eb74271 100644 --- a/src/c++/tests/CMakeLists.txt +++ b/src/c++/tests/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -70,9 +70,6 @@ install( RUNTIME DESTINATION bin ) -# -# cc_client_test -# add_executable( cc_client_test cc_client_test.cc @@ -92,34 +89,6 @@ install( RUNTIME DESTINATION bin ) -# -# client_input_test -# -add_executable( - client_input_test - client_input_test.cc - $ -) -target_include_directories( - client_input_test - PRIVATE - ${GTEST_INCLUDE_DIRS} -) -target_link_libraries( - client_input_test - PRIVATE - grpcclient_static - httpclient_static - gtest - ${GTEST_LIBRARY} - ${GTEST_MAIN_LIBRARY} - GTest::gmock -) -install( - TARGETS client_input_test - RUNTIME DESTINATION bin -) - endif() # TRITON_ENABLE_CC_HTTP AND TRITON_ENABLE_CC_GRPC endif() diff --git a/src/c++/tests/client_input_test.cc b/src/c++/tests/client_input_test.cc deleted file mode 100644 index 5b973f6d5..000000000 --- a/src/c++/tests/client_input_test.cc +++ /dev/null @@ -1,268 +0,0 @@ -// Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions -// are met: -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// * Neither the name of NVIDIA CORPORATION nor the names of its -// contributors may be used to endorse or promote products derived -// from this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -#include "gmock/gmock.h" -#include "grpc_client.h" -#include "gtest/gtest.h" -#include "http_client.h" -#include "shm_utils.h" - -namespace tc = triton::client; - -#define FAIL_IF_ERR(X, MSG) \ - do { \ - tc::Error err = (X); \ - if (!err.IsOk()) { \ - std::cerr << "error: " << (MSG) << ": " << err << std::endl; \ - exit(1); \ - } \ - } while (false) - -#define FAIL_IF_SUCCESS(X, MSG, ERR_MSG) \ - do { \ - tc::Error err = (X); \ - ASSERT_FALSE(err.IsOk()) << "error: " << (MSG) << ": "; \ - ASSERT_THAT(err.Message(), ::testing::HasSubstr(ERR_MSG)); \ - } while (false) - -namespace { - -template -class ClientInputTest : public ::testing::Test { - public: - ClientInputTest() : shape_{1, 16} {} - - void SetUp() override - { - std::string url; - std::string client_type; - if (std::is_same::value) { - url = "localhost:8001"; - client_type = "GRPC"; - } else if (std::is_same::value) { - url = "localhost:8000"; - client_type = "HTTP"; - } else { - ASSERT_TRUE(false) << "Unrecognized client class type '" - << typeid(ClientType).name() << "'"; - } - auto err = ClientType::Create(&this->client_, url); - ASSERT_TRUE(err.IsOk()) - << "failed to create " << client_type << " client: " << err.Message(); - - // Initialize vector input_data_ - for (size_t i = 0; i < 16; ++i) { - this->input_data_.emplace_back(i); - } - } - - std::unique_ptr client_; - std::vector input_data_; - std::vector shape_; -}; - -TYPED_TEST_SUITE_P(ClientInputTest); - -TYPED_TEST_P(ClientInputTest, AppendRaw) -{ - // Initialize the inputs with the data. - tc::InferInput* input0; - tc::InferInput* input1; - - FAIL_IF_ERR( - tc::InferInput::Create(&input0, "INPUT0", this->shape_, "INT32"), - "unable to get INPUT0"); - std::shared_ptr input0_ptr; - input0_ptr.reset(input0); - FAIL_IF_ERR( - tc::InferInput::Create(&input1, "INPUT1", this->shape_, "INT32"), - "unable to get INPUT1"); - std::shared_ptr input1_ptr; - input1_ptr.reset(input1); - - FAIL_IF_ERR( - input0_ptr->AppendRaw( - reinterpret_cast(&(this->input_data_[0])), - this->input_data_.size() * sizeof(int32_t)), - "unable to set data for INPUT0"); - FAIL_IF_ERR( - input1_ptr->AppendRaw( - reinterpret_cast(&(this->input_data_[0])), - this->input_data_.size() * sizeof(int32_t)), - "unable to set data for INPUT1"); - - // The inference settings. Will be using default for now. - tc::InferOptions options("simple"); - options.model_version_ = ""; - - std::vector inputs = {input0_ptr.get(), input1_ptr.get()}; - tc::InferResult* results; - - // Test 1 - inputs[1]->SetShape({1, 15}); - FAIL_IF_SUCCESS( - this->client_->Infer(&results, options, inputs), - "expect error with inference request", - "input 'INPUT1' got unexpected byte size 64, expected 60"); - - // Test 2 - inputs[0]->SetShape({2, 8}); - inputs[1]->SetShape({2, 8}); - // Assert the request reaches the server - FAIL_IF_SUCCESS( - this->client_->Infer(&results, options, inputs), - "expect error with inference request", - "unexpected shape for input 'INPUT1' for model 'simple'. Expected " - "[-1,16], got [2,8]"); -} - -TYPED_TEST_P(ClientInputTest, SetSharedMemory) -{ - // Unregistering all shared memory regions for a clean - // start. - FAIL_IF_ERR( - this->client_->UnregisterSystemSharedMemory(), - "unable to unregister all system shared memory regions"); - FAIL_IF_ERR( - this->client_->UnregisterCudaSharedMemory(), - "unable to unregister all cuda shared memory regions"); - - // Initialize the inputs with the data. - tc::InferInput* input0; - tc::InferInput* input1; - size_t input_byte_size = 64; - - FAIL_IF_ERR( - tc::InferInput::Create(&input0, "INPUT0", this->shape_, "INT32"), - "unable to get INPUT0"); - std::shared_ptr input0_ptr; - input0_ptr.reset(input0); - FAIL_IF_ERR( - tc::InferInput::Create(&input1, "INPUT1", this->shape_, "INT32"), - "unable to get INPUT1"); - std::shared_ptr input1_ptr; - input1_ptr.reset(input1); - - // Create Input0 and Input1 in Shared Memory. Initialize Input0 to unique - // integers and Input1 to all ones. - std::string shm_key = "/input_simple"; - int shm_fd_ip, *input0_shm; - FAIL_IF_ERR( - tc::CreateSharedMemoryRegion(shm_key, input_byte_size * 2, &shm_fd_ip), - ""); - FAIL_IF_ERR( - tc::MapSharedMemory( - shm_fd_ip, 0, input_byte_size * 2, (void**)&input0_shm), - ""); - FAIL_IF_ERR(tc::CloseSharedMemory(shm_fd_ip), ""); - int* input1_shm = (int*)(input0_shm + 16); - for (size_t i = 0; i < 16; ++i) { - *(input0_shm + i) = i; - *(input1_shm + i) = 1; - } - - FAIL_IF_ERR( - this->client_->RegisterSystemSharedMemory( - "input_data", shm_key, input_byte_size * 2), - "failed to register input shared memory region"); - - FAIL_IF_ERR( - input0_ptr->SetSharedMemory( - "input_data", input_byte_size, 0 /* offset */), - "unable to set shared memory for INPUT0"); - FAIL_IF_ERR( - input1_ptr->SetSharedMemory( - "input_data", input_byte_size, input_byte_size /* offset */), - "unable to set shared memory for INPUT1"); - - // The inference settings. Will be using default for now. - tc::InferOptions options("simple"); - options.model_version_ = ""; - - std::vector inputs = {input0_ptr.get(), input1_ptr.get()}; - tc::InferResult* results; - - // Test 1 - inputs[1]->SetShape({1, 15}); - FAIL_IF_SUCCESS( - this->client_->Infer(&results, options, inputs), - "expect error with inference request", - ("input 'INPUT1' got unexpected byte size " + - std::to_string(input_byte_size) + ", expected " + - std::to_string(input_byte_size - sizeof(int)))); - - // Test 2 - inputs[0]->SetShape({2, 8}); - inputs[1]->SetShape({2, 8}); - // Assert the request reaches the server - FAIL_IF_SUCCESS( - this->client_->Infer(&results, options, inputs), - "expect error with inference request", - "unexpected shape for input 'INPUT1' for model 'simple'. Expected " - "[-1,16], got [2,8]"); - - // Get shared memory regions active/registered within triton - using ClientType = TypeParam; - if constexpr (std::is_same< - ClientType, tc::InferenceServerGrpcClient>::value) { - inference::SystemSharedMemoryStatusResponse shm_status; - FAIL_IF_ERR( - this->client_->SystemSharedMemoryStatus(&shm_status), - "failed to get shared memory status"); - std::cout << "Shared Memory Status:\n" << shm_status.DebugString() << "\n"; - } else { - std::string shm_status; - FAIL_IF_ERR( - this->client_->SystemSharedMemoryStatus(&shm_status), - "failed to get shared memory status"); - std::cout << "Shared Memory Status:\n" << shm_status << "\n"; - } - - // Unregister shared memory - FAIL_IF_ERR( - this->client_->UnregisterSystemSharedMemory("input_data"), - "unable to unregister shared memory input region"); - - // Cleanup shared memory - FAIL_IF_ERR(tc::UnmapSharedMemory(input0_shm, input_byte_size * 2), ""); - FAIL_IF_ERR(tc::UnlinkSharedMemoryRegion("/input_simple"), ""); -} - -REGISTER_TYPED_TEST_SUITE_P(ClientInputTest, AppendRaw, SetSharedMemory); - -INSTANTIATE_TYPED_TEST_SUITE_P( - GRPC, ClientInputTest, tc::InferenceServerGrpcClient); -INSTANTIATE_TYPED_TEST_SUITE_P( - HTTP, ClientInputTest, tc::InferenceServerHttpClient); - -} // namespace - -int -main(int argc, char** argv) -{ - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -}