Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(ffi): Update IR stream protocol version handling in preparation for releasing the kv-pair IR stream format: #573

Merged
merged 10 commits into from
Nov 7, 2024
6 changes: 2 additions & 4 deletions components/core/src/clp/ffi/ir_stream/Deserializer.hpp
Original file line number Diff line number Diff line change
@@ -154,10 +154,8 @@ auto Deserializer<IrUnitHandler>::create(ReaderInterface& reader, IrUnitHandler
return std::errc::protocol_error;
}
auto const version = version_iter->get_ref<nlohmann::json::string_t&>();
// TODO: Just before the KV-pair IR format is formally released, we should replace this
// hard-coded version check with `ffi::ir_stream::validate_protocol_version`.
if (std::string_view{static_cast<char const*>(cProtocol::Metadata::BetaVersionValue)}
!= version)
if (ffi::ir_stream::IRProtocolErrorCode::Supported
!= ffi::ir_stream::validate_protocol_version(version))
{
return std::errc::protocol_not_supported;
}
2 changes: 1 addition & 1 deletion components/core/src/clp/ffi/ir_stream/Serializer.cpp
Original file line number Diff line number Diff line change
@@ -253,7 +253,7 @@ auto Serializer<encoded_variable_t>::create(
ir_buf.insert(ir_buf.cend(), cMagicNumber.begin(), cMagicNumber.end());

nlohmann::json metadata;
metadata.emplace(cProtocol::Metadata::VersionKey, cProtocol::Metadata::BetaVersionValue);
metadata.emplace(cProtocol::Metadata::VersionKey, cProtocol::Metadata::VersionValue);
metadata.emplace(cProtocol::Metadata::VariablesSchemaIdKey, cVariablesSchemaVersion);
metadata.emplace(
cProtocol::Metadata::VariableEncodingMethodsIdKey,
64 changes: 51 additions & 13 deletions components/core/src/clp/ffi/ir_stream/decoding_methods.cpp
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
#include "decoding_methods.hpp"

#include <array>
#include <regex>
#include <string_view>

#include "../../ir/types.hpp"
#include "byteswap.hpp"
@@ -468,33 +470,69 @@ IRErrorCode deserialize_preamble(
return IRErrorCode_Success;
}

IRProtocolErrorCode validate_protocol_version(std::string_view protocol_version) {
if ("v0.0.0" == protocol_version) {
// This version is hardcoded to support the oldest IR protocol version. When this version is
// no longer supported, this branch should be removed.
return IRProtocolErrorCode_Supported;
auto validate_protocol_version(std::string_view protocol_version) -> IRProtocolErrorCode {
// These versions are hardcoded to support the IR protocol version that predates the key-value
// pair IR format.
constexpr std::array<std::string_view, 3> cBackwardCompatibleVersions{
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we want to include any backward compatible beta versions here?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For old IR format no.
For key-value IR format, we do support 0.1.0-beta.1. I added a util to strip the pre-release and only compare the version core.

"v0.0.0",
"0.0.1",
"0.0.2"
};
for (auto const backward_compatible_version : cBackwardCompatibleVersions) {
if (backward_compatible_version == protocol_version) {
return IRProtocolErrorCode::Backward_Compatible;
}
}
LinZhihao-723 marked this conversation as resolved.
Show resolved Hide resolved
std::regex const protocol_version_regex{cProtocol::Metadata::VersionRegex};

std::regex const protocol_version_regex{
static_cast<char const*>(cProtocol::Metadata::VersionRegex)
};
if (false
== std::regex_match(
protocol_version.begin(),
protocol_version.end(),
protocol_version_regex
))
{
return IRProtocolErrorCode_Invalid;
return IRProtocolErrorCode::Invalid;
}
std::string_view current_build_protocol_version{cProtocol::Metadata::VersionValue};

std::string_view const current_build_protocol_version{
static_cast<char const*>(cProtocol::Metadata::VersionValue)
};
auto get_version_core = [](std::string_view version) -> std::string_view {
// Strip any pre-release version or build info from the version string based on the spec:
// https://semver.org/
auto const pos_pre_release{version.find('-')};
if (std::string_view::npos == pos_pre_release) {
return version.substr(0, version.find('+'));
}
return version.substr(0, pos_pre_release);
};
auto const curr_build_protocol_version_core{get_version_core(current_build_protocol_version)};
auto const protocol_version_core{get_version_core(protocol_version)};
if (curr_build_protocol_version_core < protocol_version_core) {
return IRProtocolErrorCode::Too_New;
}
LinZhihao-723 marked this conversation as resolved.
Show resolved Hide resolved

// Check major version
auto get_major_version{[](std::string_view version) {
return version.substr(0, version.find('.'));
}};
if (current_build_protocol_version < protocol_version) {
return IRProtocolErrorCode_Too_New;
if (get_major_version(curr_build_protocol_version_core)
> get_major_version(protocol_version_core))
{
return IRProtocolErrorCode::Too_Old;
LinZhihao-723 marked this conversation as resolved.
Show resolved Hide resolved
}
if (get_major_version(current_build_protocol_version) > get_major_version(protocol_version)) {
return IRProtocolErrorCode_Too_Old;

if (protocol_version_core < get_version_core(
static_cast<char const*>(cProtocol::Metadata::MinimumSupportedVersionValue)
))
{
return IRProtocolErrorCode::Too_Old;
}
return IRProtocolErrorCode_Supported;

LinZhihao-723 marked this conversation as resolved.
Show resolved Hide resolved
return IRProtocolErrorCode::Supported;
}

IRErrorCode deserialize_utc_offset_change(ReaderInterface& reader, UtcOffset& utc_offset) {
27 changes: 16 additions & 11 deletions components/core/src/clp/ffi/ir_stream/decoding_methods.hpp
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
#ifndef CLP_FFI_IR_STREAM_DECODING_METHODS_HPP
#define CLP_FFI_IR_STREAM_DECODING_METHODS_HPP

#include <cstdint>
#include <string>
#include <vector>

@@ -20,12 +21,13 @@ typedef enum {
IRErrorCode_Incomplete_IR,
} IRErrorCode;

typedef enum {
IRProtocolErrorCode_Supported,
IRProtocolErrorCode_Too_Old,
IRProtocolErrorCode_Too_New,
IRProtocolErrorCode_Invalid,
} IRProtocolErrorCode;
enum class IRProtocolErrorCode : uint8_t {
Supported,
Backward_Compatible,
LinZhihao-723 marked this conversation as resolved.
Show resolved Hide resolved
Too_Old,
Too_New,
Invalid,
};

class DecodingException : public TraceableException {
public:
@@ -193,15 +195,18 @@ IRErrorCode deserialize_utc_offset_change(ReaderInterface& reader, UtcOffset& ut
/**
* Validates whether the given protocol version can be supported by the current build.
* @param protocol_version
* @return IRProtocolErrorCode_Supported if the protocol version is supported.
* @return IRProtocolErrorCode_Too_Old if the protocol version is no longer supported by this
* @return IRProtocolErrorCode::Supported if the protocol version is supported.
* @return IRProtocolErrorCode::BackwardCompatible if the protocol version indicates a stream format
* that predates the key-value pair IR format.
LinZhihao-723 marked this conversation as resolved.
Show resolved Hide resolved
* @return IRProtocolErrorCode::Too_Old if the protocol version is no longer supported by this
* build's protocol version.
* @return IRProtocolErrorCode_Too_New if the protocol version is newer than this build's protocol
* @return IRProtocolErrorCode::Too_New if the protocol version is newer than this build's protocol
* version.
* @return IRProtocolErrorCode_Invalid if the protocol version does not follow the SemVer
* @return IRProtocolErrorCode::Invalid if the protocol version does not follow the SemVer
* specification.
*/
IRProtocolErrorCode validate_protocol_version(std::string_view protocol_version);
[[nodiscard]] auto validate_protocol_version(std::string_view protocol_version
) -> IRProtocolErrorCode;

namespace eight_byte_encoding {
/**
3 changes: 2 additions & 1 deletion components/core/src/clp/ffi/ir_stream/encoding_methods.cpp
Original file line number Diff line number Diff line change
@@ -98,7 +98,8 @@ static void add_base_metadata_fields(
string_view time_zone_id,
nlohmann::json& metadata
) {
metadata[cProtocol::Metadata::VersionKey] = cProtocol::Metadata::VersionValue;
metadata[cProtocol::Metadata::VersionKey]
= cProtocol::Metadata::LatestBackwardCompatibleVersionValue;
metadata[cProtocol::Metadata::VariablesSchemaIdKey] = cVariablesSchemaVersion;
metadata[cProtocol::Metadata::VariableEncodingMethodsIdKey] = cVariableEncodingMethodsVersion;
metadata[cProtocol::Metadata::TimestampPatternKey] = timestamp_pattern;
6 changes: 4 additions & 2 deletions components/core/src/clp/ffi/ir_stream/protocol_constants.hpp
Original file line number Diff line number Diff line change
@@ -12,8 +12,10 @@ constexpr int8_t LengthUByte = 0x11;
constexpr int8_t LengthUShort = 0x12;

constexpr char VersionKey[] = "VERSION";
constexpr char VersionValue[] = "0.0.2";
constexpr char BetaVersionValue[] = "0.1.0-beta.1";
constexpr char VersionValue[] = "0.1.0";
constexpr char MinimumSupportedVersionValue[] = "0.1.0";
// This is used for IR stream format that predates the key-value pair IR format.
LinZhihao-723 marked this conversation as resolved.
Show resolved Hide resolved
constexpr char LatestBackwardCompatibleVersionValue[] = "0.0.2";

// The following regex can be used to validate a Semantic Versioning string. The source of the
// regex can be found here: https://semver.org/
2 changes: 1 addition & 1 deletion components/core/src/clp/ir/LogEventDeserializer.cpp
Original file line number Diff line number Diff line change
@@ -42,7 +42,7 @@ auto LogEventDeserializer<encoded_variable_t>::create(ReaderInterface& reader
return std::errc::protocol_error;
}
auto metadata_version = version_iter->get_ref<nlohmann::json::string_t&>();
if (ffi::ir_stream::IRProtocolErrorCode_Supported
if (ffi::ir_stream::IRProtocolErrorCode::Backward_Compatible
!= ffi::ir_stream::validate_protocol_version(metadata_version))
{
return std::errc::protocol_not_supported;
78 changes: 62 additions & 16 deletions components/core/tests/test-ir_encoding_methods.cpp
Original file line number Diff line number Diff line change
@@ -630,8 +630,8 @@ TEMPLATE_TEST_CASE(
auto metadata_json = nlohmann::json::parse(json_metadata);
std::string const version
= metadata_json.at(clp::ffi::ir_stream::cProtocol::Metadata::VersionKey);
REQUIRE(clp::ffi::ir_stream::IRProtocolErrorCode_Supported == validate_protocol_version(version)
);
REQUIRE(clp::ffi::ir_stream::IRProtocolErrorCode::Backward_Compatible
== validate_protocol_version(version));
REQUIRE(clp::ffi::ir_stream::cProtocol::Metadata::EncodingJson == metadata_type);
set_timestamp_info(metadata_json, ts_info);
REQUIRE(timestamp_pattern_syntax == ts_info.timestamp_pattern_syntax);
@@ -844,17 +844,63 @@ TEST_CASE("decode_next_message_four_byte_timestamp_delta", "[ffi][deserialize_lo
}

TEST_CASE("validate_protocol_version", "[ffi][validate_version_protocol]") {
REQUIRE(clp::ffi::ir_stream::IRProtocolErrorCode_Invalid == validate_protocol_version("v0.0.1")
);
REQUIRE(clp::ffi::ir_stream::IRProtocolErrorCode_Invalid == validate_protocol_version("0.1"));
REQUIRE(clp::ffi::ir_stream::IRProtocolErrorCode_Invalid == validate_protocol_version("0.a.1"));

REQUIRE(clp::ffi::ir_stream::IRProtocolErrorCode_Too_New
== validate_protocol_version("1000.0.0"));
REQUIRE(clp::ffi::ir_stream::IRProtocolErrorCode_Supported
== validate_protocol_version(clp::ffi::ir_stream::cProtocol::Metadata::VersionValue));
REQUIRE(clp::ffi::ir_stream::IRProtocolErrorCode_Supported
== validate_protocol_version("v0.0.0"));
SECTION("Test invalid versions") {
auto const invalid_versions{GENERATE(
std::string_view{"v0.0.1"},
std::string_view{"0.1"},
std::string_view{"0.1.a"},
std::string_view{"0.a.1"}
)};
REQUIRE(
(clp::ffi::ir_stream::IRProtocolErrorCode::Invalid
== validate_protocol_version(invalid_versions))
);
}

SECTION("Test backward compatible versions") {
auto const backward_compatible_versions{GENERATE(
std::string_view{"v0.0.0"},
std::string_view{"0.0.1"},
std::string_view{"0.0.2"}
)};
REQUIRE(
(clp::ffi::ir_stream::IRProtocolErrorCode::Backward_Compatible
== validate_protocol_version(backward_compatible_versions))
);
}

SECTION("Test supported versions") {
auto const supported_versions{GENERATE(
std::string_view{"0.1.0-beta.1"},
std::string_view{static_cast<char const*>(
clp::ffi::ir_stream::cProtocol::Metadata::VersionValue
)}
)};
REQUIRE(
(clp::ffi::ir_stream::IRProtocolErrorCode::Supported
== validate_protocol_version(supported_versions))
);
}

SECTION("Test version too new") {
auto const old_versions{
GENERATE(std::string_view{"0.0.3"}, std::string_view{"0.0.3-beta.1"})
};
REQUIRE(
(clp::ffi::ir_stream::IRProtocolErrorCode::Too_Old
== validate_protocol_version(old_versions))
);
}
LinZhihao-723 marked this conversation as resolved.
Show resolved Hide resolved

SECTION("Test version too new") {
LinZhihao-723 marked this conversation as resolved.
Show resolved Hide resolved
auto const new_versions{
GENERATE(std::string_view{"10000.0.0"}, std::string_view{"0.10000.0"})
};
REQUIRE(
(clp::ffi::ir_stream::IRProtocolErrorCode::Too_New
== validate_protocol_version(new_versions))
);
}
}

TEMPLATE_TEST_CASE(
@@ -905,8 +951,8 @@ TEMPLATE_TEST_CASE(
string_view json_metadata{json_metadata_ptr, metadata_size};
auto metadata_json = nlohmann::json::parse(json_metadata);
string const version = metadata_json.at(clp::ffi::ir_stream::cProtocol::Metadata::VersionKey);
REQUIRE(clp::ffi::ir_stream::IRProtocolErrorCode_Supported == validate_protocol_version(version)
);
REQUIRE(clp::ffi::ir_stream::IRProtocolErrorCode::Backward_Compatible
== validate_protocol_version(version));
REQUIRE(clp::ffi::ir_stream::cProtocol::Metadata::EncodingJson == metadata_type);
set_timestamp_info(metadata_json, ts_info);
REQUIRE(timestamp_pattern_syntax == ts_info.timestamp_pattern_syntax);
@@ -1055,7 +1101,7 @@ TEMPLATE_TEST_CASE(
nlohmann::json expected_metadata;
expected_metadata.emplace(
clp::ffi::ir_stream::cProtocol::Metadata::VersionKey,
clp::ffi::ir_stream::cProtocol::Metadata::BetaVersionValue
clp::ffi::ir_stream::cProtocol::Metadata::VersionValue
);
expected_metadata.emplace(
clp::ffi::ir_stream::cProtocol::Metadata::VariablesSchemaIdKey,