From 9375e6747b1c69501a92db8b1683ec77a5bdf372 Mon Sep 17 00:00:00 2001 From: kpavlov00 Date: Fri, 5 Jul 2024 21:31:02 +0300 Subject: [PATCH 001/629] feat ydb: add use subscribable futures with native ydb clients doc tip 0673236dd5f07b46e1ae07400ba85738c73b2b45 --- ydb/include/userver/ydb/component.hpp | 3 +++ ydb/include/userver/ydb/coordination.hpp | 4 ++++ ydb/include/userver/ydb/table.hpp | 5 +++++ ydb/include/userver/ydb/topic.hpp | 6 ++++++ 4 files changed, 18 insertions(+) diff --git a/ydb/include/userver/ydb/component.hpp b/ydb/include/userver/ydb/component.hpp index 8f313ec3e1f7..93ad96b09e96 100644 --- a/ydb/include/userver/ydb/component.hpp +++ b/ydb/include/userver/ydb/component.hpp @@ -85,6 +85,9 @@ class YdbComponent final : public components::ComponentBase { /// Get native driver /// @param dbname database name from static config key + /// @warning Use with care! Facilities from + /// `` can help with + /// non-blocking wait operations. const NYdb::TDriver& GetNativeDriver(const std::string& dbname) const; /// Get database path diff --git a/ydb/include/userver/ydb/coordination.hpp b/ydb/include/userver/ydb/coordination.hpp index 6db302ce651c..e77d7e0a5e31 100644 --- a/ydb/include/userver/ydb/coordination.hpp +++ b/ydb/include/userver/ydb/coordination.hpp @@ -80,6 +80,10 @@ class CoordinationClient final { NYdb::NCoordination::TNodeDescription DescribeNode(std::string_view path); + /// Get native coordination client + /// @warning Use with care! Facilities from + /// `` can help with + /// non-blocking wait operations. NYdb::NCoordination::TClient& GetNativeCoordinationClient(); private: diff --git a/ydb/include/userver/ydb/table.hpp b/ydb/include/userver/ydb/table.hpp index 2a44ffdfae87..37cb6f9e5401 100644 --- a/ydb/include/userver/ydb/table.hpp +++ b/ydb/include/userver/ydb/table.hpp @@ -147,7 +147,12 @@ class TableClient final { const TableClient& table_client); /// @endcond + /// Get native table client + /// @warning Use with care! Facilities from + /// `` can help with + /// non-blocking wait operations. NYdb::NTable::TTableClient& GetNativeTableClient(); + utils::RetryBudget& GetRetryBudget(); private: diff --git a/ydb/include/userver/ydb/topic.hpp b/ydb/include/userver/ydb/topic.hpp index 8e50d00c6213..e14dd8ae786e 100644 --- a/ydb/include/userver/ydb/topic.hpp +++ b/ydb/include/userver/ydb/topic.hpp @@ -49,6 +49,9 @@ class TopicReadSession final { bool Close(std::chrono::milliseconds timeout); /// Get native read session + /// @warning Use with care! Facilities from + /// `` can help with + /// non-blocking wait operations. std::shared_ptr GetNativeTopicReadSession(); private: @@ -82,6 +85,9 @@ class TopicClient final { const NYdb::NTopic::TReadSessionSettings& settings); /// Get native topic client + /// @warning Use with care! Facilities from + /// `` can help with + /// non-blocking wait operations. NYdb::NTopic::TTopicClient& GetNativeTopicClient(); private: From 05f774ab2a3765f40773ba01d3d7430624dfb266 Mon Sep 17 00:00:00 2001 From: antonyzhilin Date: Mon, 8 Jul 2024 13:54:55 +0300 Subject: [PATCH 002/629] fix cache: account for skipped updates in failed-updates-before-expiration This fixes an interaction between two CachingComponentBase options. Once the number of updates specified in `failed-updates-before-expiration` is skipped via `updates-enabled: false`, the cache is set to `nullptr`, just as if the cache would have failed to update. 0ff5ad0807b664cdb066db537ead5cab214a059c --- core/include/userver/cache/cache_config.hpp | 14 ++--- core/src/cache/cache_update_trait_impl.cpp | 29 ++++++---- core/src/cache/cache_update_trait_impl.hpp | 2 + core/src/cache/cache_update_trait_test.cpp | 59 +++++++++++++++++++-- 4 files changed, 82 insertions(+), 22 deletions(-) diff --git a/core/include/userver/cache/cache_config.hpp b/core/include/userver/cache/cache_config.hpp index de94592f0f73..d045ad4ec92c 100644 --- a/core/include/userver/cache/cache_config.hpp +++ b/core/include/userver/cache/cache_config.hpp @@ -47,13 +47,13 @@ FirstUpdateType Parse(const yaml_config::YamlConfig& config, std::string_view ToString(FirstUpdateType); struct ConfigPatch final { - std::chrono::milliseconds update_interval; - std::chrono::milliseconds update_jitter; - std::chrono::milliseconds full_update_interval; - std::chrono::milliseconds full_update_jitter; - std::optional exception_interval; - bool updates_enabled; - std::uint64_t alert_on_failing_to_update_times; + std::chrono::milliseconds update_interval{}; + std::chrono::milliseconds update_jitter{}; + std::chrono::milliseconds full_update_interval{}; + std::chrono::milliseconds full_update_jitter{}; + std::optional exception_interval{}; + bool updates_enabled{true}; + std::uint64_t alert_on_failing_to_update_times{0}; }; ConfigPatch Parse(const formats::json::Value& value, diff --git a/core/src/cache/cache_update_trait_impl.cpp b/core/src/cache/cache_update_trait_impl.cpp index b9d31078de0b..1cc56bbfd60d 100644 --- a/core/src/cache/cache_update_trait_impl.cpp +++ b/core/src/cache/cache_update_trait_impl.cpp @@ -282,10 +282,10 @@ rcu::ReadablePtr CacheUpdateTrait::Impl::GetConfig() const { UpdateType CacheUpdateTrait::Impl::NextUpdateType(const Config& config) { if (dump_first_update_type_) { return *dump_first_update_type_; - } else if (last_update_ == dump::TimePoint{}) { + } + if (last_update_ == dump::TimePoint{}) { return UpdateType::kFull; } - if (force_full_update_) { return UpdateType::kFull; } @@ -326,6 +326,10 @@ void CacheUpdateTrait::Impl::DoPeriodicUpdate() { if (!config->updates_enabled && (!is_first_update || static_config_.allow_first_update_failure)) { LOG_INFO() << "Periodic updates are disabled for cache " << Name(); + OnUpdateSkipped(); + // TODO should use `exception_period` for the next sleep to make sure that + // it takes the same amount of time to MarkAsExpired in case of failed + // updates and in case of skipped updates. return; } @@ -342,6 +346,19 @@ void CacheUpdateTrait::Impl::DoPeriodicUpdate() { } void CacheUpdateTrait::Impl::OnUpdateFailure(const Config& config) { + OnUpdateSkipped(); + + if (config.alert_on_failing_to_update_times != 0 && + failed_updates_counter_ >= config.alert_on_failing_to_update_times) { + alerts_storage_.FireAlert( + "cache_update_error", + fmt::format("cache '{}' hasn't been updated for {} times", Name(), + failed_updates_counter_), + alerts::kInfinity); + } +} + +void CacheUpdateTrait::Impl::OnUpdateSkipped() { const auto failed_updates_before_expiration = static_config_.failed_updates_before_expiration; failed_updates_counter_++; @@ -352,14 +369,6 @@ void CacheUpdateTrait::Impl::OnUpdateFailure(const Config& config) { "failed updates has reached 'failed-updates-before-expiration' (" << failed_updates_before_expiration << ")"; } - if (config.alert_on_failing_to_update_times != 0 && - failed_updates_counter_ >= config.alert_on_failing_to_update_times) { - alerts_storage_.FireAlert( - "cache_update_error", - fmt::format("cache '{}' hasn't been updated for {} times", Name(), - failed_updates_counter_), - alerts::kInfinity); - } } void CacheUpdateTrait::Impl::AssertPeriodicUpdateStarted() { diff --git a/core/src/cache/cache_update_trait_impl.hpp b/core/src/cache/cache_update_trait_impl.hpp index 65723583cf08..cd2d7a4e4e73 100644 --- a/core/src/cache/cache_update_trait_impl.hpp +++ b/core/src/cache/cache_update_trait_impl.hpp @@ -85,6 +85,8 @@ class CacheUpdateTrait::Impl final { void OnUpdateFailure(const Config& config); + void OnUpdateSkipped(); + // Throws if `Update` throws void DoUpdate(UpdateType type, const Config& config); void CheckUpdateState(impl::UpdateState update_state, diff --git a/core/src/cache/cache_update_trait_test.cpp b/core/src/cache/cache_update_trait_test.cpp index 00992d90e6ed..d0d778c941cd 100644 --- a/core/src/cache/cache_update_trait_test.cpp +++ b/core/src/cache/cache_update_trait_test.cpp @@ -614,6 +614,8 @@ class ExpirableCache : public cache::CacheMockBase { const auto& GetExpiredLog() const { return expired_log_; } + bool IsExpired() const { return is_expired_; } + private: void Update(cache::UpdateType /*type*/, const std::chrono::system_clock::time_point& /*last_update*/, @@ -629,6 +631,7 @@ class ExpirableCache : public cache::CacheMockBase { void MarkAsExpired() override { is_expired_ = true; } std::function is_update_failed_; + // There is only 1 TaskProcessor thread in these tests, so no need to sync. std::vector expired_log_; bool is_expired_ = false; }; @@ -645,6 +648,20 @@ first-update-fail-ok: true {}}; } +void SetExpirableUpdatesEnabled(dynamic_config::StorageMock& config_storage, + bool updates_enabled) { + cache::ConfigPatch config; + config.update_interval = std::chrono::milliseconds{1}; + config.updates_enabled = updates_enabled; + + config_storage.Extend({ + {cache::kCacheConfigSet, + { + {std::string{ExpirableCache::kName}, config}, + }}, + }); +} + } // namespace UTEST(ExpirableCacheUpdateTrait, TwoFailed) { @@ -656,17 +673,49 @@ UTEST(ExpirableCacheUpdateTrait, TwoFailed) { return std::count(failed.begin(), failed.end(), i); }); - while (cache.GetExpiredLog().size() < 13) { + const std::vector expected{false, false, false, false, true, true, false, + false, false, false, true, true, false}; + + while (cache.GetExpiredLog().size() < expected.size()) { engine::Yield(); } - const auto& actual = cache.GetExpiredLog(); - const std::vector expected{false, false, false, false, true, - true, false, false, false, false, - true, true, false}; + auto actual = cache.GetExpiredLog(); + actual.resize(expected.size()); EXPECT_EQ(actual, expected); } +UTEST(ExpirableCacheUpdateTrait, UpdatesDisabled) { + auto config = MakeExpirableCacheConfig(2); + cache::MockEnvironment environment( + testsuite::impl::PeriodicUpdatesMode::kEnabled); + ExpirableCache cache(config, environment, [&environment](auto i) -> bool { + if (i == 3) { + SetExpirableUpdatesEnabled(environment.config_storage, false); + } + // Update always succeeds. + return false; + }); + + while (!cache.IsExpired()) { + engine::Yield(); + } + + // The cache might attempt and skip some more updates during this time. + engine::SleepFor(std::chrono::milliseconds{10}); + + SetExpirableUpdatesEnabled(environment.config_storage, true); + + while (cache.IsExpired()) { + engine::Yield(); + } + + SUCCEED() << "The cache succeeded to drop its data after certain amount of " + "update skips (because its data is considered stale at this " + "point), then once the cache updates are enabled again, it " + "repaired itself using Update"; +} + namespace { using InvalidateBeforeStartPeriodicUpdates = From 46a32b6aa8c8bf30597dff1378a3d269f47995aa Mon Sep 17 00:00:00 2001 From: Fedor Alekseev Date: Mon, 8 Jul 2024 14:04:35 +0300 Subject: [PATCH 003/629] fix docs: correct mobile header view MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix [the appearance of the burger menu in the mobile version of the documentation site](https://github.com/userver-framework/userver/issues/633): * fix burger menu style * add github link in mobile version

Tests: протестировано CI 274c1cdbf47f15f295a8b8002300f18080b00304 Pull Request resolved: https://github.com/userver-framework/userver/pull/639 --- scripts/docs/customdoxygen.css | 37 +++++++++++++++++----------------- scripts/docs/footer.html | 4 ++-- 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/scripts/docs/customdoxygen.css b/scripts/docs/customdoxygen.css index 5243415870c0..545ff5bec996 100644 --- a/scripts/docs/customdoxygen.css +++ b/scripts/docs/customdoxygen.css @@ -474,7 +474,7 @@ a.titlelink:hover { /* Burger menu */ .main-menu-btn { - margin-left: 7px; + margin-left: 5px; } .main-menu-btn-icon, @@ -635,6 +635,19 @@ doxygen-awesome-dark-mode-toggle { transform: translate(-54px, 20px) !important; } + #links { + margin-left: 5px; + } + + #telegram_channel { + display: none; + } + + #github_header { + padding: 0; + order: 3; + } + #navbar-main-menu { display: none; position: absolute; @@ -689,8 +702,7 @@ doxygen-awesome-dark-mode-toggle { } .sm.sm-dox>li { - padding: 5px 70px; - /* TODO: adaptive width */ + padding: 5px; } .sm.sm-dox>li:first-child { @@ -702,14 +714,10 @@ doxygen-awesome-dark-mode-toggle { } .main-menu-btn { - order: 3; + order: 4; position: relative; top: 0px; } - - #links { - display: none; - } } @media (max-width: 767px) { @@ -753,8 +761,7 @@ doxygen-awesome-dark-mode-toggle { } .sm.sm-dox>li { - padding: 5px 70px; - /* TODO: adaptive width */ + padding: 5px; } .sm.sm-dox>li:first-child { @@ -823,17 +830,11 @@ doxygen-awesome-dark-mode-toggle { } .main-menu-btn { - position: relative; - top: 0px; - } - - #links { - display: none; + width: 45px; } #MSearchCloseImg { - width: 34px; - transform: translate(-2px, -6px); + width: 16px; } } diff --git a/scripts/docs/footer.html b/scripts/docs/footer.html index 394ca934b43b..04d469bbcb90 100644 --- a/scripts/docs/footer.html +++ b/scripts/docs/footer.html @@ -28,8 +28,8 @@ links.id = 'links'; links.innerHTML = ` - - + + Telegram From 383105502dc0e4d5d9addcc70080b7c28c2a3d4a Mon Sep 17 00:00:00 2001 From: iliayar Date: Mon, 8 Jul 2024 14:56:12 +0300 Subject: [PATCH 004/629] feat formats: support YAML Nodes' tags 235ea19484215e0b70e5bbaad27a11828dad74ee --- .../include/userver/formats/yaml/value.hpp | 22 +++++++- universal/src/formats/yaml/value.cpp | 2 + universal/src/formats/yaml/value_test.cpp | 52 +++++++++++++++++++ 3 files changed, 75 insertions(+), 1 deletion(-) diff --git a/universal/include/userver/formats/yaml/value.hpp b/universal/include/userver/formats/yaml/value.hpp index 7143b0172005..fba9ad7a4e26 100644 --- a/universal/include/userver/formats/yaml/value.hpp +++ b/universal/include/userver/formats/yaml/value.hpp @@ -183,10 +183,30 @@ class Value final { /// @note This method available **only** for formats::yaml::Value. int GetLine() const; + /// @brief Returns YAML tag of this node. + /// If tag is not explicitly specified, its value depends on node value. For + /// explicitly sepcified tags, its value depends on used `TAG` directives and + /// node value. There is an implicit `TAG` derictive for `!!` with prefix + /// `tag:yaml.org,2002:`. + /// + /// For example: + /// - `""` if field is null, even when tag is specified + /// - `"!"` for quoted and block strings if tag is not specified + /// - `"?"` for other fields if tag is not specified + /// - `"tag:yaml.org,2002:str"` for `!!str` tag + /// - `"!!str"` for `!` tag + /// - `"!custom_tag"` for `!custom_tag` tag, if no additional `TAG` directives + /// are specified + /// + /// For details see YAML specification. + /// @throws MemberMissingException if `this->IsMissing()`. + /// @note This method available **only** for formats::yaml::Value. + std::string_view GetTag() const; + /// @brief Returns new value that is an exact copy if the existing one /// but references different memory (a deep copy of a *this). The returned /// value is a root value with path '/'. - /// @throws MemberMissingException id `this->IsMissing()`. + /// @throws MemberMissingException if `this->IsMissing()`. Value Clone() const; /// @throw MemberMissingException if `this->IsMissing()`. diff --git a/universal/src/formats/yaml/value.cpp b/universal/src/formats/yaml/value.cpp index da4d293b2f7e..d16ccce2628c 100644 --- a/universal/src/formats/yaml/value.cpp +++ b/universal/src/formats/yaml/value.cpp @@ -254,6 +254,8 @@ int Value::GetLine() const { return IsMissing() ? -1 : GetMark(GetNative()).line; } +std::string_view Value::GetTag() const { return GetNative().Tag(); } + Value Value::Clone() const { Value v; *v.value_pimpl_ = YAML::Clone(GetNative()); diff --git a/universal/src/formats/yaml/value_test.cpp b/universal/src/formats/yaml/value_test.cpp index bd3d38276910..c19c5f7e129c 100644 --- a/universal/src/formats/yaml/value_test.cpp +++ b/universal/src/formats/yaml/value_test.cpp @@ -4,6 +4,8 @@ #include #include +#include + #include USERVER_NAMESPACE_BEGIN @@ -95,4 +97,54 @@ TEST(FormatsYaml, UserDefinedLiterals) { )yaml"_yaml); } +TEST(FormatsYaml, NodesTags) { + formats::yaml::Value yaml = formats::yaml::FromString(R"( +# Indenting matters +%TAG !example! tag:example,2024: + + field_int: 10 + field_string: bar + field_bool: false + field_array: + - 1 + field_obj: + foo: bar + field_array_json: [1, 2] + field_obj_json: {"foo": "bar"} + + field_null: null + + field_string_quote: 'bar' + field_block_string: |- + 123 + + field_custom_tag: !my_tag + foo: bar + field_custom_tag_null: !my_tag null + field_custom_tag_string: !my_tag + field_internal_tag: !!str '123' + field_tag_prefix: !example!foo 42 + )"); + + for (auto scalar_field : + {"field_int", "field_string", "field_bool", "field_array", "field_obj", + "field_array_json", "field_obj_json"}) { + EXPECT_EQ(yaml[scalar_field].GetTag(), "?") << "Field: " << scalar_field; + } + + EXPECT_EQ(yaml["field_null"].GetTag(), ""); + EXPECT_EQ(yaml["field_string_quote"].GetTag(), "!"); + EXPECT_EQ(yaml["field_block_string"].GetTag(), "!"); + + EXPECT_EQ(yaml["field_custom_tag"].GetTag(), "!my_tag"); + // NOTE: Old versions of yaml-cpp "erase" explicit tag for null values + // 0.5.3 -- Empty string "" + // 0.8.0 -- Specified tag "!my_tag" + EXPECT_THAT(yaml["field_custom_tag_null"].GetTag(), + testing::AnyOf(testing::Eq(""), testing::Eq("!my_tag"))); + EXPECT_EQ(yaml["field_custom_tag_string"].GetTag(), "!my_tag"); + EXPECT_EQ(yaml["field_internal_tag"].GetTag(), "tag:yaml.org,2002:str"); + EXPECT_EQ(yaml["field_tag_prefix"].GetTag(), "tag:example,2024:foo"); +} + USERVER_NAMESPACE_END From 8074e48ebc63f4994a7621a258dc0e218822e7e7 Mon Sep 17 00:00:00 2001 From: kpavlov00 Date: Mon, 8 Jul 2024 15:33:22 +0300 Subject: [PATCH 005/629] feat ydb: ydb.md initial 543d28723916d2c31d6d0ba67184019cbe11ee58 --- .mapping.json | 1 + scripts/docs/en/index.md | 4 ++++ scripts/docs/en/userver/kafka.md | 4 ++-- scripts/docs/en/userver/mongodb.md | 2 +- scripts/docs/en/userver/ydb.md | 24 ++++++++++++++++++++++++ 5 files changed, 32 insertions(+), 3 deletions(-) create mode 100644 scripts/docs/en/userver/ydb.md diff --git a/.mapping.json b/.mapping.json index de2423eaa761..1437c40d84f7 100644 --- a/.mapping.json +++ b/.mapping.json @@ -3145,6 +3145,7 @@ "scripts/docs/en/userver/tutorial/tcp_full.md":"taxi/uservices/userver/scripts/docs/en/userver/tutorial/tcp_full.md", "scripts/docs/en/userver/tutorial/tcp_service.md":"taxi/uservices/userver/scripts/docs/en/userver/tutorial/tcp_service.md", "scripts/docs/en/userver/tutorial/websocket_service.md":"taxi/uservices/userver/scripts/docs/en/userver/tutorial/websocket_service.md", + "scripts/docs/en/userver/ydb.md":"taxi/uservices/userver/scripts/docs/en/userver/ydb.md", "scripts/docs/fontello/README.txt":"taxi/uservices/userver/scripts/docs/fontello/README.txt", "scripts/docs/fontello/config.json":"taxi/uservices/userver/scripts/docs/fontello/config.json", "scripts/docs/fontello/css/animation.css":"taxi/uservices/userver/scripts/docs/fontello/css/animation.css", diff --git a/scripts/docs/en/index.md b/scripts/docs/en/index.md index 4ee78431cc9a..17aba270f271 100644 --- a/scripts/docs/en/index.md +++ b/scripts/docs/en/index.md @@ -130,6 +130,10 @@ are available at the * @ref scripts/docs/en/userver/kafka.md +## YDB +* @ref scripts/docs/en/userver/ydb.md + + ## Non relational databases * @ref scripts/docs/en/userver/mongodb.md * @ref scripts/docs/en/userver/redis.md diff --git a/scripts/docs/en/userver/kafka.md b/scripts/docs/en/userver/kafka.md index 79df44f0283c..b6ae60cbdcc7 100644 --- a/scripts/docs/en/userver/kafka.md +++ b/scripts/docs/en/userver/kafka.md @@ -46,6 +46,6 @@ making the message polling non-blocking and leading to better library scalabilit ---------- @htmlonly
@endhtmlonly -⇦ @ref scripts/docs/en/userver/pg_user_types.md | -@ref scripts/docs/en/userver/mongodb.md ⇨ +⇦ @ref scripts/docs/en/userver/mysql/design_and_details.md | +@ref scripts/docs/en/userver/ydb.md ⇨ @htmlonly
@endhtmlonly diff --git a/scripts/docs/en/userver/mongodb.md b/scripts/docs/en/userver/mongodb.md index 717b4aa79128..8b81fb47ff55 100644 --- a/scripts/docs/en/userver/mongodb.md +++ b/scripts/docs/en/userver/mongodb.md @@ -101,5 +101,5 @@ requests and helps Mongo to return to the normal state with lower response timin ---------- @htmlonly
@endhtmlonly -⇦ @ref scripts/docs/en/userver/kafka.md | @ref scripts/docs/en/userver/redis.md ⇨ +⇦ @ref scripts/docs/en/userver/ydb.md | @ref scripts/docs/en/userver/redis.md ⇨ @htmlonly
@endhtmlonly diff --git a/scripts/docs/en/userver/ydb.md b/scripts/docs/en/userver/ydb.md new file mode 100644 index 000000000000..6b5d18393019 --- /dev/null +++ b/scripts/docs/en/userver/ydb.md @@ -0,0 +1,24 @@ + ## YDB + +**Quality:** @ref QUALITY_TIERS "Platinum Tier". + +YDB asynchronous driver provides interface to work with Tables, Topics and Coordination Service. + +* Table (userver/ydb/table.hpp); +* Topic (userver/ydb/transaction.hpp); +* Coordination Service (userver/ydb/coordination.hpp); + +## Usage + +To use YDB you have to add the component ydb::YdbComponent and configure it according to the documentation. +From component you can access required client (ydb::TableClient, ydb::TopicClient, ydb::CoordinationClient). + +## More information +- https://ydb.tech/ + +---------- + +@htmlonly
@endhtmlonly +⇦ @ref scripts/docs/en/userver/kafka.md | +@ref scripts/docs/en/userver/mongodb.md ⇨ +@htmlonly
@endhtmlonly From 6987a8f7b332ee7a93d7da1613bce2433e39c13f Mon Sep 17 00:00:00 2001 From: kpavlov00 Date: Mon, 8 Jul 2024 17:27:38 +0300 Subject: [PATCH 006/629] feat ydb: add examples section to ydb.md d80f769d0fc8373ca5dc1488d74128610f7d2a0c --- scripts/docs/en/userver/ydb.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/docs/en/userver/ydb.md b/scripts/docs/en/userver/ydb.md index 6b5d18393019..35f7b8205c0d 100644 --- a/scripts/docs/en/userver/ydb.md +++ b/scripts/docs/en/userver/ydb.md @@ -13,6 +13,11 @@ YDB asynchronous driver provides interface to work with Tables, Topics and Coord To use YDB you have to add the component ydb::YdbComponent and configure it according to the documentation. From component you can access required client (ydb::TableClient, ydb::TopicClient, ydb::CoordinationClient). +## Examples + +[Sample YDB usage](https://github.com/userver-framework/userver/tree/develop/samples/ydb_service) +[YDB tests](https://github.com/userver-framework/userver/tree/develop/ydb/tests) + ## More information - https://ydb.tech/ From 8b8fbb8039cfa44102a5b98059ebaecef30997a0 Mon Sep 17 00:00:00 2001 From: kpavlov00 Date: Mon, 8 Jul 2024 18:14:28 +0300 Subject: [PATCH 007/629] feat ydb: upd credentials docs e77a5f3e8b1c0d035cd78897853644d657764fae --- ydb/include/userver/ydb/credentials.hpp | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/ydb/include/userver/ydb/credentials.hpp b/ydb/include/userver/ydb/credentials.hpp index d31ebbf1196f..c7512a716eea 100644 --- a/ydb/include/userver/ydb/credentials.hpp +++ b/ydb/include/userver/ydb/credentials.hpp @@ -17,10 +17,27 @@ USERVER_NAMESPACE_BEGIN namespace ydb { +// clang-format off + +/// @ingroup userver_components +/// +/// @brief Credentials provider component for creating custom credentials provider factory +/// +/// Allows use custom credentials provider implementation +/// Required if `ydb::YdbComponent` comnponent config contains `databases..credentials` +/// +/// see https://ydb.tech/docs/en/concepts/auth + +// clang-format on + class CredentialsProviderComponent : public components::ComponentBase { public: using components::ComponentBase::ComponentBase; + /// @brief Create credentials provider factory + /// + /// @param credentials credentials config (`databases..credentials` + /// from `ydb::YdbComponent` component config) virtual std::shared_ptr CreateCredentialsProviderFactory( const yaml_config::YamlConfig& credentials) const = 0; From 5f0f8484925b30a9887b8c106493349077fe5fd4 Mon Sep 17 00:00:00 2001 From: kpavlov00 Date: Mon, 8 Jul 2024 18:32:31 +0300 Subject: [PATCH 008/629] feat docs: add unit-tests section to hello service tutorial 245634bad1cf26d42c88c83f8d176c8c2d753571 --- samples/hello_service/unittests/say_hello_test.cpp | 2 ++ scripts/docs/en/userver/tutorial/hello_service.md | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/samples/hello_service/unittests/say_hello_test.cpp b/samples/hello_service/unittests/say_hello_test.cpp index 1c69b145e0fd..0a9808e49d28 100644 --- a/samples/hello_service/unittests/say_hello_test.cpp +++ b/samples/hello_service/unittests/say_hello_test.cpp @@ -1,3 +1,4 @@ +/// [Unit test] #include "say_hello.hpp" #include @@ -6,3 +7,4 @@ UTEST(SayHelloTo, Basic) { EXPECT_EQ(samples::hello::SayHelloTo("Developer"), "Hello, Developer!\n"); EXPECT_EQ(samples::hello::SayHelloTo({}), "Hello, unknown user!\n"); } +/// [Unit test] diff --git a/scripts/docs/en/userver/tutorial/hello_service.md b/scripts/docs/en/userver/tutorial/hello_service.md index 64b6b959c20d..b317748a35c0 100644 --- a/scripts/docs/en/userver/tutorial/hello_service.md +++ b/scripts/docs/en/userver/tutorial/hello_service.md @@ -132,6 +132,12 @@ $ curl 127.0.0.1:8080/hello Hello, unknown user! ``` +### Unit tests + +@ref scripts/docs/en/userver/testing.md "Unit tests" could be implemented with one of UTEST macros in the following way: + +@snippet samples/hello_service/unittests/say_hello_test.cpp Unit test + ### Functional testing @ref scripts/docs/en/userver/functional_testing.md "Functional tests" for the service could be From 3f042a72b98989d70868169530a36e0025d32ea8 Mon Sep 17 00:00:00 2001 From: kpavlov00 Date: Mon, 8 Jul 2024 19:54:27 +0300 Subject: [PATCH 009/629] feat docs: add unit-tests section to mongo/redis tutorials 1e67b4556bef0255726728825f425214e14cd0d0 --- samples/mongo_service/unittests/mongo_test.cpp | 5 ++++- samples/redis_service/unittests/redis_test.cpp | 5 ++++- scripts/docs/en/userver/tutorial/mongo_service.md | 6 ++++++ scripts/docs/en/userver/tutorial/redis_service.md | 7 +++++++ 4 files changed, 21 insertions(+), 2 deletions(-) diff --git a/samples/mongo_service/unittests/mongo_test.cpp b/samples/mongo_service/unittests/mongo_test.cpp index 8c9caec7a8eb..8f684c2a1c32 100644 --- a/samples/mongo_service/unittests/mongo_test.cpp +++ b/samples/mongo_service/unittests/mongo_test.cpp @@ -1,6 +1,8 @@ -#include #include +/// [Unit test] +#include + #include using MongoTest = storages::mongo::utest::MongoTest; @@ -11,3 +13,4 @@ UTEST_F(MongoTest, Sample) { collection.InsertOne(formats::bson::MakeDoc("x", 2)); EXPECT_EQ(1, collection.Count({})); } +/// [Unit test] diff --git a/samples/redis_service/unittests/redis_test.cpp b/samples/redis_service/unittests/redis_test.cpp index 757dad93bd7f..13f42429340f 100644 --- a/samples/redis_service/unittests/redis_test.cpp +++ b/samples/redis_service/unittests/redis_test.cpp @@ -1,6 +1,8 @@ -#include #include +/// [Unit test] +#include + using RedisTest = storages::redis::utest::RedisTest; UTEST_F(RedisTest, Sample) { @@ -12,3 +14,4 @@ UTEST_F(RedisTest, Sample) { const auto length = client->Llen("sample_list", {}).Get(); EXPECT_EQ(length, 2); } +/// [Unit test] diff --git a/scripts/docs/en/userver/tutorial/mongo_service.md b/scripts/docs/en/userver/tutorial/mongo_service.md index 71dc3a88f605..0046129a46e5 100644 --- a/scripts/docs/en/userver/tutorial/mongo_service.md +++ b/scripts/docs/en/userver/tutorial/mongo_service.md @@ -114,6 +114,12 @@ $ curl -s http://localhost:8090/v1/translations?last_update=2021-11-01T12:00:00Z } ``` +### Unit tests +@ref scripts/docs/en/userver/testing.md "Unit tests" for the service could be +implemented with one of UTEST macros in the following way: + +@snippet samples/mongo_service/unittests/mongo_test.cpp Unit test + ### Functional testing @ref scripts/docs/en/userver/functional_testing.md "Functional tests" for the service could be implemented using the testsuite. To do that you have to: diff --git a/scripts/docs/en/userver/tutorial/redis_service.md b/scripts/docs/en/userver/tutorial/redis_service.md index 4c25bf82ff4e..7a65cfa67306 100644 --- a/scripts/docs/en/userver/tutorial/redis_service.md +++ b/scripts/docs/en/userver/tutorial/redis_service.md @@ -144,6 +144,13 @@ Content-Length: 1 ``` +### Unit tests +@ref scripts/docs/en/userver/testing.md "Unit tests" for the service could be +implemented with one of UTEST macros in the following way: + +@snippet samples/redis_service/unittests/redis_test.cpp Unit test + + ### Functional testing @ref scripts/docs/en/userver/functional_testing.md "Functional tests" for the service could be implemented using the testsuite. To do that you have to: From aa0648fdf89c1bb7f68822fbb5a79ac3ec4239e6 Mon Sep 17 00:00:00 2001 From: Fedor Alekseev Date: Mon, 8 Jul 2024 20:34:49 +0300 Subject: [PATCH 010/629] feat docs: feedback forms MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implemented documentation feedback forms with the collection of metrics: * landing star rating block * documentation page like/dislike buttons * Google Firebase integration for collecting feedback metrics * there is no need for own backend for the documentation site * light/dark theme support * adaptive design (mobile/tablet support) developed by @atlz253 like/dislike buttons design by @MariaGrinchenko with the support of @apolukhin ## Demo https://github.com/userver-framework/userver/assets/29949782/6627e425-6136-4db5-a6ea-26c04640cc41 ## Landing feedback form ![127 0 0 1_5501_docs_html_index html(Desktop)](https://github.com/userver-framework/userver/assets/29949782/111811b1-029b-4419-9454-5eb0c64ae0c5) ## Documentation page feedback form ![image](https://github.com/userver-framework/userver/assets/29949782/b155a926-e23c-4bf3-93a5-e31291c5091c) Tests: протестировано CI e0128a187bb515c19534564f934aadfa7bdeb74d Pull Request resolved: https://github.com/userver-framework/userver/pull/637 --- .mapping.json | 4 + scripts/docs/customdoxygen.css | 173 ++++++++++++- scripts/docs/doxygen.conf | 4 + scripts/docs/en/landing.md | 63 ++++- scripts/docs/footer.html | 7 +- scripts/docs/header.html | 1 + scripts/docs/img/dislike.svg | 3 + scripts/docs/img/feedback.svg | 25 ++ scripts/docs/img/like.svg | 3 + scripts/docs/landing.css | 106 +++++++- scripts/docs/scripts/feedback.js | 419 +++++++++++++++++++++++++++++++ scripts/docs/scripts/toc.js | 6 - 12 files changed, 793 insertions(+), 21 deletions(-) create mode 100644 scripts/docs/img/dislike.svg create mode 100644 scripts/docs/img/feedback.svg create mode 100644 scripts/docs/img/like.svg create mode 100644 scripts/docs/scripts/feedback.js diff --git a/.mapping.json b/.mapping.json index 1437c40d84f7..708dc129ec7d 100644 --- a/.mapping.json +++ b/.mapping.json @@ -3185,10 +3185,13 @@ "scripts/docs/img/asynchronous.svg":"taxi/uservices/userver/scripts/docs/img/asynchronous.svg", "scripts/docs/img/debugging.svg":"taxi/uservices/userver/scripts/docs/img/debugging.svg", "scripts/docs/img/delivery.svg":"taxi/uservices/userver/scripts/docs/img/delivery.svg", + "scripts/docs/img/dislike.svg":"taxi/uservices/userver/scripts/docs/img/dislike.svg", "scripts/docs/img/emoji.svg":"taxi/uservices/userver/scripts/docs/img/emoji.svg", "scripts/docs/img/ep_arrow-up.svg":"taxi/uservices/userver/scripts/docs/img/ep_arrow-up.svg", "scripts/docs/img/favicon.svg":"taxi/uservices/userver/scripts/docs/img/favicon.svg", + "scripts/docs/img/feedback.svg":"taxi/uservices/userver/scripts/docs/img/feedback.svg", "scripts/docs/img/github_logo.svg":"taxi/uservices/userver/scripts/docs/img/github_logo.svg", + "scripts/docs/img/like.svg":"taxi/uservices/userver/scripts/docs/img/like.svg", "scripts/docs/img/logo.svg":"taxi/uservices/userver/scripts/docs/img/logo.svg", "scripts/docs/img/logo_green.svg":"taxi/uservices/userver/scripts/docs/img/logo_green.svg", "scripts/docs/img/logo_in_circle.png":"taxi/uservices/userver/scripts/docs/img/logo_in_circle.png", @@ -3211,6 +3214,7 @@ "scripts/docs/landing.css":"taxi/uservices/userver/scripts/docs/landing.css", "scripts/docs/layout.xml":"taxi/uservices/userver/scripts/docs/layout.xml", "scripts/docs/scripts/codeHighlight.js":"taxi/uservices/userver/scripts/docs/scripts/codeHighlight.js", + "scripts/docs/scripts/feedback.js":"taxi/uservices/userver/scripts/docs/scripts/feedback.js", "scripts/docs/scripts/header.js":"taxi/uservices/userver/scripts/docs/scripts/header.js", "scripts/docs/scripts/landing.js":"taxi/uservices/userver/scripts/docs/scripts/landing.js", "scripts/docs/scripts/search.js":"taxi/uservices/userver/scripts/docs/scripts/search.js", diff --git a/scripts/docs/customdoxygen.css b/scripts/docs/customdoxygen.css index 545ff5bec996..f8065191895f 100644 --- a/scripts/docs/customdoxygen.css +++ b/scripts/docs/customdoxygen.css @@ -39,10 +39,10 @@ html { font-family: 'fontello'; src: url('fontello.eot?53870425'); src: url('fontello.eot?53870425#iefix') format('embedded-opentype'), - url('fontello.woff2?53870425') format('woff2'), - url('fontello.woff?53870425') format('woff'), - url('fontello.ttf?53870425') format('truetype'), - url('fontello.svg?53870425#fontello') format('svg'); + url('fontello.woff2?53870425') format('woff2'), + url('fontello.woff?53870425') format('woff'), + url('fontello.ttf?53870425') format('truetype'), + url('fontello.svg?53870425#fontello') format('svg'); font-weight: normal; font-style: normal; } @@ -383,7 +383,8 @@ html.dark-mode .sm.sm-dox li:last-child ul li a { width: 100%; } -.MSearchBoxInactive::before, .MSearchBoxInactive::after { +.MSearchBoxInactive::before, +.MSearchBoxInactive::after { display: block; position: absolute; top: 7px; @@ -620,7 +621,8 @@ doxygen-awesome-dark-mode-toggle { width: 390px; } - .MSearchBoxInactive::before, .MSearchBoxInactive::after { + .MSearchBoxInactive::before, + .MSearchBoxInactive::after { display: none; } @@ -798,7 +800,8 @@ doxygen-awesome-dark-mode-toggle { width: 100%; } - .MSearchBoxInactive::before, .MSearchBoxInactive::after { + .MSearchBoxInactive::before, + .MSearchBoxInactive::after { display: none; } @@ -929,6 +932,7 @@ html.dark-mode .btn-title { } /* Table of contents */ + div.toc li.level1 { margin-left: 0px; } @@ -959,6 +963,8 @@ div.contents .toc { box-sizing: border-box; min-width: 300px; padding: 16px 16px; + top: 97px; + max-height: calc(100vh - 255px); } div.toc li.level1:first-child { @@ -1011,6 +1017,157 @@ div.toc li a.active { } } +/* Page feedback */ + +page-feedback { + position: absolute; + top: 170px; + right: 30px; +} + +page-feedback>input { + -webkit-appearance: none; + appearance: none; + box-shadow: none; + margin: 0; + border: 0; + background-color: transparent; + cursor: pointer; + padding: 0; + background-color: var(--page-foreground-color); + width: 20px; + height: 20px; + mask-size: contain; + -webkit-mask-size: contain; + mask-repeat: no-repeat; + -webkit-mask-repeat: no-repeat; + mask-position: center; + -webkit-mask-position: center; + transition: filter 0.3s; +} + +page-feedback>input:not(:first-child) { + margin-left: 8px; +} + +page-feedback>input.like-checkbox { + mask-image: url("like.svg"); + -webkit-mask-image: url("like.svg"); +} + +page-feedback>input.like-checkbox:checked { + background-color: var(--primary-color); +} + +page-feedback>input.dislike-checkbox { + mask-image: url("dislike.svg"); + -webkit-mask-image: url("dislike.svg"); +} + +page-feedback>input.dislike-checkbox:checked { + background-color: var(--primary-color); +} + +page-feedback-popup { + opacity: 0; + background-color: var(--page-background-color); + border-radius: var(--border-radius-large); + border: 1px solid var(--separator-color); + width: 150px; + display: block; + top: 22px; + position: absolute; + padding: 10px; + box-shadow: 0 6px 10px 0 rgba(0, 0, 0, 0.1); + z-index: -1; +} + +page-feedback-popup>h6, +page-feedback-popup>p { + margin: 0; +} + +page-feedback-popup>h6 { + font-size: 16px; +} + +page-feedback-popup>p, +page-feedback-popup>a { + margin-top: 8px; +} + +page-feedback-popup>p { + font-size: 14px; + line-height: 18px !important; +} + +page-feedback-popup>a { + display: block; + font-size: 16px; + background-image: none !important; +} + +page-feedback-popup>a:hover { + filter: brightness(1.2); +} + +@media (prefers-color-scheme: dark) { + html:not(.light-mode) page-feedback-popup { + box-shadow: 0 6px 10px 0 rgba(128, 128, 128, 0.1); + } +} + +html.dark-mode page-feedback-popup { + box-shadow: 0 6px 10px 0 rgba(128, 128, 128, 0.1); +} + +@media (hover: hover) and (prefers-color-scheme: dark) { + page-feedback>input:hover { + filter: brightness(0.7); + } +} + +@media (hover: hover) { + page-feedback>input:hover { + filter: brightness(2.5); + } + + page-feedback>input:checked:hover { + filter: brightness(1.3); + } + + html.dark-mode page-feedback>input:hover { + filter: brightness(0.7); + } +} + +@media screen and (max-width: 1439px) { + page-feedback { + top: 135px; + } +} + +@media screen and (max-width: 999px) { + div.contents .toc { + top: unset; + margin-top: 15px; + } + + page-feedback { + position: relative; + right: 0px !important; + top: 0px; + display: flex; + justify-content: flex-end; + } + + /* in the mobile version, the popup is always on the right side */ + page-feedback-popup { + left: unset !important; + right: 0px; + } +} + /* Code examples */ /* More space and scrolling for code snippets */ @@ -1069,4 +1226,4 @@ pre { #nav-sync { display: none; -} +} \ No newline at end of file diff --git a/scripts/docs/doxygen.conf b/scripts/docs/doxygen.conf index d07b5a011c98..ff2b9ae4b384 100644 --- a/scripts/docs/doxygen.conf +++ b/scripts/docs/doxygen.conf @@ -1495,9 +1495,12 @@ HTML_EXTRA_FILES = scripts/docs/doxygen-awesome-css/doxygen-awesome-darkmo scripts/docs/img/asynchronous.svg \ scripts/docs/img/debugging.svg \ scripts/docs/img/delivery.svg \ + scripts/docs/img/dislike.svg \ scripts/docs/img/ep_arrow-up.svg \ scripts/docs/img/favicon.svg \ + scripts/docs/img/feedback.svg \ scripts/docs/img/github_logo.svg \ + scripts/docs/img/like.svg \ scripts/docs/img/logo_in_circle.png \ scripts/docs/img/logo_landing.svg \ scripts/docs/img/matchmaker.svg \ @@ -1516,6 +1519,7 @@ HTML_EXTRA_FILES = scripts/docs/doxygen-awesome-css/doxygen-awesome-darkmo scripts/docs/customdoxygen.css \ scripts/docs/scripts/styledBtn.js \ scripts/docs/scripts/codeHighlight.js \ + scripts/docs/scripts/feedback.js \ scripts/docs/scripts/telegramLanguage.js \ scripts/docs/scripts/search.js \ scripts/docs/scripts/toc.js \ diff --git a/scripts/docs/en/landing.md b/scripts/docs/en/landing.md index 95794762fd89..5a31928782db 100644 --- a/scripts/docs/en/landing.md +++ b/scripts/docs/en/landing.md @@ -139,6 +139,68 @@ void Ins(storages::postgres::Transaction& tr, + @@ -151,5 +213,4 @@ void Ins(storages::postgres::Transaction& tr, - \endhtmlonly diff --git a/scripts/docs/footer.html b/scripts/docs/footer.html index 04d469bbcb90..86f3f04aa599 100644 --- a/scripts/docs/footer.html +++ b/scripts/docs/footer.html @@ -15,7 +15,6 @@ $generatedby Doxygen $doxygenversion - @@ -46,17 +45,19 @@ init_header(); addLinks(); changeTelegramChannelLanguageForRussianSpeakingUser(); - document.getElementById("side-nav").style.display = "none"; // TODO: disable treeview and save sticky footer + document.getElementById("side-nav").style.display = "none"; const isLanding = document.getElementById('landing_logo_id') !== null; if (isLanding) return; + draw_toc(); highlight_code(); - styleNavButtons(); // TODO: code style? + styleNavButtons(); }, 0); }); }); + diff --git a/scripts/docs/header.html b/scripts/docs/header.html index 2f89a5c03bb9..18b6c560f130 100644 --- a/scripts/docs/header.html +++ b/scripts/docs/header.html @@ -58,6 +58,7 @@ DoxygenAwesomeDarkModeToggle.init() DoxygenAwesomeFragmentCopyButton.init() DoxygenAwesomeParagraphLink.init() + DoxygenAwesomeInteractiveToc.topOffset = 250; DoxygenAwesomeInteractiveToc.init() diff --git a/scripts/docs/img/dislike.svg b/scripts/docs/img/dislike.svg new file mode 100644 index 000000000000..9ce35cdb8140 --- /dev/null +++ b/scripts/docs/img/dislike.svg @@ -0,0 +1,3 @@ + + + diff --git a/scripts/docs/img/feedback.svg b/scripts/docs/img/feedback.svg new file mode 100644 index 000000000000..a3e3a17f7b1d --- /dev/null +++ b/scripts/docs/img/feedback.svg @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/scripts/docs/img/like.svg b/scripts/docs/img/like.svg new file mode 100644 index 000000000000..21ec4faf3345 --- /dev/null +++ b/scripts/docs/img/like.svg @@ -0,0 +1,3 @@ + + + diff --git a/scripts/docs/landing.css b/scripts/docs/landing.css index f6042a4bf97b..35f6c53bd126 100644 --- a/scripts/docs/landing.css +++ b/scripts/docs/landing.css @@ -7,13 +7,14 @@ html { --pagination-active-color: #ec700e; - --feedback-background-color: #f5f5f5; - --input-text-color: #352e48; --input-border-color: #e3e5ea; --input-focus-border-color: #7a6da0; --input-focus-stroke-color: #e2e1e6; + --feedback-star-inactive-color: #94919c; + --feedback-star-hover-color: #433d53; + --font-family: "fontello", "Roboto", sans-serif; } @@ -31,9 +32,11 @@ section { overflow-x: hidden; } -/* Disable standard Doxygen content padding */ .contents { + /* Disable standard Doxygen content behavior */ padding: 0 !important; + margin: 12px 0px 0px 0px !important; + max-width: unset !important; } .section { @@ -776,3 +779,100 @@ html.dark-mode .values__card { width: 43px; } } + +.feedback { + background-color: rgba(0, 0, 0, 0.1); + display: flex; + justify-content: center; + border-radius: var(--border-radius-large) var(--border-radius-large) 0px 0px; +} + +.feedback__wrapper { + flex: 1; + display: flex; + justify-content: space-between; + align-items: center; + padding-bottom: 24px; +} + +.feedback__info { + margin-top: 24px; +} + +.feedback__form { + margin-top: 26px; +} + +.feedback__stars { + border: 0; + display: flex; + padding: 11px 0; + width: min-content; + margin: 0; + margin-left: -3px; +} + +.feedback__stars>input { + -webkit-appearance: none; + appearance: none; + box-shadow: none; + margin: 0; + width: auto; +} + +.feedback__star::after { + content: '\2605'; + font-size: 64px; + cursor: pointer; +} + +.feedback__star:invalid::after, +.feedback__stars:hover .feedback__star:hover~.feedback__star:invalid::after, +.feedback__stars .feedback__star:focus~.feedback__star:invalid::after { + color: var(--feedback-star-inactive-color); +} + +.feedback__stars:hover .feedback__star:invalid::after, +.feedback__stars:focus-within .feedback__star:invalid::after { + color: var(--feedback-star-hover-color); +} + +.feedback__stars .feedback__star:valid { + color: var(--primary-color); +} + +.feedback__stars .feedback__star:checked~.feedback__star:not(:checked)::after { + content: '\2606'; + color: var(--feedback-star-inactive-color); +} + +.feedback__button { + margin-top: 24px; + display: block; + width: max-content; + opacity: 0; + transition: background-color 0.3s, border-color 0.3s, color 0.3s, opacity 0.5s; +} + +.feedback__stars:valid + .feedback__button { + opacity: 1; +} + +@media (max-width: 767px) { + .feedback__wrapper { + padding-top: 24px; + } + + .feedback__stars { + padding: 8px 0; + } + + .feedback__star::after { + content: '\2605'; + font-size: 54px; + } + + .feedback__image { + height: 150px; + } +} \ No newline at end of file diff --git a/scripts/docs/scripts/feedback.js b/scripts/docs/scripts/feedback.js new file mode 100644 index 000000000000..de12c3d973f8 --- /dev/null +++ b/scripts/docs/scripts/feedback.js @@ -0,0 +1,419 @@ +import { initializeApp } from "https://www.gstatic.com/firebasejs/10.12.2/firebase-app.js"; +import { + getDatabase, + ref, + runTransaction, +} from "https://www.gstatic.com/firebasejs/10.12.2/firebase-database.js"; + +const FEEDBACK_FORM_LINK = + "https://forms.yandex.ru/surveys/13485403.23fb10f432e2daeedfd3026ba6e618701a1832b4/"; + +const firebaseConfig = Object.freeze({ + apiKey: "AIzaSyDZVAx6BhwsFIvTe0JbBP9lf8VULw7Co6s", + authDomain: "userver-test-74c30.firebaseapp.com", + projectId: "userver-test-74c30", + storageBucket: "userver-test-74c30.appspot.com", + messagingSenderId: "949186490366", + appId: "1:949186490366:web:838a0823848dcb5a69daaf", + measurementId: "G-PSNRVCVKCL", +}); +const firebaseApp = initializeApp(firebaseConfig); + +const firebasePageFeedbackActions = Object.freeze({ + like: "like", + dislike: "dislike", +}); + +export class PageFeedback extends HTMLElement { + #likeCheckbox = this.#makeFeedbackCheckbox({ + className: "like-checkbox", + title: "Helpful", + }); + #dislikeCheckbox = this.#makeFeedbackCheckbox({ + className: "dislike-checkbox", + title: "Not helpful", + }); + + #popup = document.createElement("page-feedback-popup"); + + #docContentElement = document.getElementById("doc-content"); + #contentsElement = document.querySelector(".contents"); + #resizeObserver = new ResizeObserver(this.#updatePosition.bind(this)); + + #title = document.querySelector(".title"); + + get #pageTitle() { + return this.#title.textContent.replace(/[.$#[\]/]/g, "-"); + } + + static init() { + const toc = document.querySelector(".toc"); + + if (toc === null) { + return; + } else { + toc.parentNode.insertBefore(document.createElement("page-feedback"), toc); + } + } + + constructor() { + super(); + this.#addEventListeners(); + } + + #addEventListeners() { + this.#likeCheckbox.addEventListener( + "change", + this.#getCheckboxChangeCallback({ + checkbox: this.#likeCheckbox, + oppositeCheckbox: this.#dislikeCheckbox, + modalOffset: -152, + title: "Thank you!", + actionName: firebasePageFeedbackActions.like, + }) + ); + this.#dislikeCheckbox.addEventListener( + "change", + this.#getCheckboxChangeCallback({ + checkbox: this.#dislikeCheckbox, + oppositeCheckbox: this.#likeCheckbox, + modalOffset: -125, + title: "Tell us what's wrong", + actionName: firebasePageFeedbackActions.dislike, + }) + ); + } + + #getCheckboxChangeCallback({ + checkbox, + oppositeCheckbox, + modalOffset, + title, + actionName, + }) { + return () => { + if (checkbox.checked) { + this.#handleCheckedAction({ + oppositeCheckbox, + modalOffset, + title, + actionName, + }); + } else if (!oppositeCheckbox.checked) { + this.#handleUncheckedAction(actionName); + } + }; + } + + #handleCheckedAction({ oppositeCheckbox, modalOffset, title, actionName }) { + this.#popup.open({ modalOffset, title }); + if (oppositeCheckbox.checked) { + this.#switchFirebaseDatabaseFeedbackField(actionName); + } else { + FirebasePageDatabase.incrementField(this.#pageTitle, actionName); + } + oppositeCheckbox.checked = false; + } + + #switchFirebaseDatabaseFeedbackField(field) { + const { like: likeField, dislike: dislikeField } = + firebasePageFeedbackActions; + const oppositeField = field === likeField ? dislikeField : likeField; + FirebasePageDatabase.incrementField(this.#pageTitle, field); + FirebasePageDatabase.decrementField(this.#pageTitle, oppositeField); + } + + #handleUncheckedAction(actionName) { + FirebasePageDatabase.decrementField(this.#pageTitle, actionName); + this.#popup.close(); + } + + connectedCallback() { + this.appendChild(this.#likeCheckbox); + this.appendChild(this.#dislikeCheckbox); + this.appendChild(this.#popup); + + this.#resizeObserver.observe(document.body); + } + + disconnectedCallback() { + this.#resizeObserver.unobserve(document.body); + } + + #makeFeedbackCheckbox({ className, title }) { + const checkbox = document.createElement("input"); + checkbox.type = "checkbox"; + checkbox.title = title; + checkbox.className = className; + return checkbox; + } + + #updatePosition() { + const marginRight = parseFloat( + window.getComputedStyle(this.#contentsElement).marginRight + ); + const paddingRight = parseFloat( + window.getComputedStyle(this.#contentsElement).paddingRight + ); + const scrollbarWidth = + this.#docContentElement.offsetWidth - this.#docContentElement.clientWidth; + this.style.right = `${marginRight + paddingRight + scrollbarWidth}px`; + } +} + +class PageFeedbackPopup extends HTMLElement { + static #inactiveCloseTimeMilliseconds = 5000; + + #title = document.createElement("h6"); + #paragraph = this.#makeParagraph( + "Your opinion will help to improve our service" + ); + #link = this.#makeFeedbackLink(FEEDBACK_FORM_LINK); + + #currentFadeOutAnimation = null; + + #inactivityCloseTimerID = null; + #clickOutsidePopupHandlerCallback = null; + + get #isOpen() { + return this.style.opacity >= 1; + } + + constructor() { + super(); + this.#addEventListeners(); + } + + #addEventListeners() { + this.addEventListener( + "mouseenter", + this.#removeInactivityCloseTimer.bind(this) + ); + this.addEventListener( + "mouseleave", + this.#startInactivityCloseTimer.bind(this) + ); + } + + connectedCallback() { + this.appendChild(this.#title); + this.appendChild(this.#paragraph); + this.appendChild(this.#link); + } + + async open({ modalOffset, title }) { + if (this.#isOpen) { + await this.close(); + } + + this.#title.textContent = title; + this.style.left = `${modalOffset}px`; + await this.#fadeIn(); + this.#startInactivityCloseTimer(); + this.#addClickOutsidePopupCloseHandler(); + } + + #startInactivityCloseTimer() { + if (this.#inactivityCloseTimerID === null) { + this.#inactivityCloseTimerID = setInterval( + this.close.bind(this), + PageFeedbackPopup.#inactiveCloseTimeMilliseconds + ); + } else { + return; + } + } + + #addClickOutsidePopupCloseHandler() { + this.#clickOutsidePopupHandlerCallback = (event) => { + const elementsUnderCursor = document.elementsFromPoint( + event.clientX, + event.clientY + ); + const isPopupUnderCursor = elementsUnderCursor.some( + (element) => element === this + ); + if (!isPopupUnderCursor) { + this.close(); + } + }; + + document.addEventListener("click", this.#clickOutsidePopupHandlerCallback); + } + + async #fadeIn() { + return new Promise((resolve) => { + let opacity = 0; + this.style.zIndex = 10; + const timer = setInterval(() => { + if (opacity < 1) { + opacity += 0.05; + this.style.opacity = opacity; + } else { + this.style.opacity = 1; + clearInterval(timer); + resolve(); + } + }, 6); + }); + } + + async close() { + this.#removeInactivityCloseTimer(); + this.#removeClickOutsidePopupCloseHandler(); + await this.#fadeOut(); + } + + #removeInactivityCloseTimer() { + clearInterval(this.#inactivityCloseTimerID); + this.#inactivityCloseTimerID = null; + } + + #removeClickOutsidePopupCloseHandler() { + document.removeEventListener( + "click", + this.#clickOutsidePopupHandlerCallback + ); + this.#clickOutsidePopupHandlerCallback = null; + } + + async #fadeOut() { + if (this.#currentFadeOutAnimation === null) { + return this.#startFadeOutAnimation(); + } else { + return this.#currentFadeOutAnimation; + } + } + + #startFadeOutAnimation() { + this.#currentFadeOutAnimation = new Promise((resolve) => { + let opacity = this.style.opacity; + const timer = setInterval(() => { + if (opacity > 0) { + opacity -= 0.05; + this.style.opacity = opacity; + } else { + this.style.zIndex = 0; + this.style.opacity = 0; + clearInterval(timer); + this.#currentFadeOutAnimation = null; + resolve(); + } + }, 6); + }); + + return this.#currentFadeOutAnimation; + } + + #makeParagraph(text) { + const paragraph = document.createElement("p"); + paragraph.textContent = text; + return paragraph; + } + + #makeFeedbackLink(href) { + const link = document.createElement("a"); + link.href = href; + link.target = "_blank"; + link.textContent = "Leave a feedback >"; + link.title = "Fill out the feedback form"; + return link; + } +} + +class LandingFeedback { + static #lastStarRatingLabel = null; + + static init() { + this.#updateFormLinkInFeedbackButton(); + this.#addStarFeedbackMetricSendOnChangeEventListeners(); + } + + static #updateFormLinkInFeedbackButton() { + const feedbackButton = document.querySelector(".feedback__button"); + + if (feedbackButton !== null) { + feedbackButton.href = FEEDBACK_FORM_LINK; + } + } + + static #addStarFeedbackMetricSendOnChangeEventListeners() { + const starFeedbackRadios = document.querySelectorAll(".feedback__star"); + + starFeedbackRadios.forEach((star) => { + star.addEventListener("change", () => { + if (this.#lastStarRatingLabel !== null) { + FirebasePageDatabase.decrementField( + "Landing", + this.#lastStarRatingLabel + ); + } + + FirebasePageDatabase.incrementField("Landing", star.ariaLabel); + this.#lastStarRatingLabel = star.ariaLabel; + }); + }); + } +} + +class FirebasePageDatabase { + static incrementField(pageTitle, field) { + this.#doAction({ + callback: (page) => this.#incrementPageField(page, field), + pageTitle: pageTitle, + }); + } + + static #incrementPageField(page, field) { + if (page[field] === undefined) { + page[field] = 1; + } else { + page[field]++; + } + } + + static decrementField(pageTitle, field) { + this.#doAction({ + callback: (page) => this.#decrementPageField(page, field), + pageTitle: pageTitle, + }); + } + + static #decrementPageField(page, field) { + if (page[field] === undefined) { + page[field] = 0; + } else { + page[field]--; + } + } + + static #doAction({ callback, pageTitle }) { + const db = getDatabase(firebaseApp); + const pageRef = ref(db, `feedback/${pageTitle}`); + + runTransaction(pageRef, (page) => { + if (page === null) { + page = {}; + } + callback(page); + return page; + }); + } +} + +customElements.define("page-feedback", PageFeedback); +customElements.define("page-feedback-popup", PageFeedbackPopup); + +$(function () { + $(document).ready(function () { + setTimeout(() => { + const isLanding = document.getElementById("landing_logo_id") !== null; + + if (isLanding) { + LandingFeedback.init(); + } else { + PageFeedback.init(); + } + }, 0); + }); +}); diff --git a/scripts/docs/scripts/toc.js b/scripts/docs/scripts/toc.js index 3bd7732dcef7..c0429c38aee1 100644 --- a/scripts/docs/scripts/toc.js +++ b/scripts/docs/scripts/toc.js @@ -94,9 +94,3 @@ function draw_toc() { ]) .insertAfter("#MSearchResultsWindow"); } - -window.addEventListener("load", () => { - if (document.getElementById("landing_logo_id") === null) { - draw_toc(); - } -}); From 4120453c13e00838d23d43b6e8d8de078fa8e561 Mon Sep 17 00:00:00 2001 From: antoshkka Date: Tue, 9 Jul 2024 15:19:38 +0300 Subject: [PATCH 011/629] feat docs: cleanup menue, update docs build instructions and check for doxygen version MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tests: протестировано CI 060896a650ecaa4e2be3a53b696af7a2b031258b --- Makefile | 14 ++++++++++++++ scripts/docs/README.md | 4 ++-- scripts/docs/layout.xml | 8 ++++---- 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index ab5944ec9344..43b9ab51e711 100644 --- a/Makefile +++ b/Makefile @@ -29,6 +29,20 @@ gen: .PHONY: docs docs: @rm -rf docs/* + @$(DOXYGEN) --version >/dev/null 2>&1 || { \ + echo "!!! No Doxygen found."; \ + exit 2; \ + } + @{ \ + DOXYGEN_VERSION_MIN="1.10.0" && \ + DOXYGEN_VERSION_CUR=$$($(DOXYGEN) --version | awk -F " " '{print $$1}') && \ + DOXYGEN_VERSION_VALID=$$(printf "%s\n%s\n" "$$DOXYGEN_VERSION_MIN" "$$DOXYGEN_VERSION_CUR" | sort -C && echo 0 || echo 1) && \ + if [ "$$DOXYGEN_VERSION_VALID" != "0" ]; then \ + echo "!!! Doxygen expected version is $$DOXYGEN_VERSION_MIN, but $$DOXYGEN_VERSION_CUR found."; \ + echo "!!! See userver/scripts/docs/README.md"; \ + exit 2; \ + fi \ + } @( \ cat scripts/docs/doxygen.conf; \ echo OUTPUT_DIRECTORY=docs \ diff --git a/scripts/docs/README.md b/scripts/docs/README.md index a019e853632d..73d4959ecf72 100644 --- a/scripts/docs/README.md +++ b/scripts/docs/README.md @@ -9,8 +9,8 @@ The build was tested on Ubuntu 22.04 (and Mac OS Ventura 13.5 for development pu ## Instruction 🧾 1. install dependencies: `sudo apt install make graphviz` -2. install doxygen 1.10.0+: `wget https://www.doxygen.nl/files/doxygen-1.10.0.linux.bin.tar.gz && tar -xvzf doxygen-1.10.0.linux.bin.tar.gz && cd doxygen-1.10.0/ && sudo make install` -3. in project folder run: `make docs` +2. download doxygen 1.10.0+: `wget https://www.doxygen.nl/files/doxygen-1.10.0.linux.bin.tar.gz && tar -xvzf doxygen-1.10.0.linux.bin.tar.gz` +3. in project folder run: `make docs DOXYGEN=/PATH_TO/doxygen-1.10.0/bin/doxygen` P.S. Do not be afraid of the huge number of errors at the beginning of the build 🙃 diff --git a/scripts/docs/layout.xml b/scripts/docs/layout.xml index a1d60b5d746b..d75193066454 100644 --- a/scripts/docs/layout.xml +++ b/scripts/docs/layout.xml @@ -6,13 +6,13 @@ - + - - + + - + From cebfc5e3a389b8c66b9b31120efd868f6c734f0c Mon Sep 17 00:00:00 2001 From: antoshkka Date: Tue, 9 Jul 2024 15:24:22 +0300 Subject: [PATCH 012/629] bug core: remove stale events on Poller::Reset() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tests: протестировано локально и в CI fb64f91a4708f0f2ef9e7b343223b056dfe5424b --- core/src/engine/io/poller.cpp | 5 ++++ core/src/engine/io/poller_test.cpp | 46 ++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/core/src/engine/io/poller.cpp b/core/src/engine/io/poller.cpp index fc9447d66364..aaf1a68ed807 100644 --- a/core/src/engine/io/poller.cpp +++ b/core/src/engine/io/poller.cpp @@ -136,6 +136,11 @@ void Poller::Reset() { RemoveImpl(watcher); } watchers_->clear(); + + Event ignore_stale_event; + while (event_consumer_.PopNoblock(ignore_stale_event)) { + // do nothing + } } template diff --git a/core/src/engine/io/poller_test.cpp b/core/src/engine/io/poller_test.cpp index 52a8d9d88303..ec35b92f7838 100644 --- a/core/src/engine/io/poller_test.cpp +++ b/core/src/engine/io/poller_test.cpp @@ -327,4 +327,50 @@ UTEST(Poller, Remove) { Poller::Status::kNoEvents); } +UTEST(Poller, Reset) { + Pipe pipe; + Poller poller; + poller.Add(pipe.In(), Poller::Event::kRead); + + Poller::Event event{}; + poller.Reset(); + WriteOne(pipe.Out()); + ASSERT_EQ(poller.NextEventNoblock(event), Poller::Status::kNoEvents); + ReadOne(pipe.In()); + + poller.Reset(); + poller.Add(pipe.In(), Poller::Event::kRead); + WriteOne(pipe.Out()); + ASSERT_EQ( + poller.NextEvent(event, engine::Deadline::FromDuration(kReadTimeout)), + Poller::Status::kSuccess); + EXPECT_EQ(event.type, Poller::Event::kRead); + EXPECT_EQ(event.fd, pipe.In()); + ReadOne(pipe.In()); +} + +UTEST(Poller, ResetActiveEvent) { + Pipe pipe; + Poller poller; + Poller::Event event{}; + + WriteOne(pipe.Out()); + for (unsigned i = 0; i < 50000; ++i) { + poller.Add(pipe.In(), Poller::Event::kRead); + poller.Reset(); + ASSERT_EQ(poller.NextEventNoblock(event), Poller::Status::kNoEvents); + } + ReadOne(pipe.In()); + + poller.Add(pipe.In(), Poller::Event::kRead); + WriteOne(pipe.Out()); + ASSERT_EQ( + poller.NextEvent(event, engine::Deadline::FromDuration(kReadTimeout)), + Poller::Status::kSuccess); + EXPECT_EQ(event.type, Poller::Event::kRead); + EXPECT_EQ(event.fd, pipe.In()); + ReadOne(pipe.In()); + poller.Reset(); +} + USERVER_NAMESPACE_END From ab5840ee3b42264338e83534408c6e079499cf9f Mon Sep 17 00:00:00 2001 From: zaharov-ps Date: Tue, 9 Jul 2024 17:58:41 +0300 Subject: [PATCH 013/629] fix clickhouse: fix UInt16 column bug Tests: utests 1e5badb5b0041331c19a2d9ccbf8486ffe70d7c5 --- .mapping.json | 7 +++ .../clickhouse/io/columns/uint16_column.cpp | 2 +- .../src/storages/tests/int32_chtest.cpp | 48 ++++++++++++++++++ .../src/storages/tests/int64_chtest.cpp | 49 +++++++++++++++++++ clickhouse/src/storages/tests/int8_chtest.cpp | 48 ++++++++++++++++++ .../src/storages/tests/uint16_chtest.cpp | 48 ++++++++++++++++++ .../src/storages/tests/uint32_chtest.cpp | 48 ++++++++++++++++++ .../src/storages/tests/uint64_chtest.cpp | 48 ++++++++++++++++++ .../src/storages/tests/uint8_chtest.cpp | 48 ++++++++++++++++++ 9 files changed, 345 insertions(+), 1 deletion(-) create mode 100644 clickhouse/src/storages/tests/int32_chtest.cpp create mode 100644 clickhouse/src/storages/tests/int64_chtest.cpp create mode 100644 clickhouse/src/storages/tests/int8_chtest.cpp create mode 100644 clickhouse/src/storages/tests/uint16_chtest.cpp create mode 100644 clickhouse/src/storages/tests/uint32_chtest.cpp create mode 100644 clickhouse/src/storages/tests/uint64_chtest.cpp create mode 100644 clickhouse/src/storages/tests/uint8_chtest.cpp diff --git a/.mapping.json b/.mapping.json index 708dc129ec7d..b7158b3b509c 100644 --- a/.mapping.json +++ b/.mapping.json @@ -276,10 +276,17 @@ "clickhouse/src/storages/tests/execute_chtest.cpp":"taxi/uservices/userver/clickhouse/src/storages/tests/execute_chtest.cpp", "clickhouse/src/storages/tests/float32_chtest.cpp":"taxi/uservices/userver/clickhouse/src/storages/tests/float32_chtest.cpp", "clickhouse/src/storages/tests/float64_chtest.cpp":"taxi/uservices/userver/clickhouse/src/storages/tests/float64_chtest.cpp", + "clickhouse/src/storages/tests/int32_chtest.cpp":"taxi/uservices/userver/clickhouse/src/storages/tests/int32_chtest.cpp", + "clickhouse/src/storages/tests/int64_chtest.cpp":"taxi/uservices/userver/clickhouse/src/storages/tests/int64_chtest.cpp", + "clickhouse/src/storages/tests/int8_chtest.cpp":"taxi/uservices/userver/clickhouse/src/storages/tests/int8_chtest.cpp", "clickhouse/src/storages/tests/iterator_test.cpp":"taxi/uservices/userver/clickhouse/src/storages/tests/iterator_test.cpp", "clickhouse/src/storages/tests/metrics_chtest.cpp":"taxi/uservices/userver/clickhouse/src/storages/tests/metrics_chtest.cpp", "clickhouse/src/storages/tests/misc_chtest.cpp":"taxi/uservices/userver/clickhouse/src/storages/tests/misc_chtest.cpp", "clickhouse/src/storages/tests/nullable_chtest.cpp":"taxi/uservices/userver/clickhouse/src/storages/tests/nullable_chtest.cpp", + "clickhouse/src/storages/tests/uint16_chtest.cpp":"taxi/uservices/userver/clickhouse/src/storages/tests/uint16_chtest.cpp", + "clickhouse/src/storages/tests/uint32_chtest.cpp":"taxi/uservices/userver/clickhouse/src/storages/tests/uint32_chtest.cpp", + "clickhouse/src/storages/tests/uint64_chtest.cpp":"taxi/uservices/userver/clickhouse/src/storages/tests/uint64_chtest.cpp", + "clickhouse/src/storages/tests/uint8_chtest.cpp":"taxi/uservices/userver/clickhouse/src/storages/tests/uint8_chtest.cpp", "clickhouse/src/storages/tests/utils_test.cpp":"taxi/uservices/userver/clickhouse/src/storages/tests/utils_test.cpp", "clickhouse/src/storages/tests/utils_test.hpp":"taxi/uservices/userver/clickhouse/src/storages/tests/utils_test.hpp", "clickhouse/src/storages/tests/uuid_chtest.cpp":"taxi/uservices/userver/clickhouse/src/storages/tests/uuid_chtest.cpp", diff --git a/clickhouse/src/storages/clickhouse/io/columns/uint16_column.cpp b/clickhouse/src/storages/clickhouse/io/columns/uint16_column.cpp index 2ba327698f96..e056e33f6e6e 100644 --- a/clickhouse/src/storages/clickhouse/io/columns/uint16_column.cpp +++ b/clickhouse/src/storages/clickhouse/io/columns/uint16_column.cpp @@ -7,7 +7,7 @@ USERVER_NAMESPACE_BEGIN namespace storages::clickhouse::io::columns { namespace { -using NativeType = clickhouse::impl::clickhouse_cpp::ColumnUInt8; +using NativeType = clickhouse::impl::clickhouse_cpp::ColumnUInt16; } UInt16Column::UInt16Column(ColumnRef column) diff --git a/clickhouse/src/storages/tests/int32_chtest.cpp b/clickhouse/src/storages/tests/int32_chtest.cpp new file mode 100644 index 000000000000..7f69e1297306 --- /dev/null +++ b/clickhouse/src/storages/tests/int32_chtest.cpp @@ -0,0 +1,48 @@ +#include + +#include + +#include +#include + +#include "utils_test.hpp" + +USERVER_NAMESPACE_BEGIN + +namespace { + +struct DataWithInts final { + std::vector ints; +}; + +} // namespace + +namespace storages::clickhouse::io { + +template <> +struct CppToClickhouse { + using mapped_type = std::tuple; +}; + +} // namespace storages::clickhouse::io + +UTEST(Int32, InsertSelect) { + ClusterWrapper cluster{}; + cluster->Execute( + "CREATE TEMPORARY TABLE IF NOT EXISTS tmp_table " + "(value Int32)"); + + const DataWithInts insert_data{{std::numeric_limits::min(), -1, 0, 1, + std::numeric_limits::max()}}; + cluster->Insert("tmp_table", {"value"}, insert_data); + + const auto select_data = + cluster->Execute("SELECT * from tmp_table").As(); + ASSERT_EQ(select_data.ints.size(), 5); + + for (size_t i = 0; i < insert_data.ints.size(); ++i) { + ASSERT_EQ(insert_data.ints[i], select_data.ints[i]); + } +} + +USERVER_NAMESPACE_END diff --git a/clickhouse/src/storages/tests/int64_chtest.cpp b/clickhouse/src/storages/tests/int64_chtest.cpp new file mode 100644 index 000000000000..70af931462a8 --- /dev/null +++ b/clickhouse/src/storages/tests/int64_chtest.cpp @@ -0,0 +1,49 @@ +#include + +#include +#include + +#include +#include + +#include "utils_test.hpp" + +USERVER_NAMESPACE_BEGIN + +namespace { + +struct DataWithInts final { + std::vector ints; +}; + +} // namespace + +namespace storages::clickhouse::io { + +template <> +struct CppToClickhouse { + using mapped_type = std::tuple; +}; + +} // namespace storages::clickhouse::io + +UTEST(Int64, InsertSelect) { + ClusterWrapper cluster{}; + cluster->Execute( + "CREATE TEMPORARY TABLE IF NOT EXISTS tmp_table " + "(value Int64)"); + + const DataWithInts insert_data{{std::numeric_limits::min(), -1, 0, 1, + std::numeric_limits::max()}}; + cluster->Insert("tmp_table", {"value"}, insert_data); + + const auto select_data = + cluster->Execute("SELECT * from tmp_table").As(); + ASSERT_EQ(select_data.ints.size(), 5); + + for (size_t i = 0; i < insert_data.ints.size(); ++i) { + ASSERT_EQ(insert_data.ints[i], select_data.ints[i]); + } +} + +USERVER_NAMESPACE_END diff --git a/clickhouse/src/storages/tests/int8_chtest.cpp b/clickhouse/src/storages/tests/int8_chtest.cpp new file mode 100644 index 000000000000..2c322b6741f7 --- /dev/null +++ b/clickhouse/src/storages/tests/int8_chtest.cpp @@ -0,0 +1,48 @@ +#include + +#include + +#include +#include + +#include "utils_test.hpp" + +USERVER_NAMESPACE_BEGIN + +namespace { + +struct DataWithInts final { + std::vector ints; +}; + +} // namespace + +namespace storages::clickhouse::io { + +template <> +struct CppToClickhouse { + using mapped_type = std::tuple; +}; + +} // namespace storages::clickhouse::io + +UTEST(Int8, InsertSelect) { + ClusterWrapper cluster{}; + cluster->Execute( + "CREATE TEMPORARY TABLE IF NOT EXISTS tmp_table " + "(value Int8)"); + + const DataWithInts insert_data{{std::numeric_limits::min(), -1, 0, 1, + std::numeric_limits::max()}}; + cluster->Insert("tmp_table", {"value"}, insert_data); + + const auto select_data = + cluster->Execute("SELECT * from tmp_table").As(); + ASSERT_EQ(select_data.ints.size(), 5); + + for (size_t i = 0; i < insert_data.ints.size(); ++i) { + ASSERT_EQ(insert_data.ints[i], select_data.ints[i]); + } +} + +USERVER_NAMESPACE_END diff --git a/clickhouse/src/storages/tests/uint16_chtest.cpp b/clickhouse/src/storages/tests/uint16_chtest.cpp new file mode 100644 index 000000000000..d4de57a608e4 --- /dev/null +++ b/clickhouse/src/storages/tests/uint16_chtest.cpp @@ -0,0 +1,48 @@ +#include + +#include + +#include +#include + +#include "utils_test.hpp" + +USERVER_NAMESPACE_BEGIN + +namespace { + +struct DataWithUInts final { + std::vector ints; +}; + +} // namespace + +namespace storages::clickhouse::io { + +template <> +struct CppToClickhouse { + using mapped_type = std::tuple; +}; + +} // namespace storages::clickhouse::io + +UTEST(UInt16, InsertSelect) { + ClusterWrapper cluster{}; + cluster->Execute( + "CREATE TEMPORARY TABLE IF NOT EXISTS tmp_table " + "(value UInt16)"); + + const DataWithUInts insert_data{ + {0, 1, 10, std::numeric_limits::max()}}; + cluster->Insert("tmp_table", {"value"}, insert_data); + + const auto select_data = + cluster->Execute("SELECT * from tmp_table").As(); + ASSERT_EQ(select_data.ints.size(), 4); + + for (size_t i = 0; i < insert_data.ints.size(); ++i) { + ASSERT_EQ(insert_data.ints[i], select_data.ints[i]); + } +} + +USERVER_NAMESPACE_END diff --git a/clickhouse/src/storages/tests/uint32_chtest.cpp b/clickhouse/src/storages/tests/uint32_chtest.cpp new file mode 100644 index 000000000000..4257a3e0b73d --- /dev/null +++ b/clickhouse/src/storages/tests/uint32_chtest.cpp @@ -0,0 +1,48 @@ +#include + +#include + +#include +#include + +#include "utils_test.hpp" + +USERVER_NAMESPACE_BEGIN + +namespace { + +struct DataWithUInts final { + std::vector ints; +}; + +} // namespace + +namespace storages::clickhouse::io { + +template <> +struct CppToClickhouse { + using mapped_type = std::tuple; +}; + +} // namespace storages::clickhouse::io + +UTEST(UInt32, InsertSelect) { + ClusterWrapper cluster{}; + cluster->Execute( + "CREATE TEMPORARY TABLE IF NOT EXISTS tmp_table " + "(value UInt32)"); + + const DataWithUInts insert_data{ + {0, 1, 10, std::numeric_limits::max()}}; + cluster->Insert("tmp_table", {"value"}, insert_data); + + const auto select_data = + cluster->Execute("SELECT * from tmp_table").As(); + ASSERT_EQ(select_data.ints.size(), 4); + + for (size_t i = 0; i < insert_data.ints.size(); ++i) { + ASSERT_EQ(insert_data.ints[i], select_data.ints[i]); + } +} + +USERVER_NAMESPACE_END diff --git a/clickhouse/src/storages/tests/uint64_chtest.cpp b/clickhouse/src/storages/tests/uint64_chtest.cpp new file mode 100644 index 000000000000..4cbec930929c --- /dev/null +++ b/clickhouse/src/storages/tests/uint64_chtest.cpp @@ -0,0 +1,48 @@ +#include + +#include + +#include +#include + +#include "utils_test.hpp" + +USERVER_NAMESPACE_BEGIN + +namespace { + +struct DataWithUInts final { + std::vector ints; +}; + +} // namespace + +namespace storages::clickhouse::io { + +template <> +struct CppToClickhouse { + using mapped_type = std::tuple; +}; + +} // namespace storages::clickhouse::io + +UTEST(UInt64, InsertSelect) { + ClusterWrapper cluster{}; + cluster->Execute( + "CREATE TEMPORARY TABLE IF NOT EXISTS tmp_table " + "(value UInt64)"); + + const DataWithUInts insert_data{ + {0, 1, 10, std::numeric_limits::max()}}; + cluster->Insert("tmp_table", {"value"}, insert_data); + + const auto select_data = + cluster->Execute("SELECT * from tmp_table").As(); + ASSERT_EQ(select_data.ints.size(), 4); + + for (size_t i = 0; i < insert_data.ints.size(); ++i) { + ASSERT_EQ(insert_data.ints[i], select_data.ints[i]); + } +} + +USERVER_NAMESPACE_END diff --git a/clickhouse/src/storages/tests/uint8_chtest.cpp b/clickhouse/src/storages/tests/uint8_chtest.cpp new file mode 100644 index 000000000000..025267003441 --- /dev/null +++ b/clickhouse/src/storages/tests/uint8_chtest.cpp @@ -0,0 +1,48 @@ +#include + +#include + +#include +#include + +#include "utils_test.hpp" + +USERVER_NAMESPACE_BEGIN + +namespace { + +struct DataWithUInts final { + std::vector ints; +}; + +} // namespace + +namespace storages::clickhouse::io { + +template <> +struct CppToClickhouse { + using mapped_type = std::tuple; +}; + +} // namespace storages::clickhouse::io + +UTEST(UInt8, InsertSelect) { + ClusterWrapper cluster{}; + cluster->Execute( + "CREATE TEMPORARY TABLE IF NOT EXISTS tmp_table " + "(value UInt8)"); + + const DataWithUInts insert_data{ + {0, 1, 10, std::numeric_limits::max()}}; + cluster->Insert("tmp_table", {"value"}, insert_data); + + const auto select_data = + cluster->Execute("SELECT * from tmp_table").As(); + ASSERT_EQ(select_data.ints.size(), 4); + + for (size_t i = 0; i < insert_data.ints.size(); ++i) { + ASSERT_EQ(insert_data.ints[i], select_data.ints[i]); + } +} + +USERVER_NAMESPACE_END From bb6dcd67d273a10c35c1f8f9004f0fad8e15677a Mon Sep 17 00:00:00 2001 From: antonyzhilin Date: Wed, 10 Jul 2024 12:49:15 +0300 Subject: [PATCH 014/629] feat docs: show how to unit test ugrpc services and clients 701a6bf732956834033ddaf0c65c760e88c1a070 --- .mapping.json | 13 +- grpc/include/userver/ugrpc/client/fwd.hpp | 18 ++ samples/grpc_service/CMakeLists.txt | 31 ++- samples/grpc_service/grpc_service.cpp | 180 ------------------ samples/grpc_service/main.cpp | 30 +++ .../src/call_greeter_client_test_handler.hpp | 36 ++++ samples/grpc_service/src/greeter_client.cpp | 78 ++++++++ samples/grpc_service/src/greeter_client.hpp | 54 ++++++ samples/grpc_service/src/greeter_service.cpp | 51 +++++ samples/grpc_service/src/greeter_service.hpp | 45 +++++ samples/grpc_service/static_config.yaml | 12 +- .../{tests => testsuite}/conftest.py | 0 .../{tests => testsuite}/test_grpc.py | 0 .../unittests/greeter_service_test.cpp | 83 ++++++++ scripts/docs/en/userver/grpc.md | 9 + .../docs/en/userver/tutorial/grpc_service.md | 89 +++++++-- 16 files changed, 519 insertions(+), 210 deletions(-) create mode 100644 grpc/include/userver/ugrpc/client/fwd.hpp delete mode 100644 samples/grpc_service/grpc_service.cpp create mode 100644 samples/grpc_service/main.cpp create mode 100644 samples/grpc_service/src/call_greeter_client_test_handler.hpp create mode 100644 samples/grpc_service/src/greeter_client.cpp create mode 100644 samples/grpc_service/src/greeter_client.hpp create mode 100644 samples/grpc_service/src/greeter_service.cpp create mode 100644 samples/grpc_service/src/greeter_service.hpp rename samples/grpc_service/{tests => testsuite}/conftest.py (100%) rename samples/grpc_service/{tests => testsuite}/test_grpc.py (100%) create mode 100644 samples/grpc_service/unittests/greeter_service_test.cpp diff --git a/.mapping.json b/.mapping.json index b7158b3b509c..32842b93d978 100644 --- a/.mapping.json +++ b/.mapping.json @@ -1755,6 +1755,7 @@ "grpc/include/userver/ugrpc/client/client_factory.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/client_factory.hpp", "grpc/include/userver/ugrpc/client/client_factory_component.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/client_factory_component.hpp", "grpc/include/userver/ugrpc/client/exceptions.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/exceptions.hpp", + "grpc/include/userver/ugrpc/client/fwd.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/fwd.hpp", "grpc/include/userver/ugrpc/client/impl/async_method_invocation.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/async_method_invocation.hpp", "grpc/include/userver/ugrpc/client/impl/async_methods.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/async_methods.hpp", "grpc/include/userver/ugrpc/client/impl/call_params.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/call_params.hpp", @@ -2851,11 +2852,17 @@ "samples/grpc_middleware_service/tests/ya.make":"taxi/uservices/userver/samples/grpc_middleware_service/tests/ya.make", "samples/grpc_middleware_service/ya.make":"taxi/uservices/userver/samples/grpc_middleware_service/ya.make", "samples/grpc_service/CMakeLists.txt":"taxi/uservices/userver/samples/grpc_service/CMakeLists.txt", - "samples/grpc_service/grpc_service.cpp":"taxi/uservices/userver/samples/grpc_service/grpc_service.cpp", + "samples/grpc_service/main.cpp":"taxi/uservices/userver/samples/grpc_service/main.cpp", "samples/grpc_service/proto/samples/greeter.proto":"taxi/uservices/userver/samples/grpc_service/proto/samples/greeter.proto", + "samples/grpc_service/src/call_greeter_client_test_handler.hpp":"taxi/uservices/userver/samples/grpc_service/src/call_greeter_client_test_handler.hpp", + "samples/grpc_service/src/greeter_client.cpp":"taxi/uservices/userver/samples/grpc_service/src/greeter_client.cpp", + "samples/grpc_service/src/greeter_client.hpp":"taxi/uservices/userver/samples/grpc_service/src/greeter_client.hpp", + "samples/grpc_service/src/greeter_service.cpp":"taxi/uservices/userver/samples/grpc_service/src/greeter_service.cpp", + "samples/grpc_service/src/greeter_service.hpp":"taxi/uservices/userver/samples/grpc_service/src/greeter_service.hpp", "samples/grpc_service/static_config.yaml":"taxi/uservices/userver/samples/grpc_service/static_config.yaml", - "samples/grpc_service/tests/conftest.py":"taxi/uservices/userver/samples/grpc_service/tests/conftest.py", - "samples/grpc_service/tests/test_grpc.py":"taxi/uservices/userver/samples/grpc_service/tests/test_grpc.py", + "samples/grpc_service/testsuite/conftest.py":"taxi/uservices/userver/samples/grpc_service/testsuite/conftest.py", + "samples/grpc_service/testsuite/test_grpc.py":"taxi/uservices/userver/samples/grpc_service/testsuite/test_grpc.py", + "samples/grpc_service/unittests/greeter_service_test.cpp":"taxi/uservices/userver/samples/grpc_service/unittests/greeter_service_test.cpp", "samples/hello_service/CMakeLists.txt":"taxi/uservices/userver/samples/hello_service/CMakeLists.txt", "samples/hello_service/benchmarks/say_hello_bench.cpp":"taxi/uservices/userver/samples/hello_service/benchmarks/say_hello_bench.cpp", "samples/hello_service/main.cpp":"taxi/uservices/userver/samples/hello_service/main.cpp", diff --git a/grpc/include/userver/ugrpc/client/fwd.hpp b/grpc/include/userver/ugrpc/client/fwd.hpp new file mode 100644 index 000000000000..6cfca840f21c --- /dev/null +++ b/grpc/include/userver/ugrpc/client/fwd.hpp @@ -0,0 +1,18 @@ +#pragma once + +/// @file userver/ugrpc/client/fwd.hpp +/// @brief Forward declarations for `ugrpc::client` classes. + +USERVER_NAMESPACE_BEGIN + +namespace testsuite { +class GrpcControl; +} // namespace testsuite + +namespace ugrpc::client { + +class ClientFactory; + +} // namespace ugrpc::client + +USERVER_NAMESPACE_END diff --git a/samples/grpc_service/CMakeLists.txt b/samples/grpc_service/CMakeLists.txt index 4d2da013f4cf..a1e35f935f64 100644 --- a/samples/grpc_service/CMakeLists.txt +++ b/samples/grpc_service/CMakeLists.txt @@ -1,14 +1,35 @@ cmake_minimum_required(VERSION 3.14) project(userver-samples-grpc_service CXX) +# /// [ugrpc] find_package(userver COMPONENTS grpc REQUIRED) -add_executable(${PROJECT_NAME} grpc_service.cpp) -target_link_libraries(${PROJECT_NAME} userver::grpc) +add_library(${PROJECT_NAME}_objs OBJECT + # Note: it's nonsense to have the same client and service in the same executable. + # For test and demonstration purposes only. + src/greeter_client.cpp + src/greeter_service.cpp +) +target_link_libraries(${PROJECT_NAME}_objs PUBLIC userver::grpc) +target_include_directories(${PROJECT_NAME}_objs PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/src) +# /// [ugrpc] -# /// [gRPC sample - CMake] +# /// [add_grpc_library] userver_add_grpc_library(${PROJECT_NAME}-proto PROTOS samples/greeter.proto) -target_link_libraries(${PROJECT_NAME} ${PROJECT_NAME}-proto) -# /// [gRPC sample - CMake] +target_link_libraries(${PROJECT_NAME}_objs PUBLIC ${PROJECT_NAME}-proto) +# /// [add_grpc_library] + +add_executable(${PROJECT_NAME} main.cpp) +target_link_libraries(${PROJECT_NAME} ${PROJECT_NAME}_objs) + +# /// [gtest] +add_executable(${PROJECT_NAME}-unittest unittests/greeter_service_test.cpp) +target_link_libraries(${PROJECT_NAME}-unittest + ${PROJECT_NAME}_objs + userver::utest + userver::grpc-utest +) +add_google_tests(${PROJECT_NAME}-unittest) +# /// [gtest] userver_testsuite_add_simple() diff --git a/samples/grpc_service/grpc_service.cpp b/samples/grpc_service/grpc_service.cpp deleted file mode 100644 index 588e09693055..000000000000 --- a/samples/grpc_service/grpc_service.cpp +++ /dev/null @@ -1,180 +0,0 @@ -#include - -#include -#include -#include - -#include - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#include -#include - -namespace samples { - -/// [gRPC sample - client] -// A user-defined wrapper around api::GreeterServiceClient that provides -// a simplified interface. -class GreeterClient final : public components::ComponentBase { - public: - static constexpr std::string_view kName = "greeter-client"; - - GreeterClient(const components::ComponentConfig& config, - const components::ComponentContext& context) - : ComponentBase(config, context), - // ClientFactory is used to create gRPC clients - client_factory_( - context.FindComponent() - .GetFactory()), - // The client needs a fixed endpoint - client_(client_factory_.MakeClient( - "greeter", config["endpoint"].As())) {} - - std::string SayHello(std::string name); - - static yaml_config::Schema GetStaticConfigSchema(); - - private: - ugrpc::client::ClientFactory& client_factory_; - api::GreeterServiceClient client_; -}; -/// [gRPC sample - client] - -/// [gRPC sample - client RPC handling] -std::string GreeterClient::SayHello(std::string name) { - api::GreetingRequest request; - request.set_name(std::move(name)); - - // Deadline must be set manually for each RPC - auto context = std::make_unique(); - context->set_deadline( - engine::Deadline::FromDuration(std::chrono::seconds{20})); - - // Initiate the RPC. No actual actions have been taken thus far besides - // preparing to send the request. - auto stream = client_.SayHello(request, std::move(context)); - - // Complete the unary RPC by sending the request and receiving the response. - // The client should call `Finish` (in case of single response) or `Read` - // until `false` (in case of response stream), otherwise the RPC will be - // cancelled. - api::GreetingResponse response = stream.Finish(); - - return std::move(*response.mutable_greeting()); -} -/// [gRPC sample - client RPC handling] - -yaml_config::Schema GreeterClient::GetStaticConfigSchema() { - return yaml_config::MergeSchemas(R"( -type: object -description: > - a user-defined wrapper around api::GreeterServiceClient that provides - a simplified interface. -additionalProperties: false -properties: - endpoint: - type: string - description: > - the service endpoint (URI). We talk to our own service, - which is kind of pointless, but works for an example -)"); -} - -/// [gRPC sample - service] -class GreeterServiceComponent final - : public api::GreeterServiceBase::Component { - public: - static constexpr std::string_view kName = "greeter-service"; - - GreeterServiceComponent(const components::ComponentConfig& config, - const components::ComponentContext& context) - : api::GreeterServiceBase::Component(config, context), - prefix_(config["greeting-prefix"].As()) {} - - void SayHello(SayHelloCall& call, api::GreetingRequest&& request) override; - - static yaml_config::Schema GetStaticConfigSchema(); - - private: - const std::string prefix_; -}; -/// [gRPC sample - service] - -/// [gRPC sample - server RPC handling] -void GreeterServiceComponent::SayHello( - api::GreeterServiceBase::SayHelloCall& call, - api::GreetingRequest&& request) { - // Authentication checking could have gone here. For this example, we trust - // the world. - - api::GreetingResponse response; - response.set_greeting(fmt::format("{}, {}!", prefix_, request.name())); - - // Complete the RPC by sending the response. The service should complete - // each request by calling `Finish` or `FinishWithError`, otherwise the - // client will receive an Internal Error (500) response. - call.Finish(response); -} -/// [gRPC sample - server RPC handling] - -yaml_config::Schema GreeterServiceComponent::GetStaticConfigSchema() { - return yaml_config::MergeSchemas(R"( -type: object -description: gRPC sample greater service component -additionalProperties: false -properties: - greeting-prefix: - type: string - description: greeting prefix -)"); -} - -// Our Python tests use HTTP for all the samples, so we add an HTTP handler, -// through which we test both the client side and the server side. -class GreeterHttpHandler final : public server::handlers::HttpHandlerBase { - public: - static constexpr std::string_view kName = "greeter-http-handler"; - - GreeterHttpHandler(const components::ComponentConfig& config, - const components::ComponentContext& context) - : HttpHandlerBase(config, context), - grpc_greeter_client_(context.FindComponent()) {} - - std::string HandleRequestThrow( - const server::http::HttpRequest& request, - server::request::RequestContext&) const override { - return grpc_greeter_client_.SayHello(request.RequestBody()); - } - - private: - GreeterClient& grpc_greeter_client_; -}; - -} // namespace samples - -/// [gRPC sample - main] -int main(int argc, char* argv[]) { - const auto component_list = - /// [gRPC sample - ugrpc registration] - components::MinimalServerComponentList() - .Append() - .Append() - .Append() - /// [gRPC sample - ugrpc registration] - .Append() - .Append() - .Append(); - return utils::DaemonMain(argc, argv, component_list); -} -/// [gRPC service sample - main] diff --git a/samples/grpc_service/main.cpp b/samples/grpc_service/main.cpp new file mode 100644 index 000000000000..29f3c08c369c --- /dev/null +++ b/samples/grpc_service/main.cpp @@ -0,0 +1,30 @@ +#include +#include +#include +#include + +#include +#include + +#include +#include +#include + +/// [main] +int main(int argc, char* argv[]) { + const auto component_list = + components::MinimalServerComponentList() + .Append() + // Default client factory. You can create multiple instances of this + // component using `.Append("name")` if different gRPC clients + // require different credentials or different grpc-core options. + .Append() + // All gRPC services are registered in this component. + .Append() + // Custom components: + .Append() + .Append() + .Append(); + return utils::DaemonMain(argc, argv, component_list); +} +/// [main] diff --git a/samples/grpc_service/src/call_greeter_client_test_handler.hpp b/samples/grpc_service/src/call_greeter_client_test_handler.hpp new file mode 100644 index 000000000000..e1f701a0fe35 --- /dev/null +++ b/samples/grpc_service/src/call_greeter_client_test_handler.hpp @@ -0,0 +1,36 @@ +#pragma once + +#include +#include + +#include +#include + +#include + +namespace samples { + +// Our Python tests use HTTP for all the samples, so we add an HTTP handler, +// through which we test both the client side and the server side. +class CallGreeterClientTestHandler final + : public server::handlers::HttpHandlerBase { + public: + static constexpr std::string_view kName = "greeter-http-handler"; + + CallGreeterClientTestHandler(const components::ComponentConfig& config, + const components::ComponentContext& context) + : HttpHandlerBase(config, context), + grpc_greeter_client_( + context.FindComponent().GetClient()) {} + + std::string HandleRequestThrow( + const server::http::HttpRequest& request, + server::request::RequestContext&) const override { + return grpc_greeter_client_.SayHello(request.RequestBody()); + } + + private: + const GreeterClient& grpc_greeter_client_; +}; + +} // namespace samples diff --git a/samples/grpc_service/src/greeter_client.cpp b/samples/grpc_service/src/greeter_client.cpp new file mode 100644 index 000000000000..dcd6ea21ec81 --- /dev/null +++ b/samples/grpc_service/src/greeter_client.cpp @@ -0,0 +1,78 @@ +#include + +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace samples { + +// A user-defined wrapper around api::GreeterServiceClient that provides +// a simplified interface. +GreeterClient::GreeterClient(api::GreeterServiceClient&& raw_client) + : raw_client_(std::move(raw_client)) {} + +/// [client] +std::string GreeterClient::SayHello(std::string name) const { + api::GreetingRequest request; + request.set_name(std::move(name)); + + // Deadline must be set manually for each RPC + auto context = std::make_unique(); + context->set_deadline( + engine::Deadline::FromDuration(std::chrono::seconds{20})); + + // Initiate the RPC. No actual actions have been taken thus far besides + // preparing to send the request. + auto stream = raw_client_.SayHello(request, std::move(context)); + + // Complete the unary RPC by sending the request and receiving the response. + // The client should call `Finish` (in case of single response) or `Read` + // until `false` (in case of response stream), otherwise the RPC will be + // cancelled. + api::GreetingResponse response = stream.Finish(); + + return std::move(*response.mutable_greeting()); +} +/// [client] + +/// [component] +GreeterClientComponent::GreeterClientComponent( + const components::ComponentConfig& config, + const components::ComponentContext& context) + : ComponentBase(config, context), + // ClientFactory is used to create gRPC clients + client_factory_( + context.FindComponent() + .GetFactory()), + // The client needs a fixed endpoint + client_(client_factory_.MakeClient( + "greeter", config["endpoint"].As())) {} +/// [component] + +const GreeterClient& GreeterClientComponent::GetClient() const { + return client_; +} + +yaml_config::Schema GreeterClientComponent::GetStaticConfigSchema() { + return yaml_config::MergeSchemas(R"( +type: object +description: > + a user-defined wrapper around api::GreeterServiceClient that provides + a simplified interface. +additionalProperties: false +properties: + endpoint: + type: string + description: > + the service endpoint (URI). We talk to our own service, + which is kind of pointless, but works for an example +)"); +} + +} // namespace samples diff --git a/samples/grpc_service/src/greeter_client.hpp b/samples/grpc_service/src/greeter_client.hpp new file mode 100644 index 000000000000..a5ebc566327b --- /dev/null +++ b/samples/grpc_service/src/greeter_client.hpp @@ -0,0 +1,54 @@ +#pragma once + +#include + +#include + +/// [includes] +#include +#include +#include + +#include +/// [includes] + +namespace samples { + +/// [client] +// A user-defined wrapper around api::GreeterServiceClient that handles +// the metadata and deadline bureaucracy and provides a simplified interface. +// +// Alternatively, you can use ugrpc::client::SimpleClientComponent directly. +// +// Note that we have both service and client to that service in the same +// microservice. Ignore that, it's just for the sake of example. +class GreeterClient final { + public: + explicit GreeterClient(api::GreeterServiceClient&& raw_client); + + std::string SayHello(std::string name) const; + + private: + api::GreeterServiceClient raw_client_; +}; +/// [client] + +/// [component] +class GreeterClientComponent final : public components::ComponentBase { + public: + static constexpr std::string_view kName = "greeter-client"; + + GreeterClientComponent(const components::ComponentConfig& config, + const components::ComponentContext& context); + + const GreeterClient& GetClient() const; + + static yaml_config::Schema GetStaticConfigSchema(); + + private: + ugrpc::client::ClientFactory& client_factory_; + GreeterClient client_; +}; +/// [component] + +} // namespace samples diff --git a/samples/grpc_service/src/greeter_service.cpp b/samples/grpc_service/src/greeter_service.cpp new file mode 100644 index 000000000000..b6bd040af65d --- /dev/null +++ b/samples/grpc_service/src/greeter_service.cpp @@ -0,0 +1,51 @@ +#include + +#include + +#include +#include + +namespace samples { + +GreeterService::GreeterService(std::string prefix) + : prefix_(std::move(prefix)) {} + +/// [server RPC handling] +void GreeterService::SayHello(api::GreeterServiceBase::SayHelloCall& call, + api::GreetingRequest&& request) { + // Authentication checking could have gone here. + // Or even better, use a global gRPC authentication middleware. + + api::GreetingResponse response; + response.set_greeting(fmt::format("{}, {}!", prefix_, request.name())); + + // Complete the RPC by sending the response. The service should complete + // each request by calling `Finish` or `FinishWithError`, otherwise the + // client will receive an Internal Error (500) response. + call.Finish(response); +} +/// [server RPC handling] + +/// [service] +GreeterServiceComponent::GreeterServiceComponent( + const components::ComponentConfig& config, + const components::ComponentContext& context) + : ugrpc::server::ServiceComponentBase(config, context), + service_(config["greeting-prefix"].As()) { + RegisterService(service_); +} +/// [service] + +yaml_config::Schema GreeterServiceComponent::GetStaticConfigSchema() { + return yaml_config::MergeSchemas(R"( +type: object +description: gRPC sample greater service component +additionalProperties: false +properties: + greeting-prefix: + type: string + description: greeting prefix +)"); +} + +} // namespace samples diff --git a/samples/grpc_service/src/greeter_service.hpp b/samples/grpc_service/src/greeter_service.hpp new file mode 100644 index 000000000000..d4146697a15a --- /dev/null +++ b/samples/grpc_service/src/greeter_service.hpp @@ -0,0 +1,45 @@ +#pragma once + +#include +#include + +#include + +/// [includes] +#include +#include + +#include +/// [includes] + +namespace samples { + +/// [service] +class GreeterService final : public api::GreeterServiceBase { + public: + explicit GreeterService(std::string prefix); + + void SayHello(SayHelloCall& call, api::GreetingRequest&& request) override; + + private: + const std::string prefix_; +}; +/// [service] + +/// [component] +class GreeterServiceComponent final + : public ugrpc::server::ServiceComponentBase { + public: + static constexpr std::string_view kName = "greeter-service"; + + GreeterServiceComponent(const components::ComponentConfig& config, + const components::ComponentContext& context); + + static yaml_config::Schema GetStaticConfigSchema(); + + private: + GreeterService service_; +}; +/// [component] + +} // namespace samples diff --git a/samples/grpc_service/static_config.yaml b/samples/grpc_service/static_config.yaml index fb24b290acce..222e79d92d12 100644 --- a/samples/grpc_service/static_config.yaml +++ b/samples/grpc_service/static_config.yaml @@ -10,7 +10,7 @@ components_manager: level: debug overflow_behavior: discard -# /// [gRPC sample - static config client] +# /// [static config client] # yaml # Creates gRPC clients grpc-client-factory: @@ -26,9 +26,9 @@ components_manager: # The service endpoint (URI). We talk to our own service, # which is kind of pointless, but works for an example endpoint: '[::1]:8091' -# /// [gRPC sample - static config client] +# /// [static config client] -# /// [gRPC sample - static config server] +# /// [static config server] # yaml # Common configuration for gRPC server grpc-server: @@ -42,7 +42,7 @@ components_manager: task-processor: main-task-processor greeting-prefix: Hello middlewares: [] -# /// [gRPC sample - static config server] +# /// [static config server] # In this example, the tests still communicate with the microservice # using HTTP, so we still need an HTTP server and an HTTP handler. @@ -59,13 +59,13 @@ components_manager: default_task_processor: main-task-processor # Task processor in which components start. -# /// [gRPC sample - task processor] +# /// [task processor] # yaml task_processors: grpc-blocking-task-processor: # For blocking gRPC channel creation worker_threads: 2 thread_name: grpc-worker -# /// [gRPC sample - task processor] +# /// [task processor] main-task-processor: # For non-blocking operations worker_threads: 8 fs-task-processor: # For blocking filesystem operations diff --git a/samples/grpc_service/tests/conftest.py b/samples/grpc_service/testsuite/conftest.py similarity index 100% rename from samples/grpc_service/tests/conftest.py rename to samples/grpc_service/testsuite/conftest.py diff --git a/samples/grpc_service/tests/test_grpc.py b/samples/grpc_service/testsuite/test_grpc.py similarity index 100% rename from samples/grpc_service/tests/test_grpc.py rename to samples/grpc_service/testsuite/test_grpc.py diff --git a/samples/grpc_service/unittests/greeter_service_test.cpp b/samples/grpc_service/unittests/greeter_service_test.cpp new file mode 100644 index 000000000000..5661304e52c9 --- /dev/null +++ b/samples/grpc_service/unittests/greeter_service_test.cpp @@ -0,0 +1,83 @@ +#include + +#include +#include +#include + +#include + +#include +#include + +namespace { + +/// [service fixture] +class GreeterServiceTest : public ugrpc::tests::ServiceFixtureBase { + protected: + GreeterServiceTest() : service_(prefix_) { + RegisterService(service_); + StartServer(); + } + + ~GreeterServiceTest() override { StopServer(); } + + private: + const std::string prefix_{"Hello"}; + // We've made sure to separate the logic into samples::GreeterService that is + // detached from the component system and only depends on things obtainable + // in gtest tests. + samples::GreeterService service_; +}; +/// [service fixture] + +} // namespace + +/// [service tests] +UTEST_F(GreeterServiceTest, DirectCall) { + const auto client = MakeClient(); + + samples::api::GreetingRequest request; + request.set_name("gtest"); + const auto response = client.SayHello(request).Finish(); + + EXPECT_EQ(response.greeting(), "Hello, gtest!"); +} + +UTEST_F(GreeterServiceTest, CustomClient) { + // We've made sure to separate some logic into samples::GreeterClient that is + // detached from the component system, it only needs the gRPC client, which we + // can create in gtest tests. + const samples::GreeterClient client{ + MakeClient()}; + + const auto response = client.SayHello("gtest"); + EXPECT_EQ(response, "Hello, gtest!"); +} +/// [service tests] + +/// [client tests] +namespace { + +class GreeterMock final : public samples::api::GreeterServiceBase { + public: + void SayHello(SayHelloCall& call, + samples::api::GreetingRequest&& /*request*/) override { + samples::api::GreetingResponse response; + response.set_greeting("Mocked response"); + call.Finish(response); + } +}; + +// Default-constructs GreeterMock. +using GreeterClientTest = ugrpc::tests::ServiceFixture; + +} // namespace + +UTEST_F(GreeterClientTest, MockedServiceCustomClient) { + const samples::GreeterClient client{ + MakeClient()}; + + const auto response = client.SayHello("gtest"); + EXPECT_EQ(response, "Mocked response"); +} +/// [client tests] diff --git a/scripts/docs/en/userver/grpc.md b/scripts/docs/en/userver/grpc.md index a135b842bacb..f7bc080544d4 100644 --- a/scripts/docs/en/userver/grpc.md +++ b/scripts/docs/en/userver/grpc.md @@ -222,6 +222,15 @@ These are the metrics provided for each gRPC method: process itself, but with the infrastructure * `active` — The number of currently active RPCs (created and not finished) + +## Unit tests and benchmarks + +* @ref scripts/docs/en/userver/tutorial/grpc_service.md shows how to test + userver gRPC services and clients in gtest +* @ref ugrpc::tests::Service and ugrpc::tests::ServiceBase can be used + to benchmark userver gRPC services and clients, as well as create more + complex gtest tests with multiple services and perhaps databases. + ---------- @htmlonly
@endhtmlonly diff --git a/scripts/docs/en/userver/tutorial/grpc_service.md b/scripts/docs/en/userver/tutorial/grpc_service.md index ff0541d4ec82..203b0d95fb3b 100644 --- a/scripts/docs/en/userver/tutorial/grpc_service.md +++ b/scripts/docs/en/userver/tutorial/grpc_service.md @@ -4,55 +4,82 @@ Make sure that you can compile and run core tests and read a basic example @ref scripts/docs/en/userver/tutorial/hello_service.md. +Make sure that you understand the basic concepts of @ref scripts/docs/en/userver/grpc.md "userver grpc driver". + ## Step by step guide In this example, we will write a client side and a server side for a simple `GreeterService` from `greeter.proto` (see the schema below). Its single `SayHello` method accepts a `name` string and replies with a corresponding `greeting` string. ### Installation -We generate and link to a CMake library from our `.proto` schema: +Find and link to userver gRPC: + +@snippet samples/grpc_service/CMakeLists.txt ugrpc + +Generate and link to a CMake library from our `.proto` schema: -@snippet samples/grpc_service/CMakeLists.txt gRPC sample - CMake +@snippet samples/grpc_service/CMakeLists.txt add_grpc_library -Register the necessary `ugrpc` components: +By default, `userver_add_grpc_library` looks in `${CMAKE_CURRENT_SOURCE_DIR}/proto`, you can override this using the `SOURCE_PATH` option. -@snippet samples/grpc_service/grpc_service.cpp gRPC sample - ugrpc registration +Proto includes can be specified in `INCLUDE_DIRECTORIES` option (multiple directories can be specified). ### The client side Wrap the generated `api::GreeterServiceClient` in a component that exposes a simplified interface: -@snippet samples/grpc_service/grpc_service.cpp gRPC sample - client +@snippet samples/grpc_service/src/greeter_client.hpp includes + +@snippet samples/grpc_service/src/greeter_client.hpp client + +@snippet samples/grpc_service/src/greeter_client.hpp component + +@snippet samples/grpc_service/src/greeter_client.cpp component + +We intentionally split `GreeterClient` from `GreeterClientComponent` +to make the logic unit-testable. If you don't need gtest tests, +you can put the logic into the component directly. A single request-response RPC handling is simple: fill in `request` and `context`, initiate the RPC, receive the `response`. -@snippet samples/grpc_service/grpc_service.cpp gRPC sample - client RPC handling +@snippet samples/grpc_service/src/greeter_client.cpp client Fill in the static config entries for the client side: -@snippet samples/grpc_service/static_config.yaml gRPC sample - static config client +@snippet samples/grpc_service/static_config.yaml static config client -@snippet samples/grpc_service/static_config.yaml gRPC sample - task processor +@snippet samples/grpc_service/static_config.yaml task processor ### The server side -Implement the generated `api::GreeterServiceBase`. As a convenience, a derived `api::GreeterServiceBase::Component` class is provided for easy integration with the component system. +Implement the generated `api::GreeterServiceBase`. +As a convenience, `api::GreeterServiceBase::Component` base class is provided +that inherits from both +`api::GreeterServiceBase` and `ugrpc::server::ServiceComponentBase`. +However, for in this example we will also test our service using gtest, so we +need to split the logic from the component. + +@snippet samples/grpc_service/src/grpc_service.hpp includes + +@snippet samples/grpc_service/src/grpc_service.hpp service -@snippet samples/grpc_service/grpc_service.cpp gRPC sample - service +@snippet samples/grpc_service/src/grpc_service.hpp component + +@snippet samples/grpc_service/src/grpc_service.cpp component A single request-response RPC handling is simple: fill in the `response` and send it. -@snippet samples/grpc_service/grpc_service.cpp gRPC sample - server RPC handling +@snippet samples/grpc_service/src/grpc_service.cpp service Fill in the static config entries for the server side: -@snippet samples/grpc_service/static_config.yaml gRPC sample - static config server +@snippet samples/grpc_service/static_config.yaml static config server ### int main() Finally, we register our components and start the server. -@snippet samples/http_caching/http_caching.cpp HTTP caching sample - main +@snippet samples/grpc_service/main.cpp main ### Build and Run @@ -77,11 +104,13 @@ To start the service manually run The service is available locally at port 8091 (as per our `static_config.yaml`). -### Functional testing +### Functional testing for the sample gRPC service and client + To implement @ref scripts/docs/en/userver/functional_testing.md "Functional tests" for the service some preparational steps should be done. #### Preparations + First of all, import the required modules and add the required pytest_userver.plugins.grpc pytest plugin: @@ -115,12 +144,35 @@ Use it to do gRPC requests to the service: @snippet samples/grpc_service/tests/test_grpc.py grpc server test +### Unit testing for the sample gRPC service and client (gtest) + +First, link the unit tests to `userver::grpc-utest`: + +@snippet samples/grpc_service/CMakeLists.txt gtest + +Create a fixture that sets up the gRPC service in unit tests: + +@snippet samples/grpc_service/unittests/greeter_service_test.cpp service fixture + +Finally, we can create gRPC service and client in unit tests: + +@snippet samples/grpc_service/unittests/greeter_service_test.cpp service tests + +We can also use toy test-only gRPC services for unit tests: + +@snippet samples/grpc_service/unittests/greeter_service_test.cpp client tests + + ## Full sources See the full example at: -* @ref samples/grpc_service/grpc_service.cpp * @ref samples/grpc_service/proto/samples/greeter.proto +* @ref samples/grpc_service/src/greeter_client.hpp +* @ref samples/grpc_service/src/greeter_client.cpp +* @ref samples/grpc_service/src/greeter_service.hpp +* @ref samples/grpc_service/src/greeter_service.cpp +* @ref samples/grpc_service/main.cpp * @ref samples/grpc_service/static_config.yaml * @ref samples/grpc_service/tests/conftest.py * @ref samples/grpc_service/tests/test_grpc.py @@ -132,8 +184,13 @@ See the full example at: ⇦ @ref scripts/docs/en/userver/tutorial/flatbuf_service.md | @ref scripts/docs/en/userver/tutorial/grpc_middleware_service.md ⇨ @htmlonly
@endhtmlonly -@example samples/grpc_service/grpc_service.cpp @example samples/grpc_service/proto/samples/greeter.proto +@example samples/grpc_service/src/greeter_client.hpp +@example samples/grpc_service/src/greeter_client.cpp +@example samples/grpc_service/src/greeter_service.hpp +@example samples/grpc_service/src/greeter_service.cpp +@example samples/grpc_service/main.cpp +@example samples/grpc_service/grpc_service.cpp @example samples/grpc_service/static_config.yaml @example samples/grpc_service/tests/conftest.py @example samples/grpc_service/tests/test_grpc.py From b3696fedeb3a8137e1e97603e3bd2654517da67b Mon Sep 17 00:00:00 2001 From: segoon Date: Wed, 10 Jul 2024 16:46:15 +0300 Subject: [PATCH 015/629] docs userver: compare to POCO e53829eb818a63a1cf3ffa8de1812e9db285d3c6 --- .../docs/en/userver/framework_comparison.md | 76 ++++++++++--------- 1 file changed, 42 insertions(+), 34 deletions(-) diff --git a/scripts/docs/en/userver/framework_comparison.md b/scripts/docs/en/userver/framework_comparison.md index 53356f4dcdef..9602af0d1a4b 100644 --- a/scripts/docs/en/userver/framework_comparison.md +++ b/scripts/docs/en/userver/framework_comparison.md @@ -14,40 +14,42 @@ For missing functionality or if we found no info on the functionality we use ❌ and ❓ respectively. -| Feature | 🐙 userver | go-micro 4.7.0 | dapr 1.5.3 | actix 0.13.0 + tokio 1.19.2 | drogon 1.7.5 | -|-----------------------------------|------------------------------------------------|-------------------------|-------------------------------|------------------------|----------------------------------| -| Programming model for IO-bound apps | stackful coroutines | stackful coroutines | actors | stackless coroutines | callbacks / stackless coroutines | -| Programming language to use | С++ | Go-lang | Python, JS, .Net, PHP, Java, Go | Rust | C++ | -| Caching data from remote or DB | ✔️ @ref scripts/docs/en/userver/caches.md "[↗]" | ❌ | ❌ | ❌ | ❌ | -| Dynamic Config @ref fcmp1 "[1]" | ✔️ @ref scripts/docs/en/schemas/dynamic_configs.md "[↗]" | ✔️ [[↗]][gom-features] | ❌ | ❌ | ❌ | -| Unit testing | ✔️ C++ @ref scripts/docs/en/userver/testing.md "[↗]" | ✔️ via Go-lang | ✔️ PHP [[↗]][dapr-testig] | ✔️ | ✔️ [[↗]][drog-testig] | -| Functional Testing @ref fcmp2 "[2]" | ✔️ @ref scripts/docs/en/userver/functional_testing.md "[↗]" | ❌ | ❌ [[↗]][dapr-testig] | ❌ [[↗]][actix-test] | ❌ [[↗]][drog-testig] | -| Async synchronization primitives | ✔️ @ref scripts/docs/en/userver/synchronization.md "[↗]" | ✔️ via Go-lang | ❌ [forces turn based access][dapr-actors] | ✔️ [[↗]][tokio-sync] | ❌ | -| Dist locks | ✔️ | ✔️ [[↗]][gom-features] | ❌ [[↗]][dapr-distlock] | ± third-party libs | ❌ | -| Async HTTP client | ✔️ @ref clients::http::Client "[↗]" | ✔️ | ✔️ | ✔️ | ✔️ [[↗]][drog-http-client] | -| Async HTTP server | ✔️ @ref components::Server "[↗]" | ✔️ | ✔️ | ✔️ | ✔️ | -| Async gRPC client | ✔️ @ref scripts/docs/en/userver/grpc.md "[↗]" | ✔️ | ✔️ | ± third-party libs | ❌ | -| Async gRPC server | ✔️ @ref scripts/docs/en/userver/grpc.md "[↗]" | ✔️ | ✔️ | ± third-party libs | ❌ | -| Async PostgreSQL | ✔️ @ref pg_driver "[↗]" | ± third-party driver | ✔️ [[↗]][dapr-postgre] | ❌ [manual offloading][acti-db] | ✔️ [[↗]][drog-db] | -| PostgreSQL pipelining, binary protocol | ✔️ @ref pg_driver "[↗]" | ❌ | ❌ | ± third-party libs | ❌ | -| Async Redis | ✔️ @ref scripts/docs/en/userver/redis.md "[↗]" | ± third-party driver | ✔️ [[↗]][dapr-redis] | ± third-party libs | ✔️ [[↗]][drog-redis] | -| Async Mongo | ✔️ @ref scripts/docs/en/userver/mongodb.md "[↗]" | ± third-party driver | ✔️ [[↗]][dapr-mongo] | ❌ [manual offloading][acti-db] | ❌ [[↗]][drog-db] | -| Async ClickHouse | ✔️ @ref clickhouse_driver "[↗]" | ± third-party driver | ❌ | ± third-party libs | ❌ [[↗]][drog-db] | -| Async MySQL | ✔️ @ref mysql_driver | ± third-party driver | ✔️ [[↗]][dapr-mysql] | ❌ [[↗]][acti-db] | ✔️ [[↗]][drog-db] | -| Metrics | ✔️ @ref scripts/docs/en/userver/service_monitor.md "[↗]" | ± third-party driver | ✔️ [[↗]][dapr-configs] | ❌ | ❌ | -| No args evaluation for disabled logs | ✔️ @ref scripts/docs/en/userver/logging.md "[↗]" | ❌ | ❌ | ± third-party libs | ❌ | -| Secrets Management | ± @ref storages::secdist::SecdistConfig "[↗]" | ❓ | ✔️ | ❓ | ❓ | -| Distributed Tracing | ✔️ @ref scripts/docs/en/userver/logging.md "[↗]" | ❓ | ✔️ [[↗]][dapr-configs] | ± third-party libs | ❌ | -| JSON, BSON, YAML | ✔️ @ref scripts/docs/en/userver/formats.md "[↗]" | ± third-party libs | ± third-party libs | ± third-party libs | ± only JSON | -| Content compression/decompression | ✔️ | ✔️ | ❓ | ✔️ | ✔️ | -| Service Discovery | ✔️ DNS, DB topology discovery | ✔️ [[↗]][gom-features] | ❓ | ❓ | ❓ | -| Async TCP/UDP | ✔️ @ref engine::io::Socket "[↗]" | ✔️ | ❓ | ✔️ [[↗]][tokio-net] | ❌ | -| Async TLS Socket | ✔️ @ref engine::io::TlsWrapper "[↗]" | ✔️ | ❓ | ± third-party libs | ❌ | -| Async HTTPS client | ✔️ @ref clients::http::Client "[↗]" | ✔️ | ❓ | ✔️ | ❓ | -| Async HTTPS server | ✔️ @ref components::Server "[↗]" | ❓ | ❓ | ✔️ | ❓ | -| WebSockets Server | ✔️ @ref components::Server "[↗]" | ± third-party libs | ❌ [[↗]][dapr-websock] | ± third-party libs | ✔️ [[↗]][drogon] | -| Deadlines and Cancellations | ✔️ | ❓ | ❓ | ❓ | ± [[↗]][drog-timeout] | -| Retries and Load Balancing | ✔️ | ✔️ [[↗]][gom-features] | ✔️ | ❓ |❓ | +| Feature | 🐙 userver | go-micro 4.7.0 | dapr 1.5.3 | actix 0.13.0 + tokio 1.19.2 | drogon 1.7.5 | POCO 1.13.3 | +|-----------------------------------|------------------------------------------------|-------------------------|-------------------------------|------------------------|---------------------------------|---------------| +| Programming model for IO-bound apps | stackful coroutines | stackful coroutines | actors | stackless coroutines | callbacks / stackless coroutines | thread pool | +| Programming language to use | С++ | Go-lang | Python, JS, .Net, PHP, Java, Go | Rust | C++ | C++ | +| Caching data from remote or DB | ✔️ @ref scripts/docs/en/userver/caches.md "[↗]" | ❌ | ❌ | ❌ | ❌ | ✔️ [[↗]][poco-cache] | +| Dynamic Config @ref fcmp1 "[1]" | ✔️ @ref scripts/docs/en/schemas/dynamic_configs.md "[↗]" | ✔️ [[↗]][gom-features] | ❌ | ❌ | ❌ | ❌ | +| Unit testing | ✔️ C++ @ref scripts/docs/en/userver/testing.md "[↗]" | ✔️ via Go-lang | ✔️ PHP [[↗]][dapr-testig] | ✔️ | ✔️ [[↗]][drog-testig] | ❓ | +| Functional Testing @ref fcmp2 "[2]" | ✔️ @ref scripts/docs/en/userver/functional_testing.md "[↗]" | ❌ | ❌ [[↗]][dapr-testig] | ❌ [[↗]][actix-test] | ❌ [[↗]][drog-testig] | ❓ | +| Async synchronization primitives | ✔️ @ref scripts/docs/en/userver/synchronization.md "[↗]" | ✔️ via Go-lang | ❌ [forces turn based access][dapr-actors] | ✔️ [[↗]][tokio-sync] | ❌ | ✔️ [[↗]][poco-sync] | +| Dist locks | ✔️ | ✔️ [[↗]][gom-features] | ❌ [[↗]][dapr-distlock] | ± third-party libs | ❌ | ❓ | +| Async HTTP client | ✔️ @ref clients::http::Client "[↗]" | ✔️ | ✔️ | ✔️ | ✔️ [[↗]][drog-http-client] | ❓ | +| Async HTTP server | ✔️ @ref components::Server "[↗]" | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ [[↗]][poco-net] | +| Async gRPC client | ✔️ @ref scripts/docs/en/userver/grpc.md "[↗]" | ✔️ | ✔️ | ± third-party libs | ❌ | ❌ | +| Async gRPC server | ✔️ @ref scripts/docs/en/userver/grpc.md "[↗]" | ✔️ | ✔️ | ± third-party libs | ❌ | ❌ | +| Async PostgreSQL | ✔️ @ref pg_driver "[↗]" | ± third-party driver | ✔️ [[↗]][dapr-postgre] | ❌ [manual offloading][acti-db] | ✔️ [[↗]][drog-db] | ✔️ [[↗]][poco-db] | +| PostgreSQL pipelining, binary protocol | ✔️ @ref pg_driver "[↗]" | ❌ | ❌ | ± third-party libs | ❌ | ❓ | +| Async Redis | ✔️ @ref scripts/docs/en/userver/redis.md "[↗]" | ± third-party driver | ✔️ [[↗]][dapr-redis] | ± third-party libs | ✔️ [[↗]][drog-redis] | ❓| +| Async Mongo | ✔️ @ref scripts/docs/en/userver/mongodb.md "[↗]" | ± third-party driver | ✔️ [[↗]][dapr-mongo] | ❌ [manual offloading][acti-db] | ❌ [[↗]][drog-db] |❓| +| Async ClickHouse | ✔️ @ref clickhouse_driver "[↗]" | ± third-party driver | ❌ | ± third-party libs | ❌ [[↗]][drog-db] |❓| +| Async MySQL | ✔️ @ref mysql_driver | ± third-party driver | ✔️ [[↗]][dapr-mysql] | ❌ [[↗]][acti-db] | ✔️ [[↗]][drog-db] | ✔️ [[↗]][poco-db] | +| Async ODBC | ❌ | ❓ | ❓ |❓|❓| ✔️ [[↗]][poco-db] | +| Metrics | ✔️ @ref scripts/docs/en/userver/service_monitor.md "[↗]" | ± third-party driver | ✔️ [[↗]][dapr-configs] | ❌ | ❌ |❓| +| No args evaluation for disabled logs | ✔️ @ref scripts/docs/en/userver/logging.md "[↗]" | ❌ | ❌ | ± third-party libs | ❌ | ❌ | +| Secrets Management | ± @ref storages::secdist::SecdistConfig "[↗]" | ❓ | ✔️ | ❓ | ❓ |❓| +| Distributed Tracing | ✔️ @ref scripts/docs/en/userver/logging.md "[↗]" | ❓ | ✔️ [[↗]][dapr-configs] | ± third-party libs | ❌ |❓| +| JSON, BSON, YAML | ✔️ @ref scripts/docs/en/userver/formats.md "[↗]" | ± third-party libs | ± third-party libs | ± third-party libs | ± only JSON |❓| +| XML | ❌ | ❓ | ❓ | ❓ |❓ | ✔️ [[↗]][poco-xml] | +| Content compression/decompression | ✔️ | ✔️ | ❓ | ✔️ | ✔️ | ✔️ [[↗]][poco-streams] | +| Service Discovery | ✔️ DNS, DB topology discovery | ✔️ [[↗]][gom-features] | ❓ | ❓ | ❓ | ± [[↗]][poco-net] (DNS) | +| Async TCP/UDP | ✔️ @ref engine::io::Socket "[↗]" | ✔️ | ❓ | ✔️ [[↗]][tokio-net] | ❌ | ✔️ [[↗]][poco-net] | +| Async TLS Socket | ✔️ @ref engine::io::TlsWrapper "[↗]" | ✔️ | ❓ | ± third-party libs | ❌ | ✔️ [[↗]][poco-net] | +| Async HTTPS client | ✔️ @ref clients::http::Client "[↗]" | ✔️ | ❓ | ✔️ | ❓ | ✔️ [[↗]][poco-net] | +| Async HTTPS server | ✔️ @ref components::Server "[↗]" | ❓ | ❓ | ✔️ | ❓ |✔️ [[↗]][poco-net] | +| WebSockets Server | ✔️ @ref components::Server "[↗]" | ± third-party libs | ❌ [[↗]][dapr-websock] | ± third-party libs | ✔️ [[↗]][drogon] |❌| +| Deadlines and Cancellations | ✔️ | ❓ | ❓ | ❓ | ± [[↗]][drog-timeout] |❌| +| Retries and Load Balancing | ✔️ | ✔️ [[↗]][gom-features] | ✔️ | ❓ |❓ | ❌ | [userver-docs-pr]: https://github.com/userver-framework/userver/blob/develop/scripts/docs/en/userver/ @@ -71,6 +73,12 @@ use ❌ and ❓ respectively. [drog-timeout]: https://drogon.docsforge.com/master/session/ [tokio-sync]: https://docs.rs/tokio/0.2.18/tokio/sync/index.html [tokio-net]: https://docs.rs/tokio/0.1.22/tokio/net/index.html +[poco-cache]: https://pocoproject.org/slides/140-Cache.pdf +[poco-xml]: https://pocoproject.org/slides/170-XML.pdf +[poco-net]: https://pocoproject.org/slides/200-Network.pdf +[poco-sync]: https://pocoproject.org/slides/130-Threads.pdf +[poco-streams]: https://pocoproject.org/slides/100-Streams.pdf +[poco-db]: https://docs.pocoproject.org/current/Poco.Data.html @anchor fcmp1 [1]: "Dynamic Configs" stands for any out-of-the-box functionality that allows to change behavior of the service without downtime and restart. From 4fac6ffa109622d913476135a8cef026f79d4144 Mon Sep 17 00:00:00 2001 From: Kirill Date: Thu, 11 Jul 2024 13:45:14 +0300 Subject: [PATCH 016/629] feat build: fix wrong path when installing conan MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #525 Tests: протестировано CI e3a74a906fa52c56e470769a100ed00ed4d9f55d Pull Request resolved: https://github.com/userver-framework/userver/pull/640 --- universal/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/universal/CMakeLists.txt b/universal/CMakeLists.txt index 5ddad560a5e8..1b61d921c099 100644 --- a/universal/CMakeLists.txt +++ b/universal/CMakeLists.txt @@ -126,7 +126,7 @@ if (USERVER_COMPILER_HAS_MACRO_PREFIX_MAP) -fmacro-prefix-map=${BIN_LOG_PATH_BASE}= -fmacro-prefix-map=${BASE_PREFIX}= ) - if (NOT SRC_LOG_PATH_BASE MATCHES "${BASE_PREFIX}.*") + if (NOT SRC_LOG_PATH_BASE MATCHES "${BASE_PREFIX}.*" OR NOT "${USERVER_ROOT_DIR}" MATCHES [=[/userver$]=]) target_compile_options(userver-internal-compile-options INTERFACE -fmacro-prefix-map=${SRC_LOG_PATH_BASE}= ) From a2e7f474a13badcdbfcc13bd4cc56a9d87cf49e7 Mon Sep 17 00:00:00 2001 From: antoshkka Date: Thu, 11 Jul 2024 15:03:30 +0300 Subject: [PATCH 017/629] feat core: revert "Revert "feat mongo: use the engine::io:Poller rather than hand-written"" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit fac04cab580fb727a01dea09e9a14cc5d9a2bc14, reversing changes made to 276380b757981090d7a9374a0d5e832b5357a6d7. Tests: протестировано CI и в тестинге d5924787a45a1fb58b14dbad23c69be10ddd857a --- .mapping.json | 2 - mongo/CMakeLists.txt | 3 - .../storages/mongo/cdriver/async_stream.cpp | 80 +++++---- .../mongo/cdriver/async_stream_poller.cpp | 156 ------------------ .../mongo/cdriver/async_stream_poller.hpp | 64 ------- mongo/src/storages/mongo/cdriver/wrappers.cpp | 1 - 6 files changed, 47 insertions(+), 259 deletions(-) delete mode 100644 mongo/src/storages/mongo/cdriver/async_stream_poller.cpp delete mode 100644 mongo/src/storages/mongo/cdriver/async_stream_poller.hpp diff --git a/.mapping.json b/.mapping.json index 32842b93d978..0e94f53379db 100644 --- a/.mapping.json +++ b/.mapping.json @@ -2049,8 +2049,6 @@ "mongo/src/storages/mongo/cc_config.hpp":"taxi/uservices/userver/mongo/src/storages/mongo/cc_config.hpp", "mongo/src/storages/mongo/cdriver/async_stream.cpp":"taxi/uservices/userver/mongo/src/storages/mongo/cdriver/async_stream.cpp", "mongo/src/storages/mongo/cdriver/async_stream.hpp":"taxi/uservices/userver/mongo/src/storages/mongo/cdriver/async_stream.hpp", - "mongo/src/storages/mongo/cdriver/async_stream_poller.cpp":"taxi/uservices/userver/mongo/src/storages/mongo/cdriver/async_stream_poller.cpp", - "mongo/src/storages/mongo/cdriver/async_stream_poller.hpp":"taxi/uservices/userver/mongo/src/storages/mongo/cdriver/async_stream_poller.hpp", "mongo/src/storages/mongo/cdriver/collection_impl.cpp":"taxi/uservices/userver/mongo/src/storages/mongo/cdriver/collection_impl.cpp", "mongo/src/storages/mongo/cdriver/collection_impl.hpp":"taxi/uservices/userver/mongo/src/storages/mongo/cdriver/collection_impl.hpp", "mongo/src/storages/mongo/cdriver/cursor_impl.cpp":"taxi/uservices/userver/mongo/src/storages/mongo/cdriver/cursor_impl.cpp", diff --git a/mongo/CMakeLists.txt b/mongo/CMakeLists.txt index 35a09f077d47..c8cf61d90653 100644 --- a/mongo/CMakeLists.txt +++ b/mongo/CMakeLists.txt @@ -53,9 +53,6 @@ target_include_directories( PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src ) -target_include_directories (${PROJECT_NAME} SYSTEM PRIVATE - $ -) _userver_directory_install(COMPONENT mongo DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include diff --git a/mongo/src/storages/mongo/cdriver/async_stream.cpp b/mongo/src/storages/mongo/cdriver/async_stream.cpp index 9af98b73fab0..e1dceae0b878 100644 --- a/mongo/src/storages/mongo/cdriver/async_stream.cpp +++ b/mongo/src/storages/mongo/cdriver/async_stream.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -31,7 +32,6 @@ #include #include -#include #include #include @@ -58,13 +58,6 @@ static_assert(sizeof(ExpectedMongocStreamLayout) == sizeof(mongoc_stream_t), static_assert(std::size(mongoc_stream_t{}.padding) == 3, "Unexpected mongoc_stream_t structure layout"); -void SetWatcher(AsyncStreamPoller::WatcherPtr& old_watcher, - AsyncStreamPoller::WatcherPtr new_watcher) { - if (old_watcher == new_watcher) return; - if (old_watcher) old_watcher->Stop(); - old_watcher = std::move(new_watcher); -} - class AsyncStream : public mongoc_stream_t { public: static constexpr int kStreamType = 0x53755459; @@ -99,8 +92,6 @@ class AsyncStream : public mongoc_stream_t { const uint64_t epoch_; engine::io::Socket socket_; - AsyncStreamPoller::WatcherPtr read_watcher_; - AsyncStreamPoller::WatcherPtr write_watcher_; bool is_timed_out_{false}; bool is_created_{false}; @@ -271,17 +262,40 @@ uint64_t GetNextStreamEpoch() { // to only reset poller when a new poll cycle begins. class PollerDispenser { public: - AsyncStreamPoller& Get(uint64_t current_epoch) { + class Guard { + public: + Guard(engine::io::Poller& poller, std::atomic& is_locked) noexcept + : poller_{poller}, is_locked_{is_locked} { + UASSERT_MSG(!is_locked_.exchange(true), + "Someone is using the poller concurrently"); + } + + ~Guard() { + UASSERT_MSG(is_locked_.exchange(false), "The poller was not locked"); + } + + engine::io::Poller* operator->() const noexcept { return &poller_; } + + private: + engine::io::Poller& poller_; + std::atomic& is_locked_; + }; + + Guard Get(uint64_t current_epoch) { + Guard guard{poller_, is_locked_}; + if (seen_epoch_ < current_epoch) { poller_.Reset(); seen_epoch_ = current_epoch; } - return poller_; + + return guard; } private: uint64_t seen_epoch_{0}; - AsyncStreamPoller poller_; + engine::io::Poller poller_; + std::atomic is_locked_{false}; }; engine::TaskLocalVariable poller_dispenser; @@ -426,8 +440,11 @@ int AsyncStream::Close(mongoc_stream_t* stream) noexcept { LOG_TRACE() << "Closing async stream " << self; self->is_timed_out_ = false; - SetWatcher(self->read_watcher_, {}); - SetWatcher(self->write_watcher_, {}); + { + auto poller = poller_dispenser->Get(self->epoch_); + poller->Remove(self->socket_.Fd()); + } + try { self->socket_.Close(); } catch (const std::exception&) { @@ -589,37 +606,34 @@ ssize_t AsyncStream::Poll(mongoc_stream_poll_t* streams, size_t nstreams, current_epoch = std::max(current_epoch, stream->epoch_); stream_fds[i] = stream->socket_.Fd(); } - auto& poller = poller_dispenser->Get(current_epoch); + auto poller = poller_dispenser->Get(current_epoch); for (size_t i = 0; i < nstreams; ++i) { - // NOLINTNEXTLINE(cppcoreguidelines-pro-type-static-cast-downcast) - auto* stream = static_cast(streams[i].stream); if (streams[i].events & POLLOUT) { - SetWatcher(stream->write_watcher_, poller.AddWrite(stream_fds[i])); + poller->Add(stream_fds[i], {engine::io::Poller::Event::kError, + engine::io::Poller::Event::kWrite}); } else if (streams[i].events) { - SetWatcher(stream->read_watcher_, poller.AddRead(stream_fds[i])); + poller->Add(stream_fds[i], {engine::io::Poller::Event::kError, + engine::io::Poller::Event::kRead}); } streams[i].revents = 0; } ssize_t ready = 0; try { - AsyncStreamPoller::Event poller_event; - for (bool has_more = poller.NextEvent(poller_event, deadline); has_more; - has_more = poller.NextEventNoblock(poller_event)) { + engine::io::Poller::Event poller_event; + for (auto status = poller->NextEvent(poller_event, deadline); + status == engine::io::Poller::Status::kSuccess; + status = poller->NextEventNoblock(poller_event)) { for (size_t i = 0; i < nstreams; ++i) { if (stream_fds[i] == poller_event.fd) { ready += !streams[i].revents; - switch (poller_event.type) { - case AsyncStreamPoller::Event::kError: - streams[i].revents |= POLLERR; - break; - case AsyncStreamPoller::Event::kRead: - streams[i].revents |= streams[i].events & POLLIN; - break; - case AsyncStreamPoller::Event::kWrite: - streams[i].revents |= streams[i].events & POLLOUT; - break; + if (poller_event.type & engine::io::Poller::Event::kError) { + streams[i].revents |= POLLERR; + } else if (poller_event.type & engine::io::Poller::Event::kRead) { + streams[i].revents |= streams[i].events & POLLIN; + } else if (poller_event.type & engine::io::Poller::Event::kWrite) { + streams[i].revents |= streams[i].events & POLLOUT; } break; } diff --git a/mongo/src/storages/mongo/cdriver/async_stream_poller.cpp b/mongo/src/storages/mongo/cdriver/async_stream_poller.cpp deleted file mode 100644 index 0e272e77c64d..000000000000 --- a/mongo/src/storages/mongo/cdriver/async_stream_poller.cpp +++ /dev/null @@ -1,156 +0,0 @@ -#include - -#include - -#include -#include -#include - -USERVER_NAMESPACE_BEGIN - -namespace storages::mongo::impl::cdriver { - -namespace { - -void ConcurrentStop(int fd, std::mutex& mutex, - AsyncStreamPoller::WatchersMap& watchers) { - std::unique_lock lock(mutex); - const auto it = watchers.find(fd); - if (it == watchers.end()) return; - auto watcher = it->second.lock(); - lock.unlock(); - - // After `watcher.Stop();` mutexes, watchers and AsyncStreamPoller may be - // destroyed. But not before the Stop() call! - if (watcher) watcher->Stop(); -} - -void StopWatchers(AsyncStreamPoller::WatchersMap& watchers) { - for (auto& watcher_it : watchers) { - auto watcher = watcher_it.second.lock(); - if (watcher) watcher->Stop(); - } -} - -} // namespace - -AsyncStreamPoller::AsyncStreamPoller() - : AsyncStreamPoller(concurrent::MpscQueue::Create()) {} - -AsyncStreamPoller::~AsyncStreamPoller() { - // ResetWatchers() acquires a lock and avoids race between watchers.find(fd); - // and WatchersMap destruction. - ResetWatchers(); -} - -AsyncStreamPoller::AsyncStreamPoller( - const std::shared_ptr>& queue) - : event_consumer_(queue->GetConsumer()), - event_producer_(queue->GetProducer()) {} - -AsyncStreamPoller::WatcherPtr AsyncStreamPoller::AddRead(int fd) { - std::unique_lock lock(read_watchers_mutex_); - auto& watcher_ptr = read_watchers_[fd]; - auto watcher = watcher_ptr.lock(); - if (!watcher) { - watcher = std::make_shared>( - engine::current_task::GetEventThread(), this); - watcher->Init(&IoEventCb, fd, EV_READ); - watcher_ptr = watcher; - } - lock.unlock(); - watcher->StartAsync(); - return watcher; -} - -AsyncStreamPoller::WatcherPtr AsyncStreamPoller::AddWrite(int fd) { - std::unique_lock lock(write_watchers_mutex_); - auto& watcher_ptr = write_watchers_[fd]; - auto watcher = watcher_ptr.lock(); - if (!watcher) { - watcher = std::make_shared>( - engine::current_task::GetEventThread(), this); - watcher->Init(&IoEventCb, fd, EV_WRITE); - watcher_ptr = watcher; - } - lock.unlock(); - watcher->StartAsync(); - return watcher; -} - -void AsyncStreamPoller::Reset() { - ResetWatchers(); - - Event buf; - while (event_consumer_.PopNoblock(buf)) - ; // discard -} - -bool AsyncStreamPoller::NextEvent(Event& buf, engine::Deadline deadline) { - return event_consumer_.Pop(buf, deadline); -} - -bool AsyncStreamPoller::NextEventNoblock(Event& buf) { - return event_consumer_.PopNoblock(buf); -} - -void AsyncStreamPoller::StopRead(int fd) { - ConcurrentStop(fd, read_watchers_mutex_, read_watchers_); -} - -void AsyncStreamPoller::StopWrite(int fd) { - ConcurrentStop(fd, write_watchers_mutex_, write_watchers_); -} - -void AsyncStreamPoller::ResetWatchers() { - // destroy watchers without lock to avoid deadlocking with callback - WatchersMap watchers_buffer; - { - std::lock_guard read_lock(read_watchers_mutex_); - read_watchers_.swap(watchers_buffer); - } - StopWatchers(watchers_buffer); - watchers_buffer.clear(); - { - std::lock_guard write_lock(write_watchers_mutex_); - write_watchers_.swap(watchers_buffer); - } - StopWatchers(watchers_buffer); - watchers_buffer.clear(); -} - -void AsyncStreamPoller::IoEventCb(struct ev_loop*, ev_io* watcher, - int revents) noexcept { - UASSERT(watcher->active); - UASSERT((watcher->events & ~(EV_READ | EV_WRITE)) == 0); - - auto* poller = static_cast(watcher->data); - auto event_type = Event::kError; - if (revents & EV_ERROR) { - // highest priority, can be mixed with dummy events - event_type = Event::kError; - } else if (revents & EV_READ) { - event_type = Event::kRead; - } else if (revents & EV_WRITE) { - event_type = Event::kWrite; - } - // NOTE: it might be better to poll() here to get POLLERR/POLLHUP as well - [[maybe_unused]] bool is_sent = - poller->event_producer_.PushNoblock({watcher->fd, event_type}); - UASSERT(is_sent); - - if (watcher->events & EV_READ) { - UASSERT(!(watcher->events & EV_WRITE)); - poller->StopRead(watcher->fd); - } else if (watcher->events & EV_WRITE) { - poller->StopWrite(watcher->fd); - } else { - LOG_LIMITED_ERROR() << "Watcher is neither read nor write watcher, events=" - << watcher->events; - UASSERT_MSG(false, "Watcher is neither read nor write watcher"); - } -} - -} // namespace storages::mongo::impl::cdriver - -USERVER_NAMESPACE_END diff --git a/mongo/src/storages/mongo/cdriver/async_stream_poller.hpp b/mongo/src/storages/mongo/cdriver/async_stream_poller.hpp deleted file mode 100644 index 7d8bbaf9fd3f..000000000000 --- a/mongo/src/storages/mongo/cdriver/async_stream_poller.hpp +++ /dev/null @@ -1,64 +0,0 @@ -#pragma once - -#include -#include - -#include -#include -#include - -#include - -USERVER_NAMESPACE_BEGIN - -namespace storages::mongo::impl::cdriver { - -// AsyncStream-specific implementation of global poller object -// Not thread-safe -// Reports HUP as readiness (libev limitation) -class AsyncStreamPoller final { - public: - using WatcherPtr = std::shared_ptr>; - using WatcherWeakPtr = std::weak_ptr>; - using WatchersMap = std::unordered_map; - - struct Event { - int fd{engine::io::kInvalidFd}; - enum { kError = -1, kRead, kWrite } type{kError}; - }; - - AsyncStreamPoller(); - ~AsyncStreamPoller(); - - AsyncStreamPoller(const AsyncStreamPoller&) = delete; - AsyncStreamPoller(AsyncStreamPoller&&) = delete; - - void Reset(); - - [[nodiscard]] WatcherPtr AddRead(int fd); - [[nodiscard]] WatcherPtr AddWrite(int fd); - - bool NextEvent(Event&, engine::Deadline); - bool NextEventNoblock(Event&); - - private: - explicit AsyncStreamPoller( - const std::shared_ptr>&); - - void StopRead(int fd); - void StopWrite(int fd); - void ResetWatchers(); - - static void IoEventCb(struct ev_loop*, ev_io*, int) noexcept; - - concurrent::MpscQueue::Consumer event_consumer_; - concurrent::MpscQueue::Producer event_producer_; - std::mutex read_watchers_mutex_; - WatchersMap read_watchers_; - std::mutex write_watchers_mutex_; - WatchersMap write_watchers_; -}; - -} // namespace storages::mongo::impl::cdriver - -USERVER_NAMESPACE_END diff --git a/mongo/src/storages/mongo/cdriver/wrappers.cpp b/mongo/src/storages/mongo/cdriver/wrappers.cpp index 6d9ef5ba7250..cdf4d5cea531 100644 --- a/mongo/src/storages/mongo/cdriver/wrappers.cpp +++ b/mongo/src/storages/mongo/cdriver/wrappers.cpp @@ -14,7 +14,6 @@ #include #include -#include #include USERVER_NAMESPACE_BEGIN From 672210208effddd589baddb09f582f0683ec6666 Mon Sep 17 00:00:00 2001 From: segoon Date: Thu, 11 Jul 2024 15:33:36 +0300 Subject: [PATCH 018/629] feat userver: implement OTLP logging 9eced1c0b16e3aac5f51731422d5062db1206569 --- .mapping.json | 21 ++ CMakeLists.txt | 12 + cmake/install/userver-otlp-config.cmake | 13 + core/include/userver/logging/component.hpp | 16 +- core/include/userver/logging/logger.hpp | 12 + core/src/components/run.cpp | 10 +- core/src/logging/component.cpp | 21 +- core/src/logging/component_test.cpp | 46 --- core/src/logging/logger.cpp | 31 ++ core/src/logging/tp_logger.cpp | 20 +- core/src/logging/tp_logger_utils.cpp | 5 +- core/src/logging/tp_logger_utils.hpp | 6 +- otlp/CMakeLists.txt | 59 +++ otlp/include/userver/otlp/logs/component.hpp | 51 +++ .../opentelemetry/proto/collector/README.md | 10 + .../collector/logs/v1/logs_service.proto | 79 ++++ .../metrics/v1/metrics_service.proto | 79 ++++ .../collector/trace/v1/trace_service.proto | 79 ++++ .../proto/common/v1/common.proto | 81 ++++ .../opentelemetry/proto/logs/v1/logs.proto | 211 +++++++++++ .../proto/resource/v1/resource.proto | 37 ++ .../opentelemetry/proto/trace/v1/trace.proto | 355 ++++++++++++++++++ otlp/src/otlp/logs/component.cpp | 99 +++++ otlp/src/otlp/logs/logger.cpp | 325 ++++++++++++++++ otlp/src/otlp/logs/logger.hpp | 81 ++++ samples/CMakeLists.txt | 5 + samples/otlp_service/CMakeLists.txt | 6 + samples/otlp_service/config_vars.yaml | 18 + samples/otlp_service/otlp_service.cpp | 26 ++ samples/otlp_service/secure_data.json | 1 + samples/otlp_service/static_config.yaml | 63 ++++ samples/otlp_service/testsuite/conftest.py | 4 + samples/otlp_service/testsuite/test_hello.py | 3 + .../pytest_userver/plugins/config.py | 19 + 34 files changed, 1825 insertions(+), 79 deletions(-) create mode 100644 cmake/install/userver-otlp-config.cmake create mode 100644 otlp/CMakeLists.txt create mode 100644 otlp/include/userver/otlp/logs/component.hpp create mode 100644 otlp/proto/opentelemetry/proto/collector/README.md create mode 100644 otlp/proto/opentelemetry/proto/collector/logs/v1/logs_service.proto create mode 100644 otlp/proto/opentelemetry/proto/collector/metrics/v1/metrics_service.proto create mode 100644 otlp/proto/opentelemetry/proto/collector/trace/v1/trace_service.proto create mode 100644 otlp/proto/opentelemetry/proto/common/v1/common.proto create mode 100644 otlp/proto/opentelemetry/proto/logs/v1/logs.proto create mode 100644 otlp/proto/opentelemetry/proto/resource/v1/resource.proto create mode 100644 otlp/proto/opentelemetry/proto/trace/v1/trace.proto create mode 100644 otlp/src/otlp/logs/component.cpp create mode 100644 otlp/src/otlp/logs/logger.cpp create mode 100644 otlp/src/otlp/logs/logger.hpp create mode 100644 samples/otlp_service/CMakeLists.txt create mode 100644 samples/otlp_service/config_vars.yaml create mode 100644 samples/otlp_service/otlp_service.cpp create mode 100644 samples/otlp_service/secure_data.json create mode 100644 samples/otlp_service/static_config.yaml create mode 100644 samples/otlp_service/testsuite/conftest.py create mode 100644 samples/otlp_service/testsuite/test_hello.py diff --git a/.mapping.json b/.mapping.json index 0e94f53379db..b76354e35e17 100644 --- a/.mapping.json +++ b/.mapping.json @@ -339,6 +339,7 @@ "cmake/install/userver-kafka-config.cmake":"taxi/uservices/userver/cmake/install/userver-kafka-config.cmake", "cmake/install/userver-mongo-config.cmake":"taxi/uservices/userver/cmake/install/userver-mongo-config.cmake", "cmake/install/userver-mysql-config.cmake":"taxi/uservices/userver/cmake/install/userver-mysql-config.cmake", + "cmake/install/userver-otlp-config.cmake":"taxi/uservices/userver/cmake/install/userver-otlp-config.cmake", "cmake/install/userver-postgresql-config.cmake":"taxi/uservices/userver/cmake/install/userver-postgresql-config.cmake", "cmake/install/userver-rabbitmq-config.cmake":"taxi/uservices/userver/cmake/install/userver-rabbitmq-config.cmake", "cmake/install/userver-redis-config.cmake":"taxi/uservices/userver/cmake/install/userver-redis-config.cmake", @@ -2230,6 +2231,19 @@ "mysql/src/storages/tests/unittests/uuid_mysqltest.cpp":"taxi/uservices/userver/mysql/src/storages/tests/unittests/uuid_mysqltest.cpp", "mysql/src/storages/tests/utils_mysqltest.cpp":"taxi/uservices/userver/mysql/src/storages/tests/utils_mysqltest.cpp", "mysql/src/storages/tests/utils_mysqltest.hpp":"taxi/uservices/userver/mysql/src/storages/tests/utils_mysqltest.hpp", + "otlp/CMakeLists.txt":"taxi/uservices/userver/otlp/CMakeLists.txt", + "otlp/include/userver/otlp/logs/component.hpp":"taxi/uservices/userver/otlp/include/userver/otlp/logs/component.hpp", + "otlp/proto/opentelemetry/proto/collector/README.md":"taxi/uservices/userver/otlp/proto/opentelemetry/proto/collector/README.md", + "otlp/proto/opentelemetry/proto/collector/logs/v1/logs_service.proto":"taxi/uservices/userver/otlp/proto/opentelemetry/proto/collector/logs/v1/logs_service.proto", + "otlp/proto/opentelemetry/proto/collector/metrics/v1/metrics_service.proto":"taxi/uservices/userver/otlp/proto/opentelemetry/proto/collector/metrics/v1/metrics_service.proto", + "otlp/proto/opentelemetry/proto/collector/trace/v1/trace_service.proto":"taxi/uservices/userver/otlp/proto/opentelemetry/proto/collector/trace/v1/trace_service.proto", + "otlp/proto/opentelemetry/proto/common/v1/common.proto":"taxi/uservices/userver/otlp/proto/opentelemetry/proto/common/v1/common.proto", + "otlp/proto/opentelemetry/proto/logs/v1/logs.proto":"taxi/uservices/userver/otlp/proto/opentelemetry/proto/logs/v1/logs.proto", + "otlp/proto/opentelemetry/proto/resource/v1/resource.proto":"taxi/uservices/userver/otlp/proto/opentelemetry/proto/resource/v1/resource.proto", + "otlp/proto/opentelemetry/proto/trace/v1/trace.proto":"taxi/uservices/userver/otlp/proto/opentelemetry/proto/trace/v1/trace.proto", + "otlp/src/otlp/logs/component.cpp":"taxi/uservices/userver/otlp/src/otlp/logs/component.cpp", + "otlp/src/otlp/logs/logger.cpp":"taxi/uservices/userver/otlp/src/otlp/logs/logger.cpp", + "otlp/src/otlp/logs/logger.hpp":"taxi/uservices/userver/otlp/src/otlp/logs/logger.hpp", "postgresql/CMakeLists.txt":"taxi/uservices/userver/postgresql/CMakeLists.txt", "postgresql/README.md":"taxi/uservices/userver/postgresql/README.md", "postgresql/functional_tests/CMakeLists.txt":"taxi/uservices/userver/postgresql/functional_tests/CMakeLists.txt", @@ -2926,6 +2940,13 @@ "samples/mysql_service/static_config.yaml":"taxi/uservices/userver/samples/mysql_service/static_config.yaml", "samples/mysql_service/tests/conftest.py":"taxi/uservices/userver/samples/mysql_service/tests/conftest.py", "samples/mysql_service/tests/test_mysql.py":"taxi/uservices/userver/samples/mysql_service/tests/test_mysql.py", + "samples/otlp_service/CMakeLists.txt":"taxi/uservices/userver/samples/otlp_service/CMakeLists.txt", + "samples/otlp_service/config_vars.yaml":"taxi/uservices/userver/samples/otlp_service/config_vars.yaml", + "samples/otlp_service/otlp_service.cpp":"taxi/uservices/userver/samples/otlp_service/otlp_service.cpp", + "samples/otlp_service/secure_data.json":"taxi/uservices/userver/samples/otlp_service/secure_data.json", + "samples/otlp_service/static_config.yaml":"taxi/uservices/userver/samples/otlp_service/static_config.yaml", + "samples/otlp_service/testsuite/conftest.py":"taxi/uservices/userver/samples/otlp_service/testsuite/conftest.py", + "samples/otlp_service/testsuite/test_hello.py":"taxi/uservices/userver/samples/otlp_service/testsuite/test_hello.py", "samples/postgres-support/CMakeLists.txt":"taxi/uservices/userver/samples/postgres-support/CMakeLists.txt", "samples/postgres-support/config_vars.yaml":"taxi/uservices/userver/samples/postgres-support/config_vars.yaml", "samples/postgres-support/schemas/postgresql/service.sql":"taxi/uservices/userver/samples/postgres-support/schemas/postgresql/service.sql", diff --git a/CMakeLists.txt b/CMakeLists.txt index c3758ffaf4a1..a20f23465b0b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -175,6 +175,9 @@ set(USERVER_THIRD_PARTY_DIRS ${USERVER_ROOT_DIR}/third_party CACHE INTERNAL "") init_debian_depends() include(SetupGTest) + +option(USERVER_FEATURE_OTLP "Provide asynchronous OTLP exporters" "${USERVER_LIB_ENABLED_DEFAULT}") + if (USERVER_FEATURE_GRPC) include(SetupProtobuf) endif() @@ -228,6 +231,15 @@ if (USERVER_FEATURE_GRPC) list(APPEND USERVER_AVAILABLE_COMPONENTS grpc) endif() +if (USERVER_FEATURE_OTLP) + if (NOT USERVER_FEATURE_GRPC) + message(FATAL_ERROR "'USERVER_FEATURE_OTLP' requires 'USERVER_FEATURE_GRPC=ON'") + endif() + _require_userver_core("USERVER_FEATURE_OTLP") + add_subdirectory(otlp) + list(APPEND USERVER_AVAILABLE_COMPONENTS otlp) +endif() + if (USERVER_FEATURE_CLICKHOUSE) _require_userver_core("USERVER_FEATURE_CLICKHOUSE") add_subdirectory(clickhouse) diff --git a/cmake/install/userver-otlp-config.cmake b/cmake/install/userver-otlp-config.cmake new file mode 100644 index 000000000000..bc4d6c09995c --- /dev/null +++ b/cmake/install/userver-otlp-config.cmake @@ -0,0 +1,13 @@ +include_guard(GLOBAL) + +if(userver_otlp_FOUND) + return() +endif() + +find_package(userver REQUIRED COMPONENTS + core +) + +list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/..") + +set(userver_otlp_FOUND TRUE) diff --git a/core/include/userver/logging/component.hpp b/core/include/userver/logging/component.hpp index aa796c6b8932..0fe5c5ef22b3 100644 --- a/core/include/userver/logging/component.hpp +++ b/core/include/userver/logging/component.hpp @@ -12,6 +12,7 @@ #include #include +#include #include #include #include @@ -21,15 +22,10 @@ USERVER_NAMESPACE_BEGIN -namespace logging { -struct LoggerConfig; - -namespace impl { +namespace logging::impl { class TpLogger; class TcpSocketSink; -} // namespace impl - -} // namespace logging +} // namespace logging::impl namespace components { @@ -92,6 +88,11 @@ class Logging final : public RawComponentBase { /// @throws std::runtime_error if logger with this name is not registered logging::LoggerPtr GetLogger(const std::string& name); + /// @brief Sets a logger + /// @param name Name of the logger + /// @param logger Logger to set + void SetLogger(const std::string& name, logging::LoggerPtr logger); + /// @brief Returns a logger by its name /// @param name Name of the logger /// @returns Pointer to the Logger instance, or `nullptr` if not registered @@ -117,6 +118,7 @@ class Logging final : public RawComponentBase { engine::TaskProcessor* fs_task_processor_{nullptr}; std::unordered_map> loggers_; + rcu::RcuMap extra_loggers_; utils::PeriodicTask flush_task_; logging::impl::TcpSocketSink* socket_sink_{nullptr}; alerts::Storage& alert_storage_; diff --git a/core/include/userver/logging/logger.hpp b/core/include/userver/logging/logger.hpp index fb7bb51bd3ad..370c9a917e1d 100644 --- a/core/include/userver/logging/logger.hpp +++ b/core/include/userver/logging/logger.hpp @@ -40,6 +40,18 @@ LoggerPtr MakeStdoutLogger(const std::string& name, Format format, LoggerPtr MakeFileLogger(const std::string& name, const std::string& path, Format format, Level level = Level::kInfo); +namespace impl { +class TagWriter; +} + +namespace impl::default_ { + +bool DoShouldLog(Level) noexcept; + +void PrependCommonTags(TagWriter writer); + +} // namespace impl::default_ + } // namespace logging USERVER_NAMESPACE_END diff --git a/core/src/components/run.cpp b/core/src/components/run.cpp index dac1bc777142..0212d4ab8392 100644 --- a/core/src/components/run.cpp +++ b/core/src/components/run.cpp @@ -169,11 +169,13 @@ ManagerConfig ParseManagerConfigAndSetupLogging( const auto default_logger_config = logging::impl::ExtractDefaultLoggerConfig(manager_config); - auto default_logger = logging::impl::MakeTpLogger(default_logger_config); + if (default_logger_config) { + auto default_logger = logging::impl::MakeTpLogger(*default_logger_config); - // This line enables basic logging. Any LOG_XXX before it is meaningless, - // because it would typically go straight to a NullLogger. - log_scope.SetLogger(std::move(default_logger)); + // This line enables basic logging. Any LOG_XXX before it is meaningless, + // because it would typically go straight to a NullLogger. + log_scope.SetLogger(std::move(default_logger)); + } LOG_INFO() << "Parsed " << details; return manager_config; diff --git a/core/src/logging/component.cpp b/core/src/logging/component.cpp index aaa93a7b7de6..9080223f388f 100644 --- a/core/src/logging/component.cpp +++ b/core/src/logging/component.cpp @@ -65,6 +65,11 @@ void Logging::Init(const ComponentConfig& config, const auto logger_configs = yaml_config::ParseMapToArray(config["loggers"]); + if (logger_configs.empty()) { + // We don't own default logger, we have no stats to report + return; + } + for (const auto& logger_config : logger_configs) { const bool is_default_logger = (logger_config.logger_name == "default"); @@ -83,10 +88,12 @@ void Logging::Init(const ComponentConfig& config, if (logger_config.queue_overflow_behavior == logging::QueueOverflowBehavior::kBlock) { throw std::runtime_error( - "'default' logger should not be set to 'overflow_behavior: block'! " + "'default' logger should not be set to 'overflow_behavior: " + "block'! " "Default logger is used by the userver internals, including the " "logging internals. Blocking inside the engine internals could " - "lead to hardly reproducible hangups in some border cases of error " + "lead to hardly reproducible hangups in some border cases of " + "error " "reporting."); } @@ -137,9 +144,19 @@ void Logging::Stop() noexcept { } } +void Logging::SetLogger(const std::string& name, logging::LoggerPtr logger) { + auto [_, inserted] = extra_loggers_.Emplace(name, std::move(logger)); + if (!inserted) { + throw std::runtime_error(fmt::format("Duplicate logger name: {}", name)); + } +} + logging::LoggerPtr Logging::GetLogger(const std::string& name) { auto it = loggers_.find(name); if (it == loggers_.end()) { + auto logger = extra_loggers_.Get(name); + if (logger) return *logger; + throw std::runtime_error("logger '" + name + "' not found"); } return it->second; diff --git a/core/src/logging/component_test.cpp b/core/src/logging/component_test.cpp index f7c8a12c6663..e23be44f228f 100644 --- a/core/src/logging/component_test.cpp +++ b/core/src/logging/component_test.cpp @@ -55,52 +55,6 @@ TEST_F(ComponentList, NoLogging) { namespace { -constexpr std::string_view kNoDefaultLoggerConfig = R"( -components_manager: - coro_pool: - initial_size: 50 - max_size: 500 - default_task_processor: main-task-processor - event_thread_pool: - threads: 1 - task_processors: - main-task-processor: - worker_threads: 1 - components: - logging: - fs-task-processor: main-task-processor - loggers: - some-logger: - file_path: '@stderr' - format: tskv - tracer: - service-name: test-service -)"; - -components::ComponentList MakeNoDefaultLoggerComponentList() { - return components::ComponentList() - .Append() - .Append() - .Append() - .Append(); -} - -} // namespace - -TEST_F(ComponentList, NoDefaultLogger) { - UEXPECT_THROW_MSG( - components::RunOnce( - components::InMemoryConfig{std::string{kNoDefaultLoggerConfig}}, - MakeNoDefaultLoggerComponentList()), - std::exception, - "Error while parsing configs from in-memory config. Details: Error at " - "path " - "'components_manager.components.logging.loggers.default.file_path': " - "Field is missing"); -} - -namespace { - class TwoLoggersComponent final : public components::ComponentBase { public: static constexpr std::string_view kName = "two-loggers"; diff --git a/core/src/logging/logger.cpp b/core/src/logging/logger.cpp index 0a44cf743618..7b8de39454da 100644 --- a/core/src/logging/logger.cpp +++ b/core/src/logging/logger.cpp @@ -2,11 +2,15 @@ #include +#include #include #include #include #include +#include +#include + #include "config.hpp" USERVER_NAMESPACE_BEGIN @@ -66,6 +70,33 @@ void LogRaw(LoggerBase& logger, Level level, std::string_view message) { } // namespace impl +namespace impl::default_ { + +bool DoShouldLog(Level level) noexcept { + const auto* const span = tracing::Span::CurrentSpanUnchecked(); + if (span) { + const auto local_log_level = span->GetLocalLogLevel(); + if (local_log_level && *local_log_level > level) { + return false; + } + } + + return true; +} + +void PrependCommonTags(TagWriter writer) { + auto* const span = tracing::Span::CurrentSpanUnchecked(); + if (span) span->LogTo(writer); + + auto* const task = engine::current_task::GetCurrentTaskContextUnchecked(); + writer.PutTag("task_id", HexShort{task}); + + auto* const thread_id = reinterpret_cast(pthread_self()); + writer.PutTag("thread_id", Hex{thread_id}); +} + +} // namespace impl::default_ + } // namespace logging USERVER_NAMESPACE_END diff --git a/core/src/logging/tp_logger.cpp b/core/src/logging/tp_logger.cpp index 79e30c1a59fc..05005b4af0b6 100644 --- a/core/src/logging/tp_logger.cpp +++ b/core/src/logging/tp_logger.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -151,26 +152,11 @@ void TpLogger::Log(Level level, std::string_view msg) { } void TpLogger::PrependCommonTags(TagWriter writer) const { - auto* const span = tracing::Span::CurrentSpanUnchecked(); - if (span) span->LogTo(writer); - - auto* const task = engine::current_task::GetCurrentTaskContextUnchecked(); - writer.PutTag("task_id", HexShort{task}); - - auto* const thread_id = reinterpret_cast(pthread_self()); - writer.PutTag("thread_id", Hex{thread_id}); + impl::default_::PrependCommonTags(writer); } bool TpLogger::DoShouldLog(Level level) const noexcept { - const auto* const span = tracing::Span::CurrentSpanUnchecked(); - if (span) { - const auto local_log_level = span->GetLocalLogLevel(); - if (local_log_level && *local_log_level > level) { - return false; - } - } - - return true; + return impl::default_::DoShouldLog(level); } void TpLogger::AddSink(impl::SinkPtr&& sink) { diff --git a/core/src/logging/tp_logger_utils.cpp b/core/src/logging/tp_logger_utils.cpp index 984286a4f428..fcb92fc1592d 100644 --- a/core/src/logging/tp_logger_utils.cpp +++ b/core/src/logging/tp_logger_utils.cpp @@ -114,7 +114,7 @@ TcpSocketSink* GetTcpSocketSink(TpLogger& logger) { return nullptr; } -LoggerConfig ExtractDefaultLoggerConfig( +std::optional ExtractDefaultLoggerConfig( const components::ManagerConfig& config) { // Note: this is a slight violation of separation of concerns. The component // system itself should not specifically care about 'logging' component, but @@ -126,12 +126,13 @@ LoggerConfig ExtractDefaultLoggerConfig( return config.Name() == "logging"; }); if (iter == config.components.end()) { - throw std::runtime_error( + throw NoLoggerComponent( "No component config found for 'logging', which is a required " "component"); } const auto logger_config_yaml = (*iter)["loggers"][kDefaultLoggerName]; + if (logger_config_yaml.IsMissing()) return {}; auto logger_config = logger_config_yaml.As(); logger_config.SetName(std::string{kDefaultLoggerName}); diff --git a/core/src/logging/tp_logger_utils.hpp b/core/src/logging/tp_logger_utils.hpp index 647a1f1ad737..5b90101fa252 100644 --- a/core/src/logging/tp_logger_utils.hpp +++ b/core/src/logging/tp_logger_utils.hpp @@ -20,7 +20,11 @@ std::shared_ptr GetDefaultLoggerOrMakeTpLogger( TcpSocketSink* GetTcpSocketSink(TpLogger& logger); -LoggerConfig ExtractDefaultLoggerConfig( +class NoLoggerComponent final : public std::runtime_error { + using std::runtime_error::runtime_error; +}; + +std::optional ExtractDefaultLoggerConfig( const components::ManagerConfig& config); } // namespace logging::impl diff --git a/otlp/CMakeLists.txt b/otlp/CMakeLists.txt new file mode 100644 index 000000000000..69b6c29b3ef6 --- /dev/null +++ b/otlp/CMakeLists.txt @@ -0,0 +1,59 @@ +project(userver-otlp CXX) + +include(GrpcTargets) + +find_package(Boost REQUIRED regex) + +file(GLOB_RECURSE SOURCES + ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/*.hpp) + +file(GLOB_RECURSE UNIT_TEST_SOURCES + ${CMAKE_CURRENT_SOURCE_DIR}/*_test.cpp +) +list(REMOVE_ITEM SOURCES ${UNIT_TEST_SOURCES}) + +file(GLOB_RECURSE BENCH_SOURCES + ${CMAKE_CURRENT_SOURCE_DIR}/*_benchmark.cpp +) +list (REMOVE_ITEM SOURCES ${BENCH_SOURCES}) + +userver_add_grpc_library(${PROJECT_NAME}-proto + PROTOS + opentelemetry/proto/collector/trace/v1/trace_service.proto + opentelemetry/proto/collector/logs/v1/logs_service.proto + opentelemetry/proto/common/v1/common.proto + opentelemetry/proto/logs/v1/logs.proto + opentelemetry/proto/resource/v1/resource.proto + opentelemetry/proto/trace/v1/trace.proto +) + +add_library(${PROJECT_NAME} STATIC ${SOURCES}) + +set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE CXX) + +target_include_directories(${PROJECT_NAME} PUBLIC + $ + PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/src +) +target_include_directories (${PROJECT_NAME} PRIVATE + $ +) + +target_link_libraries(${PROJECT_NAME} + PUBLIC + userver-core + ${PROJECT_NAME}-proto +) + +_userver_directory_install(COMPONENT otlp + DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/.. +) +_userver_install_targets(COMPONENT otlp TARGETS ${PROJECT_NAME} ${PROJECT_NAME}-proto) + +_userver_directory_install(COMPONENT otlp FILES + "${USERVER_ROOT_DIR}/cmake/install/userver-otlp-config.cmake" + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/userver +) diff --git a/otlp/include/userver/otlp/logs/component.hpp b/otlp/include/userver/otlp/logs/component.hpp new file mode 100644 index 000000000000..ab917003a04c --- /dev/null +++ b/otlp/include/userver/otlp/logs/component.hpp @@ -0,0 +1,51 @@ +#pragma once + +/// @file userver/otlp/logs/component.hpp +/// @brief @copybrief otlp::logs::Component + +#include +#include + +#include +#include + +USERVER_NAMESPACE_BEGIN + +namespace otlp { + +class Logger; + +// clang-format off + +/// @ingroup userver_components +/// +/// @brief Component to configure logging via OTLP collector. +/// +/// ## Static options: +/// Name | Description | Default value +/// ---- | ----------- | ------------- +/// endpoint | URI of otel collector (e.g. 127.0.0.1:4317) | - +/// max-queue-size | Maximum async queue size | 65535 +/// max-batch-delay | Maximum batch delay | 100ms +/// service-name | Service name | unknown_service +/// attributes | Extra attributes for OTLP, object of key/value strings | - + +// clang-format on +class LoggerComponent final : public components::RawComponentBase { + public: + static constexpr std::string_view kName = "otlp-logger"; + + LoggerComponent(const components::ComponentConfig&, + const components::ComponentContext&); + + ~LoggerComponent(); + + static yaml_config::Schema GetStaticConfigSchema(); + + private: + std::shared_ptr logger_; +}; + +} // namespace otlp + +USERVER_NAMESPACE_END diff --git a/otlp/proto/opentelemetry/proto/collector/README.md b/otlp/proto/opentelemetry/proto/collector/README.md new file mode 100644 index 000000000000..f82dbb0278b0 --- /dev/null +++ b/otlp/proto/opentelemetry/proto/collector/README.md @@ -0,0 +1,10 @@ +# OpenTelemetry Collector Proto + +This package describes the OpenTelemetry collector protocol. + +## Packages + +1. `common` package contains the common messages shared between different services. +2. `trace` package contains the Trace Service protos. +3. `metrics` package contains the Metrics Service protos. +4. `logs` package contains the Logs Service protos. diff --git a/otlp/proto/opentelemetry/proto/collector/logs/v1/logs_service.proto b/otlp/proto/opentelemetry/proto/collector/logs/v1/logs_service.proto new file mode 100644 index 000000000000..8260d8aaeb82 --- /dev/null +++ b/otlp/proto/opentelemetry/proto/collector/logs/v1/logs_service.proto @@ -0,0 +1,79 @@ +// Copyright 2020, OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package opentelemetry.proto.collector.logs.v1; + +import "opentelemetry/proto/logs/v1/logs.proto"; + +option csharp_namespace = "OpenTelemetry.Proto.Collector.Logs.V1"; +option java_multiple_files = true; +option java_package = "io.opentelemetry.proto.collector.logs.v1"; +option java_outer_classname = "LogsServiceProto"; +option go_package = "go.opentelemetry.io/proto/otlp/collector/logs/v1"; + +// Service that can be used to push logs between one Application instrumented with +// OpenTelemetry and an collector, or between an collector and a central collector (in this +// case logs are sent/received to/from multiple Applications). +service LogsService { + // For performance reasons, it is recommended to keep this RPC + // alive for the entire life of the application. + rpc Export(ExportLogsServiceRequest) returns (ExportLogsServiceResponse) {} +} + +message ExportLogsServiceRequest { + // An array of ResourceLogs. + // For data coming from a single resource this array will typically contain one + // element. Intermediary nodes (such as OpenTelemetry Collector) that receive + // data from multiple origins typically batch the data before forwarding further and + // in that case this array will contain multiple elements. + repeated opentelemetry.proto.logs.v1.ResourceLogs resource_logs = 1; +} + +message ExportLogsServiceResponse { + // The details of a partially successful export request. + // + // If the request is only partially accepted + // (i.e. when the server accepts only parts of the data and rejects the rest) + // the server MUST initialize the `partial_success` field and MUST + // set the `rejected_` with the number of items it rejected. + // + // Servers MAY also make use of the `partial_success` field to convey + // warnings/suggestions to senders even when the request was fully accepted. + // In such cases, the `rejected_` MUST have a value of `0` and + // the `error_message` MUST be non-empty. + // + // A `partial_success` message with an empty value (rejected_ = 0 and + // `error_message` = "") is equivalent to it not being set/present. Senders + // SHOULD interpret it the same way as in the full success case. + ExportLogsPartialSuccess partial_success = 1; +} + +message ExportLogsPartialSuccess { + // The number of rejected log records. + // + // A `rejected_` field holding a `0` value indicates that the + // request was fully accepted. + int64 rejected_log_records = 1; + + // A developer-facing human-readable message in English. It should be used + // either to explain why the server rejected parts of the data during a partial + // success or to convey warnings/suggestions during a full success. The message + // should offer guidance on how users can address such issues. + // + // error_message is an optional field. An error_message with an empty value + // is equivalent to it not being set. + string error_message = 2; +} diff --git a/otlp/proto/opentelemetry/proto/collector/metrics/v1/metrics_service.proto b/otlp/proto/opentelemetry/proto/collector/metrics/v1/metrics_service.proto new file mode 100644 index 000000000000..dd48f1ad3a16 --- /dev/null +++ b/otlp/proto/opentelemetry/proto/collector/metrics/v1/metrics_service.proto @@ -0,0 +1,79 @@ +// Copyright 2019, OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package opentelemetry.proto.collector.metrics.v1; + +import "opentelemetry/proto/metrics/v1/metrics.proto"; + +option csharp_namespace = "OpenTelemetry.Proto.Collector.Metrics.V1"; +option java_multiple_files = true; +option java_package = "io.opentelemetry.proto.collector.metrics.v1"; +option java_outer_classname = "MetricsServiceProto"; +option go_package = "go.opentelemetry.io/proto/otlp/collector/metrics/v1"; + +// Service that can be used to push metrics between one Application +// instrumented with OpenTelemetry and a collector, or between a collector and a +// central collector. +service MetricsService { + // For performance reasons, it is recommended to keep this RPC + // alive for the entire life of the application. + rpc Export(ExportMetricsServiceRequest) returns (ExportMetricsServiceResponse) {} +} + +message ExportMetricsServiceRequest { + // An array of ResourceMetrics. + // For data coming from a single resource this array will typically contain one + // element. Intermediary nodes (such as OpenTelemetry Collector) that receive + // data from multiple origins typically batch the data before forwarding further and + // in that case this array will contain multiple elements. + repeated opentelemetry.proto.metrics.v1.ResourceMetrics resource_metrics = 1; +} + +message ExportMetricsServiceResponse { + // The details of a partially successful export request. + // + // If the request is only partially accepted + // (i.e. when the server accepts only parts of the data and rejects the rest) + // the server MUST initialize the `partial_success` field and MUST + // set the `rejected_` with the number of items it rejected. + // + // Servers MAY also make use of the `partial_success` field to convey + // warnings/suggestions to senders even when the request was fully accepted. + // In such cases, the `rejected_` MUST have a value of `0` and + // the `error_message` MUST be non-empty. + // + // A `partial_success` message with an empty value (rejected_ = 0 and + // `error_message` = "") is equivalent to it not being set/present. Senders + // SHOULD interpret it the same way as in the full success case. + ExportMetricsPartialSuccess partial_success = 1; +} + +message ExportMetricsPartialSuccess { + // The number of rejected data points. + // + // A `rejected_` field holding a `0` value indicates that the + // request was fully accepted. + int64 rejected_data_points = 1; + + // A developer-facing human-readable message in English. It should be used + // either to explain why the server rejected parts of the data during a partial + // success or to convey warnings/suggestions during a full success. The message + // should offer guidance on how users can address such issues. + // + // error_message is an optional field. An error_message with an empty value + // is equivalent to it not being set. + string error_message = 2; +} diff --git a/otlp/proto/opentelemetry/proto/collector/trace/v1/trace_service.proto b/otlp/proto/opentelemetry/proto/collector/trace/v1/trace_service.proto new file mode 100644 index 000000000000..d6fe67f9e553 --- /dev/null +++ b/otlp/proto/opentelemetry/proto/collector/trace/v1/trace_service.proto @@ -0,0 +1,79 @@ +// Copyright 2019, OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package opentelemetry.proto.collector.trace.v1; + +import "opentelemetry/proto/trace/v1/trace.proto"; + +option csharp_namespace = "OpenTelemetry.Proto.Collector.Trace.V1"; +option java_multiple_files = true; +option java_package = "io.opentelemetry.proto.collector.trace.v1"; +option java_outer_classname = "TraceServiceProto"; +option go_package = "go.opentelemetry.io/proto/otlp/collector/trace/v1"; + +// Service that can be used to push spans between one Application instrumented with +// OpenTelemetry and a collector, or between a collector and a central collector (in this +// case spans are sent/received to/from multiple Applications). +service TraceService { + // For performance reasons, it is recommended to keep this RPC + // alive for the entire life of the application. + rpc Export(ExportTraceServiceRequest) returns (ExportTraceServiceResponse) {} +} + +message ExportTraceServiceRequest { + // An array of ResourceSpans. + // For data coming from a single resource this array will typically contain one + // element. Intermediary nodes (such as OpenTelemetry Collector) that receive + // data from multiple origins typically batch the data before forwarding further and + // in that case this array will contain multiple elements. + repeated opentelemetry.proto.trace.v1.ResourceSpans resource_spans = 1; +} + +message ExportTraceServiceResponse { + // The details of a partially successful export request. + // + // If the request is only partially accepted + // (i.e. when the server accepts only parts of the data and rejects the rest) + // the server MUST initialize the `partial_success` field and MUST + // set the `rejected_` with the number of items it rejected. + // + // Servers MAY also make use of the `partial_success` field to convey + // warnings/suggestions to senders even when the request was fully accepted. + // In such cases, the `rejected_` MUST have a value of `0` and + // the `error_message` MUST be non-empty. + // + // A `partial_success` message with an empty value (rejected_ = 0 and + // `error_message` = "") is equivalent to it not being set/present. Senders + // SHOULD interpret it the same way as in the full success case. + ExportTracePartialSuccess partial_success = 1; +} + +message ExportTracePartialSuccess { + // The number of rejected spans. + // + // A `rejected_` field holding a `0` value indicates that the + // request was fully accepted. + int64 rejected_spans = 1; + + // A developer-facing human-readable message in English. It should be used + // either to explain why the server rejected parts of the data during a partial + // success or to convey warnings/suggestions during a full success. The message + // should offer guidance on how users can address such issues. + // + // error_message is an optional field. An error_message with an empty value + // is equivalent to it not being set. + string error_message = 2; +} diff --git a/otlp/proto/opentelemetry/proto/common/v1/common.proto b/otlp/proto/opentelemetry/proto/common/v1/common.proto new file mode 100644 index 000000000000..ff8a21a1fa0e --- /dev/null +++ b/otlp/proto/opentelemetry/proto/common/v1/common.proto @@ -0,0 +1,81 @@ +// Copyright 2019, OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package opentelemetry.proto.common.v1; + +option csharp_namespace = "OpenTelemetry.Proto.Common.V1"; +option java_multiple_files = true; +option java_package = "io.opentelemetry.proto.common.v1"; +option java_outer_classname = "CommonProto"; +option go_package = "go.opentelemetry.io/proto/otlp/common/v1"; + +// AnyValue is used to represent any type of attribute value. AnyValue may contain a +// primitive value such as a string or integer or it may contain an arbitrary nested +// object containing arrays, key-value lists and primitives. +message AnyValue { + // The value is one of the listed fields. It is valid for all values to be unspecified + // in which case this AnyValue is considered to be "empty". + oneof value { + string string_value = 1; + bool bool_value = 2; + int64 int_value = 3; + double double_value = 4; + ArrayValue array_value = 5; + KeyValueList kvlist_value = 6; + bytes bytes_value = 7; + } +} + +// ArrayValue is a list of AnyValue messages. We need ArrayValue as a message +// since oneof in AnyValue does not allow repeated fields. +message ArrayValue { + // Array of values. The array may be empty (contain 0 elements). + repeated AnyValue values = 1; +} + +// KeyValueList is a list of KeyValue messages. We need KeyValueList as a message +// since `oneof` in AnyValue does not allow repeated fields. Everywhere else where we need +// a list of KeyValue messages (e.g. in Span) we use `repeated KeyValue` directly to +// avoid unnecessary extra wrapping (which slows down the protocol). The 2 approaches +// are semantically equivalent. +message KeyValueList { + // A collection of key/value pairs of key-value pairs. The list may be empty (may + // contain 0 elements). + // The keys MUST be unique (it is not allowed to have more than one + // value with the same key). + repeated KeyValue values = 1; +} + +// KeyValue is a key-value pair that is used to store Span attributes, Link +// attributes, etc. +message KeyValue { + string key = 1; + AnyValue value = 2; +} + +// InstrumentationScope is a message representing the instrumentation scope information +// such as the fully qualified name and version. +message InstrumentationScope { + // An empty instrumentation scope name means the name is unknown. + string name = 1; + string version = 2; + + // Additional attributes that describe the scope. [Optional]. + // Attribute keys MUST be unique (it is not allowed to have more than one + // attribute with the same key). + repeated KeyValue attributes = 3; + uint32 dropped_attributes_count = 4; +} diff --git a/otlp/proto/opentelemetry/proto/logs/v1/logs.proto b/otlp/proto/opentelemetry/proto/logs/v1/logs.proto new file mode 100644 index 000000000000..f9b97dd74510 --- /dev/null +++ b/otlp/proto/opentelemetry/proto/logs/v1/logs.proto @@ -0,0 +1,211 @@ +// Copyright 2020, OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package opentelemetry.proto.logs.v1; + +import "opentelemetry/proto/common/v1/common.proto"; +import "opentelemetry/proto/resource/v1/resource.proto"; + +option csharp_namespace = "OpenTelemetry.Proto.Logs.V1"; +option java_multiple_files = true; +option java_package = "io.opentelemetry.proto.logs.v1"; +option java_outer_classname = "LogsProto"; +option go_package = "go.opentelemetry.io/proto/otlp/logs/v1"; + +// LogsData represents the logs data that can be stored in a persistent storage, +// OR can be embedded by other protocols that transfer OTLP logs data but do not +// implement the OTLP protocol. +// +// The main difference between this message and collector protocol is that +// in this message there will not be any "control" or "metadata" specific to +// OTLP protocol. +// +// When new fields are added into this message, the OTLP request MUST be updated +// as well. +message LogsData { + // An array of ResourceLogs. + // For data coming from a single resource this array will typically contain + // one element. Intermediary nodes that receive data from multiple origins + // typically batch the data before forwarding further and in that case this + // array will contain multiple elements. + repeated ResourceLogs resource_logs = 1; +} + +// A collection of ScopeLogs from a Resource. +message ResourceLogs { + reserved 1000; + + // The resource for the logs in this message. + // If this field is not set then resource info is unknown. + opentelemetry.proto.resource.v1.Resource resource = 1; + + // A list of ScopeLogs that originate from a resource. + repeated ScopeLogs scope_logs = 2; + + // The Schema URL, if known. This is the identifier of the Schema that the resource data + // is recorded in. To learn more about Schema URL see + // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url + // This schema_url applies to the data in the "resource" field. It does not apply + // to the data in the "scope_logs" field which have their own schema_url field. + string schema_url = 3; +} + +// A collection of Logs produced by a Scope. +message ScopeLogs { + // The instrumentation scope information for the logs in this message. + // Semantically when InstrumentationScope isn't set, it is equivalent with + // an empty instrumentation scope name (unknown). + opentelemetry.proto.common.v1.InstrumentationScope scope = 1; + + // A list of log records. + repeated LogRecord log_records = 2; + + // The Schema URL, if known. This is the identifier of the Schema that the log data + // is recorded in. To learn more about Schema URL see + // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url + // This schema_url applies to all logs in the "logs" field. + string schema_url = 3; +} + +// Possible values for LogRecord.SeverityNumber. +enum SeverityNumber { + // UNSPECIFIED is the default SeverityNumber, it MUST NOT be used. + SEVERITY_NUMBER_UNSPECIFIED = 0; + SEVERITY_NUMBER_TRACE = 1; + SEVERITY_NUMBER_TRACE2 = 2; + SEVERITY_NUMBER_TRACE3 = 3; + SEVERITY_NUMBER_TRACE4 = 4; + SEVERITY_NUMBER_DEBUG = 5; + SEVERITY_NUMBER_DEBUG2 = 6; + SEVERITY_NUMBER_DEBUG3 = 7; + SEVERITY_NUMBER_DEBUG4 = 8; + SEVERITY_NUMBER_INFO = 9; + SEVERITY_NUMBER_INFO2 = 10; + SEVERITY_NUMBER_INFO3 = 11; + SEVERITY_NUMBER_INFO4 = 12; + SEVERITY_NUMBER_WARN = 13; + SEVERITY_NUMBER_WARN2 = 14; + SEVERITY_NUMBER_WARN3 = 15; + SEVERITY_NUMBER_WARN4 = 16; + SEVERITY_NUMBER_ERROR = 17; + SEVERITY_NUMBER_ERROR2 = 18; + SEVERITY_NUMBER_ERROR3 = 19; + SEVERITY_NUMBER_ERROR4 = 20; + SEVERITY_NUMBER_FATAL = 21; + SEVERITY_NUMBER_FATAL2 = 22; + SEVERITY_NUMBER_FATAL3 = 23; + SEVERITY_NUMBER_FATAL4 = 24; +} + +// LogRecordFlags represents constants used to interpret the +// LogRecord.flags field, which is protobuf 'fixed32' type and is to +// be used as bit-fields. Each non-zero value defined in this enum is +// a bit-mask. To extract the bit-field, for example, use an +// expression like: +// +// (logRecord.flags & LOG_RECORD_FLAGS_TRACE_FLAGS_MASK) +// +enum LogRecordFlags { + // The zero value for the enum. Should not be used for comparisons. + // Instead use bitwise "and" with the appropriate mask as shown above. + LOG_RECORD_FLAGS_DO_NOT_USE = 0; + + // Bits 0-7 are used for trace flags. + LOG_RECORD_FLAGS_TRACE_FLAGS_MASK = 0x000000FF; + + // Bits 8-31 are reserved for future use. +} + +// A log record according to OpenTelemetry Log Data Model: +// https://github.com/open-telemetry/oteps/blob/main/text/logs/0097-log-data-model.md +message LogRecord { + reserved 4; + + // time_unix_nano is the time when the event occurred. + // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January 1970. + // Value of 0 indicates unknown or missing timestamp. + fixed64 time_unix_nano = 1; + + // Time when the event was observed by the collection system. + // For events that originate in OpenTelemetry (e.g. using OpenTelemetry Logging SDK) + // this timestamp is typically set at the generation time and is equal to Timestamp. + // For events originating externally and collected by OpenTelemetry (e.g. using + // Collector) this is the time when OpenTelemetry's code observed the event measured + // by the clock of the OpenTelemetry code. This field MUST be set once the event is + // observed by OpenTelemetry. + // + // For converting OpenTelemetry log data to formats that support only one timestamp or + // when receiving OpenTelemetry log data by recipients that support only one timestamp + // internally the following logic is recommended: + // - Use time_unix_nano if it is present, otherwise use observed_time_unix_nano. + // + // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January 1970. + // Value of 0 indicates unknown or missing timestamp. + fixed64 observed_time_unix_nano = 11; + + // Numerical value of the severity, normalized to values described in Log Data Model. + // [Optional]. + SeverityNumber severity_number = 2; + + // The severity text (also known as log level). The original string representation as + // it is known at the source. [Optional]. + string severity_text = 3; + + // A value containing the body of the log record. Can be for example a human-readable + // string message (including multi-line) describing the event in a free form or it can + // be a structured data composed of arrays and maps of other values. [Optional]. + opentelemetry.proto.common.v1.AnyValue body = 5; + + // Additional attributes that describe the specific event occurrence. [Optional]. + // Attribute keys MUST be unique (it is not allowed to have more than one + // attribute with the same key). + repeated opentelemetry.proto.common.v1.KeyValue attributes = 6; + uint32 dropped_attributes_count = 7; + + // Flags, a bit field. 8 least significant bits are the trace flags as + // defined in W3C Trace Context specification. 24 most significant bits are reserved + // and must be set to 0. Readers must not assume that 24 most significant bits + // will be zero and must correctly mask the bits when reading 8-bit trace flag (use + // flags & LOG_RECORD_FLAGS_TRACE_FLAGS_MASK). [Optional]. + fixed32 flags = 8; + + // A unique identifier for a trace. All logs from the same trace share + // the same `trace_id`. The ID is a 16-byte array. An ID with all zeroes OR + // of length other than 16 bytes is considered invalid (empty string in OTLP/JSON + // is zero-length and thus is also invalid). + // + // This field is optional. + // + // The receivers SHOULD assume that the log record is not associated with a + // trace if any of the following is true: + // - the field is not present, + // - the field contains an invalid value. + bytes trace_id = 9; + + // A unique identifier for a span within a trace, assigned when the span + // is created. The ID is an 8-byte array. An ID with all zeroes OR of length + // other than 8 bytes is considered invalid (empty string in OTLP/JSON + // is zero-length and thus is also invalid). + // + // This field is optional. If the sender specifies a valid span_id then it SHOULD also + // specify a valid trace_id. + // + // The receivers SHOULD assume that the log record is not associated with a + // span if any of the following is true: + // - the field is not present, + // - the field contains an invalid value. + bytes span_id = 10; +} diff --git a/otlp/proto/opentelemetry/proto/resource/v1/resource.proto b/otlp/proto/opentelemetry/proto/resource/v1/resource.proto new file mode 100644 index 000000000000..6637560bc354 --- /dev/null +++ b/otlp/proto/opentelemetry/proto/resource/v1/resource.proto @@ -0,0 +1,37 @@ +// Copyright 2019, OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package opentelemetry.proto.resource.v1; + +import "opentelemetry/proto/common/v1/common.proto"; + +option csharp_namespace = "OpenTelemetry.Proto.Resource.V1"; +option java_multiple_files = true; +option java_package = "io.opentelemetry.proto.resource.v1"; +option java_outer_classname = "ResourceProto"; +option go_package = "go.opentelemetry.io/proto/otlp/resource/v1"; + +// Resource information. +message Resource { + // Set of attributes that describe the resource. + // Attribute keys MUST be unique (it is not allowed to have more than one + // attribute with the same key). + repeated opentelemetry.proto.common.v1.KeyValue attributes = 1; + + // dropped_attributes_count is the number of dropped attributes. If the value is 0, then + // no attributes were dropped. + uint32 dropped_attributes_count = 2; +} diff --git a/otlp/proto/opentelemetry/proto/trace/v1/trace.proto b/otlp/proto/opentelemetry/proto/trace/v1/trace.proto new file mode 100644 index 000000000000..5cb2f3ce1cd2 --- /dev/null +++ b/otlp/proto/opentelemetry/proto/trace/v1/trace.proto @@ -0,0 +1,355 @@ +// Copyright 2019, OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package opentelemetry.proto.trace.v1; + +import "opentelemetry/proto/common/v1/common.proto"; +import "opentelemetry/proto/resource/v1/resource.proto"; + +option csharp_namespace = "OpenTelemetry.Proto.Trace.V1"; +option java_multiple_files = true; +option java_package = "io.opentelemetry.proto.trace.v1"; +option java_outer_classname = "TraceProto"; +option go_package = "go.opentelemetry.io/proto/otlp/trace/v1"; + +// TracesData represents the traces data that can be stored in a persistent storage, +// OR can be embedded by other protocols that transfer OTLP traces data but do +// not implement the OTLP protocol. +// +// The main difference between this message and collector protocol is that +// in this message there will not be any "control" or "metadata" specific to +// OTLP protocol. +// +// When new fields are added into this message, the OTLP request MUST be updated +// as well. +message TracesData { + // An array of ResourceSpans. + // For data coming from a single resource this array will typically contain + // one element. Intermediary nodes that receive data from multiple origins + // typically batch the data before forwarding further and in that case this + // array will contain multiple elements. + repeated ResourceSpans resource_spans = 1; +} + +// A collection of ScopeSpans from a Resource. +message ResourceSpans { + reserved 1000; + + // The resource for the spans in this message. + // If this field is not set then no resource info is known. + opentelemetry.proto.resource.v1.Resource resource = 1; + + // A list of ScopeSpans that originate from a resource. + repeated ScopeSpans scope_spans = 2; + + // The Schema URL, if known. This is the identifier of the Schema that the resource data + // is recorded in. To learn more about Schema URL see + // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url + // This schema_url applies to the data in the "resource" field. It does not apply + // to the data in the "scope_spans" field which have their own schema_url field. + string schema_url = 3; +} + +// A collection of Spans produced by an InstrumentationScope. +message ScopeSpans { + // The instrumentation scope information for the spans in this message. + // Semantically when InstrumentationScope isn't set, it is equivalent with + // an empty instrumentation scope name (unknown). + opentelemetry.proto.common.v1.InstrumentationScope scope = 1; + + // A list of Spans that originate from an instrumentation scope. + repeated Span spans = 2; + + // The Schema URL, if known. This is the identifier of the Schema that the span data + // is recorded in. To learn more about Schema URL see + // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url + // This schema_url applies to all spans and span events in the "spans" field. + string schema_url = 3; +} + +// A Span represents a single operation performed by a single component of the system. +// +// The next available field id is 17. +message Span { + // A unique identifier for a trace. All spans from the same trace share + // the same `trace_id`. The ID is a 16-byte array. An ID with all zeroes OR + // of length other than 16 bytes is considered invalid (empty string in OTLP/JSON + // is zero-length and thus is also invalid). + // + // This field is required. + bytes trace_id = 1; + + // A unique identifier for a span within a trace, assigned when the span + // is created. The ID is an 8-byte array. An ID with all zeroes OR of length + // other than 8 bytes is considered invalid (empty string in OTLP/JSON + // is zero-length and thus is also invalid). + // + // This field is required. + bytes span_id = 2; + + // trace_state conveys information about request position in multiple distributed tracing graphs. + // It is a trace_state in w3c-trace-context format: https://www.w3.org/TR/trace-context/#tracestate-header + // See also https://github.com/w3c/distributed-tracing for more details about this field. + string trace_state = 3; + + // The `span_id` of this span's parent span. If this is a root span, then this + // field must be empty. The ID is an 8-byte array. + bytes parent_span_id = 4; + + // Flags, a bit field. + // + // Bits 0-7 (8 least significant bits) are the trace flags as defined in W3C Trace + // Context specification. To read the 8-bit W3C trace flag, use + // `flags & SPAN_FLAGS_TRACE_FLAGS_MASK`. + // + // See https://www.w3.org/TR/trace-context-2/#trace-flags for the flag definitions. + // + // Bits 8 and 9 represent the 3 states of whether a span's parent + // is remote. The states are (unknown, is not remote, is remote). + // To read whether the value is known, use `(flags & SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK) != 0`. + // To read whether the span is remote, use `(flags & SPAN_FLAGS_CONTEXT_IS_REMOTE_MASK) != 0`. + // + // When creating span messages, if the message is logically forwarded from another source + // with an equivalent flags fields (i.e., usually another OTLP span message), the field SHOULD + // be copied as-is. If creating from a source that does not have an equivalent flags field + // (such as a runtime representation of an OpenTelemetry span), the high 22 bits MUST + // be set to zero. + // Readers MUST NOT assume that bits 10-31 (22 most significant bits) will be zero. + // + // [Optional]. + fixed32 flags = 16; + + // A description of the span's operation. + // + // For example, the name can be a qualified method name or a file name + // and a line number where the operation is called. A best practice is to use + // the same display name at the same call point in an application. + // This makes it easier to correlate spans in different traces. + // + // This field is semantically required to be set to non-empty string. + // Empty value is equivalent to an unknown span name. + // + // This field is required. + string name = 5; + + // SpanKind is the type of span. Can be used to specify additional relationships between spans + // in addition to a parent/child relationship. + enum SpanKind { + // Unspecified. Do NOT use as default. + // Implementations MAY assume SpanKind to be INTERNAL when receiving UNSPECIFIED. + SPAN_KIND_UNSPECIFIED = 0; + + // Indicates that the span represents an internal operation within an application, + // as opposed to an operation happening at the boundaries. Default value. + SPAN_KIND_INTERNAL = 1; + + // Indicates that the span covers server-side handling of an RPC or other + // remote network request. + SPAN_KIND_SERVER = 2; + + // Indicates that the span describes a request to some remote service. + SPAN_KIND_CLIENT = 3; + + // Indicates that the span describes a producer sending a message to a broker. + // Unlike CLIENT and SERVER, there is often no direct critical path latency relationship + // between producer and consumer spans. A PRODUCER span ends when the message was accepted + // by the broker while the logical processing of the message might span a much longer time. + SPAN_KIND_PRODUCER = 4; + + // Indicates that the span describes consumer receiving a message from a broker. + // Like the PRODUCER kind, there is often no direct critical path latency relationship + // between producer and consumer spans. + SPAN_KIND_CONSUMER = 5; + } + + // Distinguishes between spans generated in a particular context. For example, + // two spans with the same name may be distinguished using `CLIENT` (caller) + // and `SERVER` (callee) to identify queueing latency associated with the span. + SpanKind kind = 6; + + // start_time_unix_nano is the start time of the span. On the client side, this is the time + // kept by the local machine where the span execution starts. On the server side, this + // is the time when the server's application handler starts running. + // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January 1970. + // + // This field is semantically required and it is expected that end_time >= start_time. + fixed64 start_time_unix_nano = 7; + + // end_time_unix_nano is the end time of the span. On the client side, this is the time + // kept by the local machine where the span execution ends. On the server side, this + // is the time when the server application handler stops running. + // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January 1970. + // + // This field is semantically required and it is expected that end_time >= start_time. + fixed64 end_time_unix_nano = 8; + + // attributes is a collection of key/value pairs. Note, global attributes + // like server name can be set using the resource API. Examples of attributes: + // + // "/http/user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36" + // "/http/server_latency": 300 + // "example.com/myattribute": true + // "example.com/score": 10.239 + // + // The OpenTelemetry API specification further restricts the allowed value types: + // https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/common/README.md#attribute + // Attribute keys MUST be unique (it is not allowed to have more than one + // attribute with the same key). + repeated opentelemetry.proto.common.v1.KeyValue attributes = 9; + + // dropped_attributes_count is the number of attributes that were discarded. Attributes + // can be discarded because their keys are too long or because there are too many + // attributes. If this value is 0, then no attributes were dropped. + uint32 dropped_attributes_count = 10; + + // Event is a time-stamped annotation of the span, consisting of user-supplied + // text description and key-value pairs. + message Event { + // time_unix_nano is the time the event occurred. + fixed64 time_unix_nano = 1; + + // name of the event. + // This field is semantically required to be set to non-empty string. + string name = 2; + + // attributes is a collection of attribute key/value pairs on the event. + // Attribute keys MUST be unique (it is not allowed to have more than one + // attribute with the same key). + repeated opentelemetry.proto.common.v1.KeyValue attributes = 3; + + // dropped_attributes_count is the number of dropped attributes. If the value is 0, + // then no attributes were dropped. + uint32 dropped_attributes_count = 4; + } + + // events is a collection of Event items. + repeated Event events = 11; + + // dropped_events_count is the number of dropped events. If the value is 0, then no + // events were dropped. + uint32 dropped_events_count = 12; + + // A pointer from the current span to another span in the same trace or in a + // different trace. For example, this can be used in batching operations, + // where a single batch handler processes multiple requests from different + // traces or when the handler receives a request from a different project. + message Link { + // A unique identifier of a trace that this linked span is part of. The ID is a + // 16-byte array. + bytes trace_id = 1; + + // A unique identifier for the linked span. The ID is an 8-byte array. + bytes span_id = 2; + + // The trace_state associated with the link. + string trace_state = 3; + + // attributes is a collection of attribute key/value pairs on the link. + // Attribute keys MUST be unique (it is not allowed to have more than one + // attribute with the same key). + repeated opentelemetry.proto.common.v1.KeyValue attributes = 4; + + // dropped_attributes_count is the number of dropped attributes. If the value is 0, + // then no attributes were dropped. + uint32 dropped_attributes_count = 5; + + // Flags, a bit field. + // + // Bits 0-7 (8 least significant bits) are the trace flags as defined in W3C Trace + // Context specification. To read the 8-bit W3C trace flag, use + // `flags & SPAN_FLAGS_TRACE_FLAGS_MASK`. + // + // See https://www.w3.org/TR/trace-context-2/#trace-flags for the flag definitions. + // + // Bits 8 and 9 represent the 3 states of whether the link is remote. + // The states are (unknown, is not remote, is remote). + // To read whether the value is known, use `(flags & SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK) != 0`. + // To read whether the link is remote, use `(flags & SPAN_FLAGS_CONTEXT_IS_REMOTE_MASK) != 0`. + // + // Readers MUST NOT assume that bits 10-31 (22 most significant bits) will be zero. + // When creating new spans, bits 10-31 (most-significant 22-bits) MUST be zero. + // + // [Optional]. + fixed32 flags = 6; + } + + // links is a collection of Links, which are references from this span to a span + // in the same or different trace. + repeated Link links = 13; + + // dropped_links_count is the number of dropped links after the maximum size was + // enforced. If this value is 0, then no links were dropped. + uint32 dropped_links_count = 14; + + // An optional final status for this span. Semantically when Status isn't set, it means + // span's status code is unset, i.e. assume STATUS_CODE_UNSET (code = 0). + Status status = 15; +} + +// The Status type defines a logical error model that is suitable for different +// programming environments, including REST APIs and RPC APIs. +message Status { + reserved 1; + + // A developer-facing human readable error message. + string message = 2; + + // For the semantics of status codes see + // https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/api.md#set-status + enum StatusCode { + // The default status. + STATUS_CODE_UNSET = 0; + // The Span has been validated by an Application developer or Operator to + // have completed successfully. + STATUS_CODE_OK = 1; + // The Span contains an error. + STATUS_CODE_ERROR = 2; + }; + + // The status code. + StatusCode code = 3; +} + +// SpanFlags represents constants used to interpret the +// Span.flags field, which is protobuf 'fixed32' type and is to +// be used as bit-fields. Each non-zero value defined in this enum is +// a bit-mask. To extract the bit-field, for example, use an +// expression like: +// +// (span.flags & SPAN_FLAGS_TRACE_FLAGS_MASK) +// +// See https://www.w3.org/TR/trace-context-2/#trace-flags for the flag definitions. +// +// Note that Span flags were introduced in version 1.1 of the +// OpenTelemetry protocol. Older Span producers do not set this +// field, consequently consumers should not rely on the absence of a +// particular flag bit to indicate the presence of a particular feature. +enum SpanFlags { + // The zero value for the enum. Should not be used for comparisons. + // Instead use bitwise "and" with the appropriate mask as shown above. + SPAN_FLAGS_DO_NOT_USE = 0; + + // Bits 0-7 are used for trace flags. + SPAN_FLAGS_TRACE_FLAGS_MASK = 0x000000FF; + + // Bits 8 and 9 are used to indicate that the parent span or link span is remote. + // Bit 8 (`HAS_IS_REMOTE`) indicates whether the value is known. + // Bit 9 (`IS_REMOTE`) indicates whether the span or link is remote. + SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK = 0x00000100; + SPAN_FLAGS_CONTEXT_IS_REMOTE_MASK = 0x00000200; + + // Bits 10-31 are reserved for future use. +} diff --git a/otlp/src/otlp/logs/component.cpp b/otlp/src/otlp/logs/component.cpp new file mode 100644 index 000000000000..997c6c50d1d5 --- /dev/null +++ b/otlp/src/otlp/logs/component.cpp @@ -0,0 +1,99 @@ +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "logger.hpp" + +#include + +USERVER_NAMESPACE_BEGIN + +namespace otlp { + +LoggerComponent::LoggerComponent(const components::ComponentConfig& config, + const components::ComponentContext& context) { + auto& client_factory = + context.FindComponent() + .GetFactory(); + + auto endpoint = config["endpoint"].As(); + auto client = std::make_shared< + opentelemetry::proto::collector::logs::v1::LogsServiceClient>( + client_factory.MakeClient< + opentelemetry::proto::collector::logs::v1::LogsServiceClient>( + "otlp-logger", endpoint)); + + auto trace_client = std::make_shared< + opentelemetry::proto::collector::trace::v1::TraceServiceClient>( + client_factory.MakeClient< + opentelemetry::proto::collector::trace::v1::TraceServiceClient>( + "otlp-tracer", endpoint)); + + LoggerConfig logger_config; + logger_config.max_queue_size = config["max-queue-size"].As(65535); + logger_config.max_batch_delay = + config["max-batch-delay"].As(100); + logger_config.service_name = + config["service-name"].As("unknown_service"); + logger_config.log_level = + config["log-level"].As(); + logger_ = std::make_shared(std::move(client), std::move(trace_client), + std::move(logger_config)); + + // We must init after the default logger is initialized + context.FindComponent(); + + logging::impl::SetDefaultLoggerRef(*logger_); +} + +LoggerComponent::~LoggerComponent() { + std::cerr << "Destroying default logger\n"; + logging::impl::SetDefaultLoggerRef(logging::GetNullLogger()); +} + +yaml_config::Schema LoggerComponent::GetStaticConfigSchema() { + return yaml_config::MergeSchemas(R"( +type: object +description: > + OpenTelemetry logger component +additionalProperties: false +properties: + endpoint: + type: string + description: > + Hostname:port of otel collector (gRPC). + log-level: + type: string + description: log level + max-queue-size: + type: integer + description: max async queue size + max-batch-delay: + type: string + description: max delay between send batches (e.g. 100ms or 1s) + service-name: + type: string + description: service name + attributes: + type: object + description: extra OTLP attributes + properties: {} + additionalProperties: + type: string + description: attribute value + +)"); +} + +} // namespace otlp + +USERVER_NAMESPACE_END diff --git a/otlp/src/otlp/logs/logger.cpp b/otlp/src/otlp/logs/logger.cpp new file mode 100644 index 000000000000..7ca1fa4e73ea --- /dev/null +++ b/otlp/src/otlp/logs/logger.cpp @@ -0,0 +1,325 @@ +#include "logger.hpp" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +USERVER_NAMESPACE_BEGIN + +namespace otlp { + +namespace { +constexpr std::string_view kTelemetrySdkLanguage = "telemetry.sdk.language"; +constexpr std::string_view kTelemetrySdkName = "telemetry.sdk.name"; +constexpr std::string_view kServiceName = "service.name"; + +constexpr std::string_view kStopwatchNameEq = "stopwatch_name="; + +const std::string kTimestampFormat = "%Y-%m-%dT%H:%M:%E*S"; +} // namespace + +Logger::Logger( + std::shared_ptr< + opentelemetry::proto::collector::logs::v1::LogsServiceClient> + client, + std::shared_ptr< + opentelemetry::proto::collector::trace::v1::TraceServiceClient> + trace_client, + LoggerConfig&& config) + : LoggerBase(logging::Format::kTskv), + client_(std::move(client)), + trace_client_(std::move(trace_client)), + config_(std::move(config)), + queue_(Queue::Create(config_.max_queue_size)), + queue_producer_(queue_->GetMultiProducer()) { + SetLevel(config_.log_level); + Log(logging::Level::kInfo, "OTLP logger has started"); + std::cerr << "OTLP logger has started\n"; + + sender_task_ = engine::CriticalAsyncNoSpan( + [this, consumer = queue_->GetConsumer()]() mutable { + SendingLoop(consumer); + }); + ; +} + +Logger::~Logger() { sender_task_.SyncCancel(); } + +void Logger::Log(logging::Level, std::string_view msg) { + // Trim trailing \n + if (msg.size() > 1 && msg[msg.size() - 1] == '\n') { + msg = msg.substr(0, msg.size() - 1); + } + + std::vector key_values = + utils::text::SplitIntoStringViewVector(msg, "\t"); + + if (IsTracingEntry(key_values)) { + HandleTracing(key_values); + } else { + HandleLog(key_values); + } +} + +void Logger::PrependCommonTags(logging::impl::TagWriter writer) const { + logging::impl::default_::PrependCommonTags(writer); +} + +bool Logger::DoShouldLog(logging::Level level) const noexcept { + return logging::impl::default_::DoShouldLog(level); +} + +bool Logger::IsTracingEntry( + const std::vector& key_values) const { + for (const auto& key_value : key_values) { + if (key_value.substr(0, kStopwatchNameEq.size()) == kStopwatchNameEq) + return true; + } + return false; +} + +void Logger::HandleLog(const std::vector& key_values) { + ::opentelemetry::proto::logs::v1::LogRecord log_records; + std::string text; + std::string trace_id; + std::string span_id; + std::string level; + std::chrono::system_clock::time_point timestamp; + + for (const auto key_value : key_values) { + auto eq_pos = key_value.find('='); + if (eq_pos == std::string::npos) continue; + + auto key = key_value.substr(0, eq_pos); + auto value = key_value.substr(eq_pos + 1); + + if (key == "text") { + text = std::string{value}; + continue; + } + if (key == "trace_id") { + trace_id = utils::encoding::FromHex(value); + continue; + } + if (key == "span_id") { + span_id = utils::encoding::FromHex(value); + continue; + } + if (key == "timestamp") { + timestamp = utils::datetime::Stringtime(std::string{value}, + utils::datetime::kDefaultTimezone, + kTimestampFormat); + continue; + } + if (key == "level") { + level = value; + continue; + } + + auto attributes = log_records.add_attributes(); + attributes->set_key(std::string{key}); + attributes->mutable_value()->set_string_value(std::string{value}); + } + + log_records.set_severity_text(grpc::string(level)); + log_records.mutable_body()->set_string_value(grpc::string(text)); + log_records.set_trace_id(std::move(trace_id)); + log_records.set_span_id(std::move(span_id)); + + auto nanoseconds = std::chrono::duration_cast( + timestamp.time_since_epoch()); + log_records.set_time_unix_nano(nanoseconds.count()); + + // Drop a log if overflown + [[maybe_unused]] auto ok = + queue_producer_.PushNoblock(std::move(log_records)); + + // TODO: count drops +} + +void Logger::HandleTracing(const std::vector& key_values) { + ::opentelemetry::proto::trace::v1::Span span; + std::string trace_id; + std::string span_id; + std::string parent_span_id; + std::string_view level; + std::string_view stopwatch_name; + + std::string start_timestamp; + std::string total_time; + + for (const auto key_value : key_values) { + auto eq_pos = key_value.find('='); + if (eq_pos == std::string::npos) continue; + + auto key = key_value.substr(0, eq_pos); + auto value = key_value.substr(eq_pos + 1); + + if (key == "trace_id") { + trace_id = utils::encoding::FromHex(value); + continue; + } + if (key == "span_id") { + span_id = utils::encoding::FromHex(value); + continue; + } + if (key == "parent_span_id") { + parent_span_id = utils::encoding::FromHex(value); + continue; + } + if (key == "stopwatch_name") { + stopwatch_name = value; + continue; + } + if (key == "total_time") { + total_time = value; + continue; + } + if (key == "start_timestamp") { + start_timestamp = value; + continue; + } + if (key == "level") { + level = value; + continue; + } + if (key == "timestamp" || key == "text") { + continue; + } + + auto attributes = span.add_attributes(); + attributes->set_key(std::string{key}); + attributes->mutable_value()->set_string_value(std::string{value}); + } + + span.set_trace_id(std::string(trace_id)); + span.set_span_id(std::string(span_id)); + span.set_parent_span_id(std::string(parent_span_id)); + span.set_name(std::string(stopwatch_name)); + + auto start_timestamp_double = std::stod(start_timestamp); + span.set_start_time_unix_nano(start_timestamp_double * 1'000'000'000); + span.set_end_time_unix_nano((start_timestamp_double + std::stod(total_time)) * + 1'000'000'000); + + // Drop a trace if overflown + [[maybe_unused]] auto ok = queue_producer_.PushNoblock(std::move(span)); + + // TODO: count drops +} + +void Logger::SendingLoop(Queue::Consumer& consumer) { + // Create dummy span to completely disable logging in current coroutine + tracing::Span span(""); + span.SetLocalLogLevel(logging::Level::kNone); + + ::opentelemetry::proto::collector::logs::v1::ExportLogsServiceRequest + log_request; + auto resource_logs = log_request.add_resource_logs(); + auto scope_logs = resource_logs->add_scope_logs(); + FillAttributes(*resource_logs->mutable_resource()); + + ::opentelemetry::proto::collector::trace::v1::ExportTraceServiceRequest + trace_request; + auto resource_spans = trace_request.add_resource_spans(); + auto scope_spans = resource_spans->add_scope_spans(); + FillAttributes(*resource_spans->mutable_resource()); + + Action action{}; + while (consumer.Pop(action)) { + scope_logs->clear_log_records(); + scope_spans->clear_spans(); + + auto deadline = engine::Deadline::FromDuration(config_.max_batch_delay); + + do { + std::visit( + utils::Overloaded{ + [&scope_spans]( + const opentelemetry::proto::trace::v1::Span& action) { + auto span = scope_spans->add_spans(); + *span = action; + }, + [&scope_logs]( + const opentelemetry::proto::logs::v1::LogRecord& action) { + auto log_records = scope_logs->add_log_records(); + *log_records = action; + }}, + action); + } while (consumer.Pop(action, deadline)); + + DoLog(log_request); + DoTrace(trace_request); + } +} + +void Logger::FillAttributes( + ::opentelemetry::proto::resource::v1::Resource& resource) { + { + auto* attr = resource.add_attributes(); + attr->set_key(std::string{kTelemetrySdkLanguage}); + attr->mutable_value()->set_string_value("cpp"); + } + + { + auto* attr = resource.add_attributes(); + attr->set_key(std::string{kTelemetrySdkName}); + attr->mutable_value()->set_string_value("userver"); + } + + { + auto* attr = resource.add_attributes(); + attr->set_key(std::string{kServiceName}); + attr->mutable_value()->set_string_value(std::string{config_.service_name}); + } + + for (const auto& [key, value] : config_.attributes) { + auto* attr = resource.add_attributes(); + attr->set_key(std::string{key}); + attr->mutable_value()->set_string_value(std::string{value}); + } +} + +void Logger::DoLog( + const opentelemetry::proto::collector::logs::v1::ExportLogsServiceRequest& + request) { + try { + auto call = client_->Export(request); + auto response = call.Finish(); + } catch (const ugrpc::client::RpcCancelledError&) { + std::cerr << "Stopping OTLP sender task\n"; + throw; + } catch (const std::exception& e) { + std::cerr << "Failed to write down OTLP log(s): " << e.what() + << typeid(e).name() << "\n"; + } + // TODO: count exceptions +} + +void Logger::DoTrace( + const opentelemetry::proto::collector::trace::v1::ExportTraceServiceRequest& + request) { + try { + auto call = trace_client_->Export(request); + auto response = call.Finish(); + } catch (const ugrpc::client::RpcCancelledError&) { + std::cerr << "Stopping OTLP sender task\n"; + throw; + } catch (const std::exception& e) { + std::cerr << "Failed to write down OTLP trace(s): " << e.what() + << typeid(e).name() << "\n"; + } + // TODO: count exceptions +} + +} // namespace otlp + +USERVER_NAMESPACE_END diff --git a/otlp/src/otlp/logs/logger.hpp b/otlp/src/otlp/logs/logger.hpp new file mode 100644 index 000000000000..ee137279ac42 --- /dev/null +++ b/otlp/src/otlp/logs/logger.hpp @@ -0,0 +1,81 @@ +#pragma once + +#include +#include + +#include +#include + +#include +#include +#include + +USERVER_NAMESPACE_BEGIN + +namespace otlp { + +struct LoggerConfig { + size_t max_queue_size{10000}; + std::chrono::milliseconds max_batch_delay{}; + + std::string service_name; + std::unordered_map attributes; + logging::Level log_level{logging::Level::kInfo}; +}; + +class Logger final : public logging::impl::LoggerBase { + public: + explicit Logger( + std::shared_ptr< + opentelemetry::proto::collector::logs::v1::LogsServiceClient> + client, + std::shared_ptr< + opentelemetry::proto::collector::trace::v1::TraceServiceClient> + trace_client, + LoggerConfig&& config); + + ~Logger() override; + + void Log(logging::Level level, std::string_view msg) override; + + void PrependCommonTags(logging::impl::TagWriter writer) const override; + + protected: + bool DoShouldLog(logging::Level level) const noexcept override; + + private: + using Action = std::variant<::opentelemetry::proto::logs::v1::LogRecord, + ::opentelemetry::proto::trace::v1::Span>; + using Queue = concurrent::NonFifoMpscQueue; + + void SendingLoop(Queue::Consumer& consumer); + + bool IsTracingEntry(const std::vector& key_values) const; + + void HandleTracing(const std::vector& key_values); + + void HandleLog(const std::vector& key_values); + + void FillAttributes(::opentelemetry::proto::resource::v1::Resource& resource); + + void DoLog( + const opentelemetry::proto::collector::logs::v1::ExportLogsServiceRequest& + request); + + void DoTrace(const opentelemetry::proto::collector::trace::v1:: + ExportTraceServiceRequest& request); + + std::shared_ptr + client_; + std::shared_ptr< + opentelemetry::proto::collector::trace::v1::TraceServiceClient> + trace_client_; + const LoggerConfig config_; + std::shared_ptr queue_; + Queue::MultiProducer queue_producer_; + engine::Task sender_task_; // Must be the last member +}; + +} // namespace otlp + +USERVER_NAMESPACE_END diff --git a/samples/CMakeLists.txt b/samples/CMakeLists.txt index 0f00961da1da..24f9a111d050 100644 --- a/samples/CMakeLists.txt +++ b/samples/CMakeLists.txt @@ -16,6 +16,11 @@ if (USERVER_FEATURE_CHAOTIC) add_dependencies(${PROJECT_NAME} ${PROJECT_NAME}-chaotic_service) endif() +if (USERVER_FEATURE_OTLP) + add_subdirectory(otlp_service) + add_dependencies(${PROJECT_NAME} ${PROJECT_NAME}-otlp_service) +endif() + if (USERVER_FEATURE_GRPC) add_subdirectory(grpc_service) add_dependencies(${PROJECT_NAME} ${PROJECT_NAME}-grpc_service) diff --git a/samples/otlp_service/CMakeLists.txt b/samples/otlp_service/CMakeLists.txt new file mode 100644 index 000000000000..79c81f703f0b --- /dev/null +++ b/samples/otlp_service/CMakeLists.txt @@ -0,0 +1,6 @@ +project(userver-samples-otlp_service CXX) + +add_executable(${PROJECT_NAME} "otlp_service.cpp") +target_link_libraries(${PROJECT_NAME} userver::otlp) + +userver_testsuite_add_simple() diff --git a/samples/otlp_service/config_vars.yaml b/samples/otlp_service/config_vars.yaml new file mode 100644 index 000000000000..f5e3d5fc236f --- /dev/null +++ b/samples/otlp_service/config_vars.yaml @@ -0,0 +1,18 @@ +# yaml +server-name: sample-otlp-service 1.0 +service-name: sample-otlp-service +logger-level: info + +config-server-url: http://localhost:8083/ +server-port: 8085 +monitor-server-port: 8086 + +testsuite-enabled: false + +userver-dumps-root: /var/cache/otlp_service/userver-dumps/ +access-log-path: /var/log/otlp_service/access.log +access-tskv-log-path: /var/log/otlp_service/access_tskv.log +default-log-path: /var/log/otlp_service/server.log +secdist-path: /etc/otlp_service/secure_data.json + +config-cache: /var/cache/otlp_service/config_cache.json diff --git a/samples/otlp_service/otlp_service.cpp b/samples/otlp_service/otlp_service.cpp new file mode 100644 index 000000000000..7c54be9a0192 --- /dev/null +++ b/samples/otlp_service/otlp_service.cpp @@ -0,0 +1,26 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +int main(int argc, char* argv[]) { + const auto component_list = + components::MinimalServerComponentList() + .Append() + .Append() + .Append() + .Append() + .Append(); + return utils::DaemonMain(argc, argv, component_list); +} diff --git a/samples/otlp_service/secure_data.json b/samples/otlp_service/secure_data.json new file mode 100644 index 000000000000..0967ef424bce --- /dev/null +++ b/samples/otlp_service/secure_data.json @@ -0,0 +1 @@ +{} diff --git a/samples/otlp_service/static_config.yaml b/samples/otlp_service/static_config.yaml new file mode 100644 index 000000000000..56541843be37 --- /dev/null +++ b/samples/otlp_service/static_config.yaml @@ -0,0 +1,63 @@ +# yaml +components_manager: + components: + # The required common components + logging: + fs-task-processor: fs-task-processor + loggers: {} + # default: + # file_path: '@stderr' + # level: debug + # overflow_behavior: discard + +# /// [gRPC sample - static config client] +# yaml + # Creates gRPC clients + grpc-client-factory: + # The TaskProcessor for blocking connection initiation + task-processor: grpc-blocking-task-processor + + # Optional channel parameters for gRPC Core + # https://grpc.github.io/grpc/core/group__grpc__arg__keys.html + channel-args: {} + + otlp-logger: + endpoint: '0.0.0.0:4317' + service-name: 'otlp-example' + log-level: 'debug' + + handler-server-monitor: + path: /metrics + method: GET + task_processor: main-task-processor + format: prometheus + handler-ping: + path: /ping + method: GET + task_processor: main-task-processor + throttling_enabled: false + url_trailing_slash: strict-match + + server: + listener: + port: 8092 + task_processor: main-task-processor + listener-monitor: + port: 8093 + task_processor: main-task-processor + + testsuite-support: + + default_task_processor: main-task-processor # Task processor in which components start. + +# /// [gRPC sample - task processor] +# yaml + task_processors: + grpc-blocking-task-processor: # For blocking gRPC channel creation + worker_threads: 2 + thread_name: grpc-worker +# /// [gRPC sample - task processor] + main-task-processor: # For non-blocking operations + worker_threads: 8 + fs-task-processor: # For blocking filesystem operations + worker_threads: 2 diff --git a/samples/otlp_service/testsuite/conftest.py b/samples/otlp_service/testsuite/conftest.py new file mode 100644 index 000000000000..41ffab87fc60 --- /dev/null +++ b/samples/otlp_service/testsuite/conftest.py @@ -0,0 +1,4 @@ +# /// [registration] +# Adding a plugin from userver/testsuite/pytest_plugins/ +pytest_plugins = ['pytest_userver.plugins.core'] +# /// [registration] diff --git a/samples/otlp_service/testsuite/test_hello.py b/samples/otlp_service/testsuite/test_hello.py new file mode 100644 index 000000000000..a5e7edf1aa9d --- /dev/null +++ b/samples/otlp_service/testsuite/test_hello.py @@ -0,0 +1,3 @@ +async def test_ping(service_client): + response = await service_client.get('/ping') + assert response.status == 200 diff --git a/testsuite/pytest_plugins/pytest_userver/plugins/config.py b/testsuite/pytest_plugins/pytest_userver/plugins/config.py index 8532e3ee3191..e6b6415a6748 100644 --- a/testsuite/pytest_plugins/pytest_userver/plugins/config.py +++ b/testsuite/pytest_plugins/pytest_userver/plugins/config.py @@ -33,6 +33,7 @@ 'userver_config_http_server', 'userver_config_http_client', 'userver_config_logging', + 'userver_config_logging_otlp', 'userver_config_testsuite', 'userver_config_secdist', 'userver_config_testsuite_middleware', @@ -453,6 +454,24 @@ def _patch_config(config_yaml, config_vars): return _patch_config +@pytest.fixture(scope='session') +def userver_config_logging_otlp(): + """ + Returns a function that adjusts the static configuration file for testsuite. + Sets the `otlp-logger.load-enabled` to `false` to disable OTLP logging and + leave the default file logger. + + @ingroup userver_testsuite_fixtures + """ + + def _patch_config(config_yaml, config_vars): + components = config_yaml['components_manager']['components'] + if 'otlp-logger' in components: + components['otlp-logger']['load-enabled'] = False + + return _patch_config + + @pytest.fixture(scope='session') def userver_config_testsuite(pytestconfig, mockserver_info): """ From 78dd0669c50d0c5a99528a60737fd0e281300a34 Mon Sep 17 00:00:00 2001 From: antonyzhilin Date: Fri, 12 Jul 2024 09:33:05 +0300 Subject: [PATCH 019/629] refactor engine: use GetSignalOrAppend in TaskContext::Wait cd74cfd67bdda06e50f578caf4350ed639fd9a06 --- .../userver/engine/impl/wait_list_fwd.hpp | 3 +- core/src/engine/impl/generic_wait_list.cpp | 80 +++++++++++-------- core/src/engine/impl/generic_wait_list.hpp | 18 ++++- core/src/engine/task/task_context.cpp | 72 +++-------------- core/src/engine/task/task_context.hpp | 5 +- 5 files changed, 70 insertions(+), 108 deletions(-) diff --git a/core/include/userver/engine/impl/wait_list_fwd.hpp b/core/include/userver/engine/impl/wait_list_fwd.hpp index ca0382b1c876..50a33b1329ac 100644 --- a/core/include/userver/engine/impl/wait_list_fwd.hpp +++ b/core/include/userver/engine/impl/wait_list_fwd.hpp @@ -1,6 +1,5 @@ #pragma once -#include #include USERVER_NAMESPACE_BEGIN @@ -16,7 +15,7 @@ class WaitListLight; using FastPimplWaitListLight = utils::FastPimpl; class GenericWaitList; -using FastPimplGenericWaitList = utils::FastPimpl; +using FastPimplGenericWaitList = utils::FastPimpl; } // namespace engine::impl diff --git a/core/src/engine/impl/generic_wait_list.cpp b/core/src/engine/impl/generic_wait_list.cpp index 148d0a62b6db..b900d01804ca 100644 --- a/core/src/engine/impl/generic_wait_list.cpp +++ b/core/src/engine/impl/generic_wait_list.cpp @@ -8,59 +8,69 @@ USERVER_NAMESPACE_BEGIN namespace engine::impl { -namespace { - -auto CreateWaitList(Task::WaitMode wait_mode) noexcept { - using ReturnType = std::variant; +auto GenericWaitList::CreateWaitList(Task::WaitMode wait_mode) noexcept { + using ReturnType = std::variant; if (wait_mode == Task::WaitMode::kSingleWaiter) { return ReturnType{std::in_place_type}; } - UASSERT_MSG(wait_mode == Task::WaitMode::kMultipleWaiters, "Unexpected Task::WaitMode"); - return ReturnType{std::in_place_type}; + return ReturnType{std::in_place_type}; } -} // namespace - GenericWaitList::GenericWaitList(Task::WaitMode wait_mode) noexcept : waiters_(CreateWaitList(wait_mode)) {} -// noexcept: waiters_ are never valueless_by_exception -void GenericWaitList::Append( - boost::intrusive_ptr context) noexcept { - std::visit(utils::Overloaded{[&context](WaitListLight& ws) { - ws.Append(std::move(context)); - }, - [&context](WaitList& ws) { - WaitList::Lock lock{ws}; - ws.Append(lock, std::move(context)); - }}, - waiters_); +bool GenericWaitList::GetSignalOrAppend( + boost::intrusive_ptr&& context) noexcept { + return utils::Visit( + waiters_, // + [&context](WaitListLight& ws) { + return ws.GetSignalOrAppend(std::move(context)); + }, + [&context](WaitListAndSignal& ws) { + if (ws.signal.load()) return true; + WaitList::Lock lock{ws.wl}; + if (ws.signal.load()) return true; + ws.wl.Append(lock, std::move(context)); + return false; + }); } // noexcept: waiters_ are never valueless_by_exception -void GenericWaitList::Remove(impl::TaskContext& context) noexcept { - std::visit( - utils::Overloaded{[&context](WaitListLight& ws) { ws.Remove(context); }, - [&context](WaitList& ws) { - WaitList::Lock lock{ws}; - ws.Remove(lock, context); - }}, - waiters_); +void GenericWaitList::Remove(TaskContext& context) noexcept { + utils::Visit( + waiters_, // + [&context](WaitListLight& ws) { ws.Remove(context); }, + [&context](WaitListAndSignal& ws) { + WaitList::Lock lock{ws.wl}; + ws.wl.Remove(lock, context); + }); +} + +void GenericWaitList::SetSignalAndWakeupAll() { + utils::Visit( + waiters_, // + [](WaitListLight& ws) { ws.SetSignalAndWakeupOne(); }, + [](WaitListAndSignal& ws) { + if (ws.signal.load()) return; + WaitList::Lock lock{ws.wl}; + if (ws.signal.load()) return; + // seq_cst is important for the "Append-Check-Wakeup" sequence. + ws.signal.store(true, std::memory_order_seq_cst); + ws.wl.WakeupAll(lock); + }); } -void GenericWaitList::WakeupAll() { - std::visit(utils::Overloaded{[](WaitListLight& ws) { ws.WakeupOne(); }, - [](WaitList& ws) { - WaitList::Lock lock{ws}; - ws.WakeupAll(lock); - }}, - waiters_); +bool GenericWaitList::IsSignaled() const noexcept { + return utils::Visit( + waiters_, // + [](const WaitListLight& ws) { return ws.IsSignaled(); }, + [](const WaitListAndSignal& ws) { return ws.signal.load(); }); } bool GenericWaitList::IsShared() const noexcept { - return std::holds_alternative(waiters_); + return std::holds_alternative(waiters_); } } // namespace engine::impl diff --git a/core/src/engine/impl/generic_wait_list.hpp b/core/src/engine/impl/generic_wait_list.hpp index b4b62869a89e..72fd3f40606d 100644 --- a/core/src/engine/impl/generic_wait_list.hpp +++ b/core/src/engine/impl/generic_wait_list.hpp @@ -1,5 +1,6 @@ #pragma once +#include #include #include @@ -14,16 +15,25 @@ class GenericWaitList final { public: explicit GenericWaitList(Task::WaitMode wait_mode) noexcept; - void Append(boost::intrusive_ptr context) noexcept; + bool GetSignalOrAppend(boost::intrusive_ptr&& context) noexcept; - void Remove(impl::TaskContext& context) noexcept; + void Remove(TaskContext& context) noexcept; - void WakeupAll(); + void SetSignalAndWakeupAll(); + + bool IsSignaled() const noexcept; bool IsShared() const noexcept; private: - std::variant waiters_; + struct WaitListAndSignal final { + std::atomic signal{false}; + WaitList wl{}; + }; + + static auto CreateWaitList(Task::WaitMode wait_mode) noexcept; + + std::variant waiters_; }; } // namespace engine::impl diff --git a/core/src/engine/task/task_context.cpp b/core/src/engine/task/task_context.cpp index 3243bb60c2fa..8988862813dc 100644 --- a/core/src/engine/task/task_context.cpp +++ b/core/src/engine/task/task_context.cpp @@ -159,6 +159,10 @@ bool TaskContext::IsSharedWaitAllowed() const { return finish_waiters_->IsShared(); } +bool TaskContext::IsFinished() const noexcept { + return finish_waiters_->IsSignaled(); +} + void TaskContext::SetDetached(DetachedTasksSyncBlock::Token& token) noexcept { DetachedTasksSyncBlock::Token* expected = nullptr; if (!detached_token_.compare_exchange_strong(expected, &token)) { @@ -234,7 +238,7 @@ void TaskContext::DoStep() { } SetState(new_state); deadline_timer_.Finalize(); - finish_waiters_->WakeupAll(); + finish_waiters_->SetSignalAndWakeupAll(); TraceStateTransition(new_state); } break; @@ -551,12 +555,7 @@ bool TaskContext::IsReady() const noexcept { return IsFinished(); } EarlyWakeup TaskContext::TryAppendWaiter(TaskContext& waiter) { if (&waiter == this) ReportDeadlock(); - finish_waiters_->Append(&waiter); - if (IsFinished()) { - finish_waiters_->WakeupAll(); - return EarlyWakeup{true}; - } - return EarlyWakeup{false}; + return EarlyWakeup{finish_waiters_->GetSignalOrAppend(&waiter)}; } void TaskContext::RemoveWaiter(TaskContext& waiter) noexcept { @@ -608,62 +607,9 @@ TaskContext::WakeupSource TaskContext::GetPrimaryWakeupSource( bool TaskContext::WasStartedAsCritical() const { return is_critical_; } void TaskContext::SetState(Task::State new_state) { - auto old_state = Task::State::kNew; - - // CAS optimization - switch (new_state) { - case Task::State::kQueued: - old_state = Task::State::kSuspended; - break; - case Task::State::kRunning: - old_state = Task::State::kQueued; - break; - case Task::State::kSuspended: - case Task::State::kCompleted: - old_state = Task::State::kRunning; - break; - case Task::State::kCancelled: - old_state = Task::State::kSuspended; - break; - case Task::State::kInvalid: - case Task::State::kNew: - UASSERT(!"Invalid new task state"); - } - - if (new_state == Task::State::kRunning || - new_state == Task::State::kSuspended) { - if (new_state == Task::State::kRunning) { - UASSERT(IsCurrent()); - } else { - UASSERT(current_task::GetCurrentTaskContextUnchecked() == nullptr); - } - UASSERT(old_state == state_); - // For kRunning we don't care other threads see old state_ (kQueued) for - // some time. - // For kSuspended synchronization point is DoStep()'s - // sleep_state_.FetchOr(). - state_.store(new_state, std::memory_order_relaxed); - return; - } - if (new_state == Task::State::kQueued) { - UASSERT(old_state == state_ || state_ == Task::State::kNew); - // Synchronization point is TaskProcessor::Schedule() - state_.store(new_state, std::memory_order_relaxed); - return; - } - - // use strong CAS here to avoid losing transitions to finished - while (!state_.compare_exchange_strong(old_state, new_state)) { - if (old_state == new_state) { - // someone else did the job - return; - } - if (old_state == Task::State::kCompleted || - old_state == Task::State::kCancelled) { - // leave as finished, do not wakeup - return; - } - } + // 'release', because if someone detects kCompleted or kCancelled by running + // in a loop, they should acquire the task's results. + state_.store(new_state, std::memory_order_release); } void TaskContext::Schedule() { diff --git a/core/src/engine/task/task_context.hpp b/core/src/engine/task/task_context.hpp index 6d3981e6efd2..c23d60f83177 100644 --- a/core/src/engine/task/task_context.hpp +++ b/core/src/engine/task/task_context.hpp @@ -106,10 +106,7 @@ class TaskContext final : public ContextAccessor { bool IsSharedWaitAllowed() const; // whether user code finished executing, coroutine may still be running - bool IsFinished() const noexcept { - return state_ == Task::State::kCompleted || - state_ == Task::State::kCancelled; - } + bool IsFinished() const noexcept; void SetDetached(DetachedTasksSyncBlock::Token& token) noexcept; void FinishDetached() noexcept; From 0041e789182f3772e2cf30310e92bd4dddf317a3 Mon Sep 17 00:00:00 2001 From: segoon Date: Fri, 12 Jul 2024 10:41:26 +0300 Subject: [PATCH 020/629] feat otlp: don't UB on graceful service stop 6266705849d1d82ddc1bc621e38fe03bb0faeed5 --- otlp/src/otlp/logs/component.cpp | 17 ++++++++------- otlp/src/otlp/logs/logger.cpp | 36 ++++++++++++++++---------------- otlp/src/otlp/logs/logger.hpp | 31 +++++++++++++-------------- 3 files changed, 42 insertions(+), 42 deletions(-) diff --git a/otlp/src/otlp/logs/component.cpp b/otlp/src/otlp/logs/component.cpp index 997c6c50d1d5..5092fc1a8eef 100644 --- a/otlp/src/otlp/logs/component.cpp +++ b/otlp/src/otlp/logs/component.cpp @@ -26,17 +26,13 @@ LoggerComponent::LoggerComponent(const components::ComponentConfig& config, .GetFactory(); auto endpoint = config["endpoint"].As(); - auto client = std::make_shared< + auto client = client_factory.MakeClient< opentelemetry::proto::collector::logs::v1::LogsServiceClient>( - client_factory.MakeClient< - opentelemetry::proto::collector::logs::v1::LogsServiceClient>( - "otlp-logger", endpoint)); + "otlp-logger", endpoint); - auto trace_client = std::make_shared< + auto trace_client = client_factory.MakeClient< opentelemetry::proto::collector::trace::v1::TraceServiceClient>( - client_factory.MakeClient< - opentelemetry::proto::collector::trace::v1::TraceServiceClient>( - "otlp-tracer", endpoint)); + "otlp-tracer", endpoint); LoggerConfig logger_config; logger_config.max_queue_size = config["max-queue-size"].As(65535); @@ -58,6 +54,11 @@ LoggerComponent::LoggerComponent(const components::ComponentConfig& config, LoggerComponent::~LoggerComponent() { std::cerr << "Destroying default logger\n"; logging::impl::SetDefaultLoggerRef(logging::GetNullLogger()); + + logger_->Stop(); + + static std::shared_ptr logger_pin; + logger_pin = logger_; } yaml_config::Schema LoggerComponent::GetStaticConfigSchema() { diff --git a/otlp/src/otlp/logs/logger.cpp b/otlp/src/otlp/logs/logger.cpp index 7ca1fa4e73ea..4295120875ab 100644 --- a/otlp/src/otlp/logs/logger.cpp +++ b/otlp/src/otlp/logs/logger.cpp @@ -27,16 +27,10 @@ const std::string kTimestampFormat = "%Y-%m-%dT%H:%M:%E*S"; } // namespace Logger::Logger( - std::shared_ptr< - opentelemetry::proto::collector::logs::v1::LogsServiceClient> - client, - std::shared_ptr< - opentelemetry::proto::collector::trace::v1::TraceServiceClient> - trace_client, + opentelemetry::proto::collector::logs::v1::LogsServiceClient client, + opentelemetry::proto::collector::trace::v1::TraceServiceClient trace_client, LoggerConfig&& config) : LoggerBase(logging::Format::kTskv), - client_(std::move(client)), - trace_client_(std::move(trace_client)), config_(std::move(config)), queue_(Queue::Create(config_.max_queue_size)), queue_producer_(queue_->GetMultiProducer()) { @@ -45,13 +39,16 @@ Logger::Logger( std::cerr << "OTLP logger has started\n"; sender_task_ = engine::CriticalAsyncNoSpan( - [this, consumer = queue_->GetConsumer()]() mutable { - SendingLoop(consumer); + [this, consumer = queue_->GetConsumer(), log_client = std::move(client), + trace_client = std::move(trace_client)]() mutable { + SendingLoop(consumer, log_client, trace_client); }); ; } -Logger::~Logger() { sender_task_.SyncCancel(); } +Logger::~Logger() { Stop(); } + +void Logger::Stop() noexcept { sender_task_.SyncCancel(); } void Logger::Log(logging::Level, std::string_view msg) { // Trim trailing \n @@ -216,7 +213,8 @@ void Logger::HandleTracing(const std::vector& key_values) { // TODO: count drops } -void Logger::SendingLoop(Queue::Consumer& consumer) { +void Logger::SendingLoop(Queue::Consumer& consumer, LogClient& log_client, + TraceClient& trace_client) { // Create dummy span to completely disable logging in current coroutine tracing::Span span(""); span.SetLocalLogLevel(logging::Level::kNone); @@ -256,8 +254,8 @@ void Logger::SendingLoop(Queue::Consumer& consumer) { action); } while (consumer.Pop(action, deadline)); - DoLog(log_request); - DoTrace(trace_request); + DoLog(log_request, log_client); + DoTrace(trace_request, trace_client); } } @@ -290,9 +288,10 @@ void Logger::FillAttributes( void Logger::DoLog( const opentelemetry::proto::collector::logs::v1::ExportLogsServiceRequest& - request) { + request, + LogClient& client) { try { - auto call = client_->Export(request); + auto call = client.Export(request); auto response = call.Finish(); } catch (const ugrpc::client::RpcCancelledError&) { std::cerr << "Stopping OTLP sender task\n"; @@ -306,9 +305,10 @@ void Logger::DoLog( void Logger::DoTrace( const opentelemetry::proto::collector::trace::v1::ExportTraceServiceRequest& - request) { + request, + TraceClient& trace_client) { try { - auto call = trace_client_->Export(request); + auto call = trace_client.Export(request); auto response = call.Finish(); } catch (const ugrpc::client::RpcCancelledError&) { std::cerr << "Stopping OTLP sender task\n"; diff --git a/otlp/src/otlp/logs/logger.hpp b/otlp/src/otlp/logs/logger.hpp index ee137279ac42..9f460aec20a5 100644 --- a/otlp/src/otlp/logs/logger.hpp +++ b/otlp/src/otlp/logs/logger.hpp @@ -25,14 +25,13 @@ struct LoggerConfig { class Logger final : public logging::impl::LoggerBase { public: - explicit Logger( - std::shared_ptr< - opentelemetry::proto::collector::logs::v1::LogsServiceClient> - client, - std::shared_ptr< - opentelemetry::proto::collector::trace::v1::TraceServiceClient> - trace_client, - LoggerConfig&& config); + using LogClient = + opentelemetry::proto::collector::logs::v1::LogsServiceClient; + using TraceClient = + opentelemetry::proto::collector::trace::v1::TraceServiceClient; + + explicit Logger(LogClient client, TraceClient trace_client, + LoggerConfig&& config); ~Logger() override; @@ -40,6 +39,8 @@ class Logger final : public logging::impl::LoggerBase { void PrependCommonTags(logging::impl::TagWriter writer) const override; + void Stop() noexcept; + protected: bool DoShouldLog(logging::Level level) const noexcept override; @@ -48,7 +49,8 @@ class Logger final : public logging::impl::LoggerBase { ::opentelemetry::proto::trace::v1::Span>; using Queue = concurrent::NonFifoMpscQueue; - void SendingLoop(Queue::Consumer& consumer); + void SendingLoop(Queue::Consumer& consumer, LogClient& log_client, + TraceClient& trace_client); bool IsTracingEntry(const std::vector& key_values) const; @@ -60,16 +62,13 @@ class Logger final : public logging::impl::LoggerBase { void DoLog( const opentelemetry::proto::collector::logs::v1::ExportLogsServiceRequest& - request); + request, + LogClient& client); void DoTrace(const opentelemetry::proto::collector::trace::v1:: - ExportTraceServiceRequest& request); + ExportTraceServiceRequest& request, + TraceClient& trace_client); - std::shared_ptr - client_; - std::shared_ptr< - opentelemetry::proto::collector::trace::v1::TraceServiceClient> - trace_client_; const LoggerConfig config_; std::shared_ptr queue_; Queue::MultiProducer queue_producer_; From 08c58b84d4a577b916e3a13ffa9e9ec00d1391a6 Mon Sep 17 00:00:00 2001 From: arudenko02 Date: Fri, 12 Jul 2024 12:22:52 +0300 Subject: [PATCH 021/629] fix postgresql: clean busy connection don't drop connections on task cancellation 4b12c92897898d9911589aac574e9d902522acaf --- .../postgres/detail/connection_impl.cpp | 63 +++++++++---------- .../src/storages/postgres/detail/pool.cpp | 2 +- .../postgres/tests/connection_pgtest.cpp | 28 ++++++--- 3 files changed, 52 insertions(+), 41 deletions(-) diff --git a/postgresql/src/storages/postgres/detail/connection_impl.cpp b/postgresql/src/storages/postgres/detail/connection_impl.cpp index 20e9e14e6de9..9f33d123a8fb 100644 --- a/postgresql/src/storages/postgres/detail/connection_impl.cpp +++ b/postgresql/src/storages/postgres/detail/connection_impl.cpp @@ -560,40 +560,37 @@ void ConnectionImpl::CancelAndCleanup(TimeoutDuration timeout) { } bool ConnectionImpl::Cleanup(TimeoutDuration timeout) { - auto deadline = testsuite_pg_ctl_.MakeExecuteDeadline(timeout); - if (conn_wrapper_.TryConsumeInput(deadline, nullptr)) { - auto state = GetConnectionState(); - if (state == ConnectionState::kOffline) { - return false; - } - if (state == ConnectionState::kTranActive) { - return false; - } - if (state > ConnectionState::kIdle) { - Rollback(); - } - // We might need more timeout here - // We are no more bound with SLA, user has his exception. - // We need to try and save the connection without canceling current query - // not to kill the pgbouncer - SetConnectionStatementTimeout(GetDefaultCommandControl().statement, - deadline); - if (IsPipelineActive()) { - // In pipeline mode SetConnectionStatementTimeout writes a query into - // connection query queue without waiting for its result. - // We should process the results of this query, otherwise the connection - // is not IDLE and gets deleted by the pool. - // - // If the query timeouts we won't be IDLE, and apart from timeouts there's - // no other way for the query to fail, so just discard its result. - conn_wrapper_.DiscardInput(deadline); - } else if (settings_.pipeline_mode == PipelineMode::kEnabled) { - // Reenter pipeline mode if necessary - conn_wrapper_.EnterPipelineMode(); - } - return true; + const auto deadline = testsuite_pg_ctl_.MakeExecuteDeadline(timeout); + conn_wrapper_.DiscardInput(deadline); + auto state = GetConnectionState(); + if (state == ConnectionState::kOffline) { + return false; + } + if (state == ConnectionState::kTranActive) { + return false; + } + // We might need more timeout here + // We are no more bound with SLA, user has his exception. + // We need to try and save the connection without canceling current query + // not to kill the pgbouncer + SetConnectionStatementTimeout(GetDefaultCommandControl().statement, deadline); + if (IsPipelineActive()) { + // In pipeline mode SetConnectionStatementTimeout writes a query into + // connection query queue without waiting for its result. + // We should process the results of this query, otherwise the connection + // is not IDLE and gets deleted by the pool. + // + // If the query timeouts we won't be IDLE, and apart from timeouts there's + // no other way for the query to fail, so just discard its result. + conn_wrapper_.DiscardInput(deadline); + } else if (settings_.pipeline_mode == PipelineMode::kEnabled) { + // Reenter pipeline mode if necessary + conn_wrapper_.EnterPipelineMode(); + } + if (state > ConnectionState::kIdle) { + Rollback(); } - return false; + return true; } void ConnectionImpl::SetParameter(std::string_view name, std::string_view value, diff --git a/postgresql/src/storages/postgres/detail/pool.cpp b/postgresql/src/storages/postgres/detail/pool.cpp index affae8149ab8..9e1c368b1330 100644 --- a/postgresql/src/storages/postgres/detail/pool.cpp +++ b/postgresql/src/storages/postgres/detail/pool.cpp @@ -30,7 +30,7 @@ constexpr std::chrono::seconds kRecentErrorPeriod{15}; constexpr std::size_t kCancelRatio = 2; constexpr std::chrono::seconds kCancelPeriod{1}; -constexpr std::chrono::seconds kCleanupTimeout{1}; +constexpr std::chrono::seconds kCleanupTimeout{2}; constexpr std::chrono::seconds kMaintainInterval{30}; constexpr std::chrono::seconds kMaxIdleDuration{15}; diff --git a/postgresql/src/storages/postgres/tests/connection_pgtest.cpp b/postgresql/src/storages/postgres/tests/connection_pgtest.cpp index aa29697262ee..f94dfa7a0042 100644 --- a/postgresql/src/storages/postgres/tests/connection_pgtest.cpp +++ b/postgresql/src/storages/postgres/tests/connection_pgtest.cpp @@ -361,9 +361,9 @@ UTEST_P(PostgreConnection, StatementTimeout) { EXPECT_FALSE(GetConn()->IsBroken()); } -UTEST_P(PostgreConnection, QueryTaskCancel) { - CheckConnection(GetConn()); - EXPECT_EQ(pg::ConnectionState::kIdle, GetConn()->GetState()); +void CleanupConnectionTest(storages::postgres::detail::ConnectionPtr& conn, + bool use_cancel) { + EXPECT_EQ(pg::ConnectionState::kIdle, conn->GetState()); DefaultCommandControlScope scope( pg::CommandControl{utest::kMaxTestWaitTime, utest::kMaxTestWaitTime}); @@ -371,7 +371,7 @@ UTEST_P(PostgreConnection, QueryTaskCancel) { engine::SingleConsumerEvent task_started; auto task = engine::AsyncNoSpan([&] { task_started.Send(); - UEXPECT_THROW(GetConn()->Execute("select pg_sleep(1)"), + UEXPECT_THROW(conn->Execute("select pg_sleep(1)"), pg::ConnectionInterrupted); }); ASSERT_TRUE(task_started.WaitForEventFor(utest::kMaxTestWaitTime)); @@ -379,9 +379,23 @@ UTEST_P(PostgreConnection, QueryTaskCancel) { task.WaitFor(utest::kMaxTestWaitTime); ASSERT_TRUE(task.IsFinished()); - EXPECT_EQ(pg::ConnectionState::kTranActive, GetConn()->GetState()); - UEXPECT_NO_THROW(GetConn()->CancelAndCleanup(utest::kMaxTestWaitTime)); - EXPECT_EQ(pg::ConnectionState::kIdle, GetConn()->GetState()); + EXPECT_EQ(pg::ConnectionState::kTranActive, conn->GetState()); + if (use_cancel) { + UEXPECT_NO_THROW(conn->CancelAndCleanup(utest::kMaxTestWaitTime)); + } else { + UEXPECT_NO_THROW(conn->Cleanup(std::chrono::seconds{2})); + } + EXPECT_EQ(pg::ConnectionState::kIdle, conn->GetState()); +} + +UTEST_P(PostgreConnection, QueryTaskCancelAndCleanup) { + CheckConnection(GetConn()); + CleanupConnectionTest(GetConn(), /*use cancel=*/true); +} + +UTEST_P(PostgreConnection, QueryTaskCleanup) { + CheckConnection(GetConn()); + CleanupConnectionTest(GetConn(), /*use cancel=*/false); } UTEST_P(PostgreConnection, CachedPlanChange) { From e97e901b316eefbc8571ebd8acfd308571655986 Mon Sep 17 00:00:00 2001 From: antoshkka Date: Fri, 12 Jul 2024 18:09:32 +0300 Subject: [PATCH 022/629] feat docs: update changelog and fix typos MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tests: протестировано CI 7916fd1134e3af9baeef766f72878572e04fecf0 --- README.md | 2 +- chaotic/chaotic/back/cpp/translator.py | 2 +- chaotic/chaotic/compilers/dynamic_config.py | 4 +-- chaotic/chaotic/main.py | 4 +-- .../ydb_service/components/topic_reader.cpp | 2 +- scripts/docs/en/landing.md | 2 +- scripts/docs/en/schemas/dynamic_configs.md | 8 ++--- scripts/docs/en/userver/chaotic.md | 8 ++--- .../docs/en/userver/development/stability.md | 2 +- scripts/docs/en/userver/kafka.md | 4 +-- .../docs/en/userver/roadmap_and_changelog.md | 35 +++++++++++++++++-- .../include/userver/formats/yaml/value.hpp | 2 +- 12 files changed, 53 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 13c5eea34643..9a0a99b94f34 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ You can learn more about history and key features of userver from our ## Other Features * Efficient asynchronous drivers for databases (MongoDB, PostgreSQL, Redis, - ClickHouse, MySQL/MariaDB ...) and data transfer protocols + ClickHouse, MySQL/MariaDB, YDB ...) and data transfer protocols (HTTP, gRPC, AMQP 0-9-1, Kafka, TCP, TLS, WebSocket ...), tasks construction and cancellation. * Rich set of high-level components for caches, tasks, distributed locking, diff --git a/chaotic/chaotic/back/cpp/translator.py b/chaotic/chaotic/back/cpp/translator.py index bb168a21fe25..4ac7060d7fdd 100644 --- a/chaotic/chaotic/back/cpp/translator.py +++ b/chaotic/chaotic/back/cpp/translator.py @@ -494,7 +494,7 @@ def _gen_array(self, name: str, schema: types.Array) -> cpp_types.CppType: user_cpp_type = None return cpp_types.CppArray( - raw_cpp_type='NOT_USED', # _cpp_type() is overriden in array + raw_cpp_type='NOT_USED', # _cpp_type() is overridden in array json_schema=schema, nullable=schema.nullable, user_cpp_type=user_cpp_type, diff --git a/chaotic/chaotic/compilers/dynamic_config.py b/chaotic/chaotic/compilers/dynamic_config.py index 93098929f34e..18ca45a2b334 100644 --- a/chaotic/chaotic/compilers/dynamic_config.py +++ b/chaotic/chaotic/compilers/dynamic_config.py @@ -54,8 +54,8 @@ def write_file(filepath: str, content: str) -> None: if os.path.exists(filepath): with open(filepath, 'r', encoding='utf8') as ifile: - old_conent = ifile.read() - if old_conent == content: + old_content = ifile.read() + if old_content == content: return with open(filepath, 'w') as ofile: diff --git a/chaotic/chaotic/main.py b/chaotic/chaotic/main.py index 612042c614f9..3538b284aefc 100644 --- a/chaotic/chaotic/main.py +++ b/chaotic/chaotic/main.py @@ -232,8 +232,8 @@ def read_schemas( def write_file(filepath: str, content: str) -> None: if os.path.exists(filepath): with open(filepath, 'r', encoding='utf8') as ifile: - old_conent = ifile.read() - if old_conent == content: + old_content = ifile.read() + if old_content == content: return with open(filepath, 'w') as ofile: diff --git a/samples/ydb_service/components/topic_reader.cpp b/samples/ydb_service/components/topic_reader.cpp index 781f1b2ab155..9da50882437f 100644 --- a/samples/ydb_service/components/topic_reader.cpp +++ b/samples/ydb_service/components/topic_reader.cpp @@ -137,7 +137,7 @@ class SessionReadTask { /* If it throws, read session will be closed and recreated in - {restart_session_delay} ms All uncommited messages will be received again + {restart_session_delay} ms All uncommitted messages will be received again */ } diff --git a/scripts/docs/en/landing.md b/scripts/docs/en/landing.md index 5a31928782db..5bc8417fa0f7 100644 --- a/scripts/docs/en/landing.md +++ b/scripts/docs/en/landing.md @@ -99,7 +99,7 @@ void Ins(storages::postgres::Transaction& tr,

Efficient asynchronous drivers for databases (MongoDB, PostgreSQL, - MySQL/MariaDB, Redis, ClickHouse, ...) and data + MySQL/MariaDB, Redis, ClickHouse, YDB, ...) and data transfer protocols (HTTP, WEbSockets, gRPC, TCP, AMQP-0.9.1, Apache Kafka, ...), tasks construction and cancellation.

diff --git a/scripts/docs/en/schemas/dynamic_configs.md b/scripts/docs/en/schemas/dynamic_configs.md index c47120e4b063..bb3692e577e9 100644 --- a/scripts/docs/en/schemas/dynamic_configs.md +++ b/scripts/docs/en/schemas/dynamic_configs.md @@ -139,19 +139,19 @@ schema: description: Percent of errors to enable СС type: number deactivate-delta: - description: СС turned off if this amount of free conections is reached + description: СС turned off if this amount of free connections is reached type: integer timings-burst-times-threshold: description: CC is turned on if request times grow to this value type: number min-timings-ms: - description: minimal value of timeings after wich the CC heuristics turn on + description: minimal value of timeings after which the CC heuristics turn on type: integer min-limit: - description: minimal value of connections after wich the CC heuristics turn on + description: minimal value of connections after which the CC heuristics turn on type: integer min-qps: - description: minimal value of queries per second after wich the CC heuristics turn on + description: minimal value of queries per second after which the CC heuristics turn on type: integer ``` diff --git a/scripts/docs/en/userver/chaotic.md b/scripts/docs/en/userver/chaotic.md index 58fd93a2085d..74718bc4a073 100644 --- a/scripts/docs/en/userver/chaotic.md +++ b/scripts/docs/en/userver/chaotic.md @@ -1,4 +1,4 @@ -## Chaotic codegen +## JSON schema codegen - the Chaotic Sometimes it is required to declare a data structure and to define parse/serialize methods for it. It is OK if you do it manually once a month, but it becomes uncomfortable to do it manually more frequently. @@ -18,7 +18,7 @@ You can use chaotic in three simple steps: 3) use generated .hpp and .cpp files in your C++ project. -Also we've made `chaotic-gen` cmake wrappers for your convinience. +Also we've made `chaotic-gen` cmake wrappers for your convenience. Let's go through the list number by number. @@ -54,7 +54,7 @@ Some frequently used parameters are described below. * `-n` defines types mapping from in-yaml object path to C++ type name (with namespace). The path regex is written first, then equal sign `=`, then C++ type name. `-n` can be passed multiple times. -* `-f` defines file mapping from yaml filenames to C++ filenames (exluding the extentions). +* `-f` defines file mapping from yaml filenames to C++ filenames (excluding the extensions). Usually as-is mapping is used. * `--parse-extra-formats` generates YAML and YAML config parsers besides JSON parser. * `--generate-serializers` generates serializers into JSON besides JSON parser from `formats::json::Value`. @@ -208,7 +208,7 @@ You may change the container type of `extra` field with `x-usrv-cpp-extra-type`: Any unknown field leads to a validation failure in case of `additionalProperties: false`. -It can be overriden by setting `x-usrv-strict-parsing: false`. +It can be overridden by setting `x-usrv-strict-parsing: false`. In this case unknown fields will be ignored. diff --git a/scripts/docs/en/userver/development/stability.md b/scripts/docs/en/userver/development/stability.md index de87fe3fc5eb..7d2c80d1d68c 100644 --- a/scripts/docs/en/userver/development/stability.md +++ b/scripts/docs/en/userver/development/stability.md @@ -62,7 +62,7 @@ There are tiers to differentiate technologies: * **Platinum Tier** - driver is known to be used in multiple high load critical to uptime services in huge companies. -* **Golden Tier** - driver that has not enought usage +* **Golden Tier** - driver that has not enough usage feedback from huge companies. Still fine for production usage. * **Silver Tier** - early days of the driver. It passes all the tests and works fine, but more feedback/time required to become a Golden Tier driver. Fine diff --git a/scripts/docs/en/userver/kafka.md b/scripts/docs/en/userver/kafka.md index b6ae60cbdcc7..e3da4d195ed6 100644 --- a/scripts/docs/en/userver/kafka.md +++ b/scripts/docs/en/userver/kafka.md @@ -7,7 +7,7 @@ two independent interfaces: kafka::ProducerComponent and kafka::ConsumerComponent. They expose APIs for sending messages to Kafka topics, suspending -coroutine until message delivery succeeded or failed, and for proccess message +coroutine until message delivery succeeded or failed, and for process message batches from the subscribed set of topics, asynchronously polling messages in separate task processor. @@ -34,7 +34,7 @@ in separate task processor. ## Planned Enhancements - Transfer from raw polling with timeouts to events processing, making the message polling non-blocking and leading to better library scalability; -- on_error callback for Consumer for convinient error processing; +- on_error callback for Consumer for convenient error processing; - testsuite Kafka support in OSS; - Support of different compression codecs (GZIP, LZ4, ZSTD, etc..); - Support more SASL authentication mechanisms (GSSAPI, OAUTHBEARER); diff --git a/scripts/docs/en/userver/roadmap_and_changelog.md b/scripts/docs/en/userver/roadmap_and_changelog.md index 54983da17492..cf16730064d3 100644 --- a/scripts/docs/en/userver/roadmap_and_changelog.md +++ b/scripts/docs/en/userver/roadmap_and_changelog.md @@ -16,10 +16,10 @@ Changelog news also go to the ## Roadmap -* 👨‍💻 Codegen parsers and serializers by JSON schema +* ✔️ Codegen parsers and serializers by JSON schema * 👨‍💻 Improve Kafka driver. * 👨‍💻 Add retry budget or retry circuit breaker for clients. -* Improve OpenTelemetry Protocol (OTLP) support. +* 👨‍💻 Improve OpenTelemetry Protocol (OTLP) support. * Add web interface to the [uservice-dynconf](https://github.com/userver-framework/uservice-dynconf) * Generate full-blown accessories for OpenAPI: * clients @@ -28,6 +28,34 @@ Changelog news also go to the ## Changelog +### Release v2.2 + +* Added @ref scripts/docs/en/userver/chaotic.md "codegen parsers and serializers by JSON schema" +* Improved the ability to unit test of gRPC clients and servers, Redis and + Mongo databases, logs. Samples and docs were improved. +* Implemented feedback forms and likes/dislikes for each documentation page. + **Feedback would be appreciated!** + Many thanks to [Fedor Alekseev](https://github.com/atlz253) for the PR and to + [MariaGrinchenko](https://github.com/MariaGrinchenko) for the buttons design! +* Added @ref scripts/docs/en/userver/ydb.md "docs on YDB". +* Mobile header view was improved. Many thanks to + [Fedor Alekseev](https://github.com/atlz253) for the PR. +* engine::subprocess::ProcessStarter::Exec now can lookup binaries via + `PATH` variable. +* Fixed gRPC generation for nested namespaces with repetitions. Many thanks to + [nfrmtk](https://github.com/nfrmtk) for the PR! +* Handle both websocket and plain HTTP requests for the same path. Many thanks + to [Hovard Smith](https://github.com/w15eacre) for the PR! +* Support setting client + CA certs in RabbitMQ. Many thanks to + [Alexey Dyumin](https://github.com/dyumin) for the PR! +* yaml_config::YamlConfig now can read files via `#file`. Now the static + config of the service could refer to other files. +* Added support of bit operations to Redis. +* Added quick start for beginners to @ref scripts/docs/en/userver/tutorial/build.md. + Many thanks to [Fedor Alekseev](https://github.com/atlz253) for the PR. +* Improved path to sources trimming for Conan builds. Many thanks to + [Kirill](https://github.com/KVolodin) for the PR! +* Multiple minor improvements to build, packaging, docs and testing. ### Release v2.1 (May 2024) @@ -61,6 +89,9 @@ Changelog news also go to the * `human_logs.py` now supports more options and has more examples and docs embedded. Thanks to [TertiumOrganum1](https://github.com/TertiumOrganum1) for the PR! +* server::http::HttpStatus and client::http::Status are now aliases to + http::StatusCode. Many thanks to + [SidorovichPavel](https://github.com/SidorovichPavel) for the PR! * Docs and build: * `find_package(userver)` now implicitly calls `userver_setup_environment()`, diff --git a/universal/include/userver/formats/yaml/value.hpp b/universal/include/userver/formats/yaml/value.hpp index fba9ad7a4e26..ec3c943ef730 100644 --- a/universal/include/userver/formats/yaml/value.hpp +++ b/universal/include/userver/formats/yaml/value.hpp @@ -185,7 +185,7 @@ class Value final { /// @brief Returns YAML tag of this node. /// If tag is not explicitly specified, its value depends on node value. For - /// explicitly sepcified tags, its value depends on used `TAG` directives and + /// explicitly specified tags, its value depends on used `TAG` directives and /// node value. There is an implicit `TAG` derictive for `!!` with prefix /// `tag:yaml.org,2002:`. /// From a8771e93866bef651626e18ed5e88127ce049ba9 Mon Sep 17 00:00:00 2001 From: antonyzhilin Date: Fri, 12 Jul 2024 22:58:13 +0300 Subject: [PATCH 023/629] fix server: log deadline propagation exceptions with WARNING, not ERROR 02ccdb01869b4dcee7a069f1d6bc33a6842313c5 --- core/src/server/middlewares/deadline_propagation.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/src/server/middlewares/deadline_propagation.cpp b/core/src/server/middlewares/deadline_propagation.cpp index 12ae840fb681..e7fa24a19af2 100644 --- a/core/src/server/middlewares/deadline_propagation.cpp +++ b/core/src/server/middlewares/deadline_propagation.cpp @@ -230,6 +230,8 @@ void DeadlinePropagation::CompleteDeadlinePropagation( if (cancelled_by_deadline && !dp_scope.shared_dp_context.IsCancelledByDeadline()) { + dp_scope.shared_dp_context.SetCancelledByDeadline(); + const auto& original_body = response.GetData(); if (!original_body.empty() && span_opt && span_opt->ShouldLogDefault()) { span_opt->AddNonInheritableTag("dp_original_body_size", From 98a5016d71034415e099d5e26ebf856bd5df868d Mon Sep 17 00:00:00 2001 From: Fedor Alekseev Date: Mon, 15 Jul 2024 12:59:31 +0300 Subject: [PATCH 024/629] feature docs: documentation improvements MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ![127 0 0 1_5501_docs_html_de_d6a_md_en_2index html](https://github.com/user-attachments/assets/5d6e420e-d41d-4228-9f3b-4ff1f05e83b4) * doxygen-awesome update (with scroll lags fix) * disabled the unnecessary graphs to speed up doxygen documentation generation * fix interactive TOC init error * improved the style of the site layout Tests: протестировано CI 28a74fa82d05f9edbb2a632f8b9678fcf5625af1 Pull Request resolved: https://github.com/userver-framework/userver/pull/644 --- scripts/docs/customdoxygen.css | 63 ++++++++-------- scripts/docs/doxygen-awesome-css/README.md | 5 +- .../doxygen-awesome-interactive-toc.js | 73 ++++++++++++------- .../doxygen-awesome-tabs.js | 6 +- .../doxygen-awesome-css/doxygen-awesome.css | 36 +++++---- scripts/docs/doxygen.conf | 10 +-- scripts/docs/footer.html | 3 +- scripts/docs/header.html | 4 +- scripts/docs/landing.css | 9 ++- scripts/docs/scripts/feedback.js | 16 +--- scripts/docs/scripts/toc.js | 18 +++++ 11 files changed, 137 insertions(+), 106 deletions(-) diff --git a/scripts/docs/customdoxygen.css b/scripts/docs/customdoxygen.css index f8065191895f..4b30e61e2c50 100644 --- a/scripts/docs/customdoxygen.css +++ b/scripts/docs/customdoxygen.css @@ -91,6 +91,9 @@ html { --font-family-monospace: "fontello", "Roboto Mono", ui-monospace, SFMono-Regular, SF Mono, Menlo, Consolas, Liberation Mono, monospace; + /* The content is centered and constraint in it's width. To make the content fill the whole page, set the variable to auto.*/ + --content-maxwidth: 1440px; + /* table colors */ --tablehead-background: var(--primary-color-1); @@ -227,10 +230,6 @@ span.paramname em { color: var(--primary-dark-color); } -/* Too wide page makes people uncomfortable, keep content in center */ -div.contents { - max-width: 1440px; -} /* Scale down image to the page width */ .image img { @@ -253,7 +252,7 @@ hr { justify-content: space-between; background-color: var(--black-color-1); color: white !important; - padding: 20px; + padding: 15px 20px; } #titlearea { @@ -271,7 +270,7 @@ hr { } #projectlogo img { - width: 70px; + width: 55px; max-height: none; } @@ -279,18 +278,20 @@ hr { padding-left: 0.5em; } -#projectalign div { - line-height: 26px; +#projectname a, +#projectbrief a { + color: white !important; } -#projectname a { - font-size: 42px; - color: white !important; +#projectname { + line-height: 26px; + font-size: 38px; } -#projectbrief a { - font-size: 17px; - color: white !important; +#projectbrief { + margin-top: 2px; + font-size: 16px; + line-height: 18px; } #links a:hover { @@ -406,10 +407,6 @@ html.dark-mode .sm.sm-dox li:last-child ul li a { right: 15px; } -#projectbrief { - font-size: 120%; -} - #projectlogo img { margin: 0px; } @@ -426,6 +423,12 @@ div.header .summary { display: none; } +div.header .title { + margin-bottom: 0; + padding-top: 0; + padding-bottom: 0; +} + .sm-dox ul a:hover, .sm-dox ul a:active, .sm-dox ul a:focus { @@ -599,22 +602,18 @@ doxygen-awesome-dark-mode-toggle { max-height: none; } - #projectalign div { - line-height: 12px; - } - td#projectalign { padding: 5px; } - #projectname a { - font-size: 15px; - color: white !important; + #projectname { + font-size: 18px; + line-height: 12px; } - #projectbrief a { - font-size: 9px; - color: white !important; + #projectbrief { + line-height: 12px; + font-size: 10px; } #MSearchBox { @@ -778,10 +777,6 @@ doxygen-awesome-dark-mode-toggle { padding: 5px; } - #projectalign div { - line-height: 10px; - } - #projectalign { display: none; } @@ -963,7 +958,7 @@ div.contents .toc { box-sizing: border-box; min-width: 300px; padding: 16px 16px; - top: 97px; + top: 77px; max-height: calc(100vh - 255px); } @@ -1021,7 +1016,7 @@ div.toc li a.active { page-feedback { position: absolute; - top: 170px; + top: 136px; right: 30px; } diff --git a/scripts/docs/doxygen-awesome-css/README.md b/scripts/docs/doxygen-awesome-css/README.md index f51c8ae25aaf..cc761515ab98 100644 --- a/scripts/docs/doxygen-awesome-css/README.md +++ b/scripts/docs/doxygen-awesome-css/README.md @@ -2,4 +2,7 @@ Doxygen Awesome repository: [link](https://github.com/jothepro/doxygen-awesome-css) -Version 2.3.1 without changes +Version 2.3.3 (commit 28ed396de19cd3d803bcb483dceefdb6d03b1b2b) + +changes: +* doxygen-awesome-interactive-toc.js patched for correct loading diff --git a/scripts/docs/doxygen-awesome-css/doxygen-awesome-interactive-toc.js b/scripts/docs/doxygen-awesome-css/doxygen-awesome-interactive-toc.js index 20a9669d7df0..4113f030540a 100644 --- a/scripts/docs/doxygen-awesome-css/doxygen-awesome-interactive-toc.js +++ b/scripts/docs/doxygen-awesome-css/doxygen-awesome-interactive-toc.js @@ -33,35 +33,44 @@ class DoxygenAwesomeInteractiveToc { static headers = [] static init() { - window.addEventListener("load", () => { - let toc = document.querySelector(".contents > .toc") - if(toc) { - toc.classList.add("interactive") - if(!DoxygenAwesomeInteractiveToc.hideMobileMenu) { + if (document.readyState === "loading") { + window.addEventListener( + "load", + () => DoxygenAwesomeInteractiveToc.#parseHeaders() + ); + } + else { + DoxygenAwesomeInteractiveToc.#parseHeaders(); + } + DoxygenAwesomeInteractiveToc.update(); + } + + static #parseHeaders() { + DoxygenAwesomeInteractiveToc.headers = []; + let toc = document.querySelector(".contents > .toc") + if(toc) { + toc.classList.add("interactive") + if(!DoxygenAwesomeInteractiveToc.hideMobileMenu) { + toc.classList.add("open") + } + document.querySelector(".contents > .toc > h3")?.addEventListener("click", () => { + if(toc.classList.contains("open")) { + toc.classList.remove("open") + } else { toc.classList.add("open") } - document.querySelector(".contents > .toc > h3")?.addEventListener("click", () => { - if(toc.classList.contains("open")) { - toc.classList.remove("open") - } else { - toc.classList.add("open") - } - }) - - document.querySelectorAll(".contents > .toc > ul a").forEach((node) => { - let id = node.getAttribute("href").substring(1) - DoxygenAwesomeInteractiveToc.headers.push({ - node: node, - headerNode: document.getElementById(id) - }) + }) - document.getElementById("doc-content")?.addEventListener("scroll", () => { - DoxygenAwesomeInteractiveToc.update() - }) + document.querySelectorAll(".contents > .toc > ul a").forEach((node) => { + let id = node.getAttribute("href").substring(1) + DoxygenAwesomeInteractiveToc.headers.push({ + node: node, + headerNode: document.getElementById(id) }) - DoxygenAwesomeInteractiveToc.update() - } - }) + + document.getElementById("doc-content")?.addEventListener("scroll",this.throttle(DoxygenAwesomeInteractiveToc.update, 100)) + }) + } } static update() { @@ -78,4 +87,16 @@ class DoxygenAwesomeInteractiveToc { active?.classList.add("active") active?.classList.remove("aboveActive") } -} \ No newline at end of file + + static throttle(func, delay) { + let lastCall = 0; + return function (...args) { + const now = new Date().getTime(); + if (now - lastCall < delay) { + return; + } + lastCall = now; + return setTimeout(() => {func(...args)}, delay); + }; + } +} diff --git a/scripts/docs/doxygen-awesome-css/doxygen-awesome-tabs.js b/scripts/docs/doxygen-awesome-css/doxygen-awesome-tabs.js index 8a7c3f18adcc..06dfd3d66f4e 100644 --- a/scripts/docs/doxygen-awesome-css/doxygen-awesome-tabs.js +++ b/scripts/docs/doxygen-awesome-css/doxygen-awesome-tabs.js @@ -33,7 +33,7 @@ class DoxygenAwesomeTabs { window.addEventListener("load", () => { document.querySelectorAll(".tabbed:not(:empty)").forEach((tabbed, tabbedIndex) => { let tabLinkList = [] - tabbed.querySelectorAll("li").forEach((tab, tabIndex) => { + tabbed.querySelectorAll(":scope > ul > li").forEach((tab, tabIndex) => { tab.id = "tab_" + tabbedIndex + "_" + tabIndex let header = tab.querySelector(".tab-title") let tabLink = document.createElement("button") @@ -41,7 +41,7 @@ class DoxygenAwesomeTabs { tabLink.appendChild(header) header.title = header.textContent tabLink.addEventListener("click", () => { - tabbed.querySelectorAll("li").forEach((tab) => { + tabbed.querySelectorAll(":scope > ul > li").forEach((tab) => { tab.classList.remove("selected") }) tabLinkList.forEach((tabLink) => { @@ -68,7 +68,7 @@ class DoxygenAwesomeTabs { function resize() { let maxTabHeight = 0 - tabbed.querySelectorAll("li").forEach((tab, tabIndex) => { + tabbed.querySelectorAll(":scope > ul > li").forEach((tab, tabIndex) => { let visibility = tab.style.display tab.style.display = "block" maxTabHeight = Math.max(tab.offsetHeight, maxTabHeight) diff --git a/scripts/docs/doxygen-awesome-css/doxygen-awesome.css b/scripts/docs/doxygen-awesome-css/doxygen-awesome.css index ac7f0608ec69..a2715e268c55 100644 --- a/scripts/docs/doxygen-awesome-css/doxygen-awesome.css +++ b/scripts/docs/doxygen-awesome-css/doxygen-awesome.css @@ -313,7 +313,7 @@ body { body, table, div, p, dl, #nav-tree .label, .title, .sm-dox a, .sm-dox a:hover, .sm-dox a:focus, #projectname, .SelectItem, #MSearchField, .navpath li.navelem a, -.navpath li.navelem a:hover, p.reference, p.definition { +.navpath li.navelem a:hover, p.reference, p.definition, div.toc li, div.toc h3 { font-family: var(--font-family); } @@ -334,6 +334,7 @@ p.reference, p.definition { a:link, a:visited, a:hover, a:focus, a:active { color: var(--primary-color) !important; font-weight: 500; + background: none; } a.anchor { @@ -806,6 +807,10 @@ html.dark-mode iframe#MSearchResults { line-height: var(--tree-item-height); } +#nav-tree .item > a:focus { + outline: none; +} + #nav-sync { bottom: 12px; right: 12px; @@ -843,6 +848,7 @@ html.dark-mode iframe#MSearchResults { #nav-tree .arrow { opacity: var(--side-nav-arrow-opacity); + background: none; } .arrow { @@ -1040,7 +1046,7 @@ blockquote::after { blockquote p { margin: var(--spacing-small) 0 var(--spacing-medium) 0; } -.paramname { +.paramname, .paramname em { font-weight: 600; color: var(--primary-dark-color); } @@ -1090,7 +1096,7 @@ div.contents .toc { border: 0; border-left: 1px solid var(--separator-color); border-radius: 0; - background-color: transparent; + background-color: var(--page-background-color); box-shadow: none; position: sticky; top: var(--toc-sticky-top); @@ -1982,14 +1988,16 @@ hr { } .contents hr { - box-shadow: 100px 0 0 var(--separator-color), - -100px 0 0 var(--separator-color), - 500px 0 0 var(--separator-color), - -500px 0 0 var(--separator-color), - 1500px 0 0 var(--separator-color), - -1500px 0 0 var(--separator-color), - 2000px 0 0 var(--separator-color), - -2000px 0 0 var(--separator-color); + box-shadow: 100px 0 var(--separator-color), + -100px 0 var(--separator-color), + 500px 0 var(--separator-color), + -500px 0 var(--separator-color), + 900px 0 var(--separator-color), + -900px 0 var(--separator-color), + 1400px 0 var(--separator-color), + -1400px 0 var(--separator-color), + 1900px 0 var(--separator-color), + -1900px 0 var(--separator-color); } .contents img, .contents .center, .contents center, .contents div.image object { @@ -2460,17 +2468,17 @@ h2:hover a.anchorlink, h1:hover a.anchorlink, h3:hover a.anchorlink, h4:hover a. Optional tab feature */ -.tabbed ul { +.tabbed > ul { padding-inline-start: 0px; margin: 0; padding: var(--spacing-small) 0; } -.tabbed li { +.tabbed > ul > li { display: none; } -.tabbed li.selected { +.tabbed > ul > li.selected { display: block; } diff --git a/scripts/docs/doxygen.conf b/scripts/docs/doxygen.conf index ff2b9ae4b384..46eaf3f54c30 100644 --- a/scripts/docs/doxygen.conf +++ b/scripts/docs/doxygen.conf @@ -2704,7 +2704,7 @@ CLASS_GRAPH = YES # The default value is: YES. # This tag requires that the tag HAVE_DOT is set to YES. -COLLABORATION_GRAPH = YES +COLLABORATION_GRAPH = NO # If the GROUP_GRAPHS tag is set to YES then doxygen will generate a graph for # groups, showing the direct groups dependencies. See also the chapter Grouping @@ -2712,7 +2712,7 @@ COLLABORATION_GRAPH = YES # The default value is: YES. # This tag requires that the tag HAVE_DOT is set to YES. -GROUP_GRAPHS = YES +GROUP_GRAPHS = NO # If the UML_LOOK tag is set to YES, doxygen will generate inheritance and # collaboration diagrams in a style similar to the OMG's Unified Modeling @@ -2774,7 +2774,7 @@ TEMPLATE_RELATIONS = NO # The default value is: YES. # This tag requires that the tag HAVE_DOT is set to YES. -INCLUDE_GRAPH = YES +INCLUDE_GRAPH = NO # If the INCLUDED_BY_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are # set to YES then doxygen will generate a graph for each documented file showing @@ -2817,7 +2817,7 @@ CALLER_GRAPH = NO # The default value is: YES. # This tag requires that the tag HAVE_DOT is set to YES. -GRAPHICAL_HIERARCHY = YES +GRAPHICAL_HIERARCHY = NO # If the DIRECTORY_GRAPH tag is set to YES then doxygen will show the # dependencies a directory has on other directories in a graphical way. The @@ -2829,7 +2829,7 @@ GRAPHICAL_HIERARCHY = YES # The default value is: YES. # This tag requires that the tag HAVE_DOT is set to YES. -DIRECTORY_GRAPH = YES +DIRECTORY_GRAPH = NO # The DIR_GRAPH_MAX_DEPTH tag can be used to limit the maximum number of levels # of child directories generated in directory dependency graphs by dot. diff --git a/scripts/docs/footer.html b/scripts/docs/footer.html index 86f3f04aa599..458620e20ef7 100644 --- a/scripts/docs/footer.html +++ b/scripts/docs/footer.html @@ -51,13 +51,12 @@ if (isLanding) return; - draw_toc(); highlight_code(); styleNavButtons(); }, 0); }); }); - + diff --git a/scripts/docs/header.html b/scripts/docs/header.html index 18b6c560f130..ae10a713f5ae 100644 --- a/scripts/docs/header.html +++ b/scripts/docs/header.html @@ -53,13 +53,11 @@ - diff --git a/scripts/docs/landing.css b/scripts/docs/landing.css index 35f6c53bd126..13da73f353a0 100644 --- a/scripts/docs/landing.css +++ b/scripts/docs/landing.css @@ -351,12 +351,15 @@ a.button.button_outline { display: block; top: -84px; right: -200px; - background-color: var(--primary-color); border-radius: 280px; transition: background-color 0.3s; } -.info__logo:hover { +#landing_logo_id { + background-color: var(--primary-color); +} + +#landing_logo_id:hover { background-color: var(--primary-dark-color); } @@ -854,7 +857,7 @@ html.dark-mode .values__card { transition: background-color 0.3s, border-color 0.3s, color 0.3s, opacity 0.5s; } -.feedback__stars:valid + .feedback__button { +.feedback__stars:valid+.feedback__button { opacity: 1; } diff --git a/scripts/docs/scripts/feedback.js b/scripts/docs/scripts/feedback.js index de12c3d973f8..eba2b098ae41 100644 --- a/scripts/docs/scripts/feedback.js +++ b/scripts/docs/scripts/feedback.js @@ -321,7 +321,7 @@ class PageFeedbackPopup extends HTMLElement { } } -class LandingFeedback { +export class LandingFeedback { static #lastStarRatingLabel = null; static init() { @@ -403,17 +403,3 @@ class FirebasePageDatabase { customElements.define("page-feedback", PageFeedback); customElements.define("page-feedback-popup", PageFeedbackPopup); - -$(function () { - $(document).ready(function () { - setTimeout(() => { - const isLanding = document.getElementById("landing_logo_id") !== null; - - if (isLanding) { - LandingFeedback.init(); - } else { - PageFeedback.init(); - } - }, 0); - }); -}); diff --git a/scripts/docs/scripts/toc.js b/scripts/docs/scripts/toc.js index c0429c38aee1..99e6b30a9f1b 100644 --- a/scripts/docs/scripts/toc.js +++ b/scripts/docs/scripts/toc.js @@ -1,3 +1,5 @@ +import { LandingFeedback, PageFeedback } from "./feedback.js"; + const LOWER_CASE_TRANSLITTERATION_MAPPING = { а: "a", б: "b", @@ -94,3 +96,19 @@ function draw_toc() { ]) .insertAfter("#MSearchResultsWindow"); } + +$(function () { + $(document).ready(function () { + setTimeout(() => { + const isLanding = document.getElementById("landing_logo_id") !== null; + + if (isLanding) { + LandingFeedback.init(); + } else { + draw_toc(); + DoxygenAwesomeInteractiveToc.init(); + PageFeedback.init(); + } + }, 0); + }); +}); From dd1f54e61c45fc36f4dc22e6703a5a1b53c775a6 Mon Sep 17 00:00:00 2001 From: kosstin Date: Mon, 15 Jul 2024 13:16:23 +0300 Subject: [PATCH 025/629] feat core: grpc header propagation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tests: протестировано CI a142a452d89f66df558fb3ec2ad4bb5101eb7df2 --- .mapping.json | 3 +- .../userver/server/http/http_response.hpp | 5 ++ .../server/request/task_inherited_request.hpp | 34 +++++++------- core/src/server/http/http_request_handler.cpp | 26 +++++++++- core/src/server/http/http_response.cpp | 4 ++ .../server/request/task_inherited_request.cpp | 47 +++++++------------ .../request/task_inherited_request_impl.hpp | 18 ------- .../basic_server/grpc_service.cpp | 45 +++++++++++++++++- .../basic_server/proto/samples/greeter.proto | 1 + .../basic_server/static_config.yaml | 10 ++++ .../tests-unix-socket/conftest.py | 13 ++++- .../test_header_propagation.py | 39 +++++++++++++++ .../ugrpc/server/impl/request_container.hpp | 37 +++++++++++++++ .../ugrpc/server/impl/service_worker_impl.hpp | 5 ++ 14 files changed, 215 insertions(+), 72 deletions(-) delete mode 100644 core/src/server/request/task_inherited_request_impl.hpp create mode 100644 grpc/functional_tests/basic_server/tests-unix-socket/test_header_propagation.py create mode 100644 grpc/include/userver/ugrpc/server/impl/request_container.hpp diff --git a/.mapping.json b/.mapping.json index b76354e35e17..7c931937e77a 100644 --- a/.mapping.json +++ b/.mapping.json @@ -1449,7 +1449,6 @@ "core/src/server/request/response_base.cpp":"taxi/uservices/userver/core/src/server/request/response_base.cpp", "core/src/server/request/task_inherited_data.cpp":"taxi/uservices/userver/core/src/server/request/task_inherited_data.cpp", "core/src/server/request/task_inherited_request.cpp":"taxi/uservices/userver/core/src/server/request/task_inherited_request.cpp", - "core/src/server/request/task_inherited_request_impl.hpp":"taxi/uservices/userver/core/src/server/request/task_inherited_request_impl.hpp", "core/src/server/requests_view.cpp":"taxi/uservices/userver/core/src/server/requests_view.cpp", "core/src/server/requests_view.hpp":"taxi/uservices/userver/core/src/server/requests_view.hpp", "core/src/server/server.cpp":"taxi/uservices/userver/core/src/server/server.cpp", @@ -1717,6 +1716,7 @@ "grpc/functional_tests/basic_server/static_config.yaml":"taxi/uservices/userver/grpc/functional_tests/basic_server/static_config.yaml", "grpc/functional_tests/basic_server/tests-unix-socket/conftest.py":"taxi/uservices/userver/grpc/functional_tests/basic_server/tests-unix-socket/conftest.py", "grpc/functional_tests/basic_server/tests-unix-socket/test_grpc.py":"taxi/uservices/userver/grpc/functional_tests/basic_server/tests-unix-socket/test_grpc.py", + "grpc/functional_tests/basic_server/tests-unix-socket/test_header_propagation.py":"taxi/uservices/userver/grpc/functional_tests/basic_server/tests-unix-socket/test_header_propagation.py", "grpc/functional_tests/metrics/CMakeLists.txt":"taxi/uservices/userver/grpc/functional_tests/metrics/CMakeLists.txt", "grpc/functional_tests/metrics/config_vars.yaml":"taxi/uservices/userver/grpc/functional_tests/metrics/config_vars.yaml", "grpc/functional_tests/metrics/proto/samples/greeter.proto":"taxi/uservices/userver/grpc/functional_tests/metrics/proto/samples/greeter.proto", @@ -1795,6 +1795,7 @@ "grpc/include/userver/ugrpc/server/impl/codegen_definitions.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/server/impl/codegen_definitions.hpp", "grpc/include/userver/ugrpc/server/impl/error_code.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/server/impl/error_code.hpp", "grpc/include/userver/ugrpc/server/impl/queue_holder.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/server/impl/queue_holder.hpp", + "grpc/include/userver/ugrpc/server/impl/request_container.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/server/impl/request_container.hpp", "grpc/include/userver/ugrpc/server/impl/service_worker.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/server/impl/service_worker.hpp", "grpc/include/userver/ugrpc/server/impl/service_worker_impl.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/server/impl/service_worker_impl.hpp", "grpc/include/userver/ugrpc/server/middlewares/baggage/component.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/server/middlewares/baggage/component.hpp", diff --git a/core/include/userver/server/http/http_response.hpp b/core/include/userver/server/http/http_response.hpp index fa4f7614237a..0a6c1dd1f260 100644 --- a/core/include/userver/server/http/http_response.hpp +++ b/core/include/userver/server/http/http_response.hpp @@ -65,6 +65,11 @@ class HttpResponse final : public request::ResponseBase { /// were already sent for stream'ed response and the new header was not set. bool SetHeader(std::string_view name, std::string value); + /// @brief Add a new response header or rewrite an existing one. + /// @returns true if the header was set. Returns false if headers + /// were already sent for stream'ed response and the new header was not set. + bool SetHeader(std::string_view name, std::string_view value); + /// @overload bool SetHeader( const USERVER_NAMESPACE::http::headers::PredefinedHeader& header, diff --git a/core/include/userver/server/request/task_inherited_request.hpp b/core/include/userver/server/request/task_inherited_request.hpp index ff2cd0cf4224..7bc493c6e926 100644 --- a/core/include/userver/server/request/task_inherited_request.hpp +++ b/core/include/userver/server/request/task_inherited_request.hpp @@ -1,38 +1,38 @@ #pragma once /// @file userver/server/request/task_inherited_request.hpp -/// @brief Functions that provide access to HttpRequest stored in +/// @brief Functions that provide access to incoming request stored in /// TaskInheritedVariable. -#include +#include #include USERVER_NAMESPACE_BEGIN -namespace http::headers { -class PredefinedHeader; -} - namespace server::request { -/// @brief Get a header from server::http::HttpRequest that is handled by the +/// @brief Class for representing incoming request that is handled by the /// current task hierarchy. -/// @return Header value or an empty string, if none such -const std::string& GetTaskInheritedHeader(std::string_view header_name); +class RequestContainer { + public: + virtual std::string_view GetHeader(std::string_view header_name) const = 0; + virtual bool HasHeader(std::string_view header_name) const = 0; + virtual ~RequestContainer() = default; +}; + +/// @brief Set a request that is handled by the current task hierarchy. +void SetTaskInheritedRequest(const RequestContainer& request); -/// @overload -const std::string& GetTaskInheritedHeader( - const USERVER_NAMESPACE::http::headers::PredefinedHeader& header_name); +/// @brief Get a header from incoming request that is handled by the +/// current task hierarchy. +/// @return Header value or an empty string, if none such +std::string_view GetTaskInheritedHeader(std::string_view header_name); -/// @brief Checks whether specified header exists in server::http::HttpRequest +/// @brief Checks whether specified header exists in incoming request /// that is handled by the current task hierarchy. /// @return `true` if the header exists, `false` otherwise bool HasTaskInheritedHeader(std::string_view header_name); -/// @overload -bool HasTaskInheritedHeader( - const USERVER_NAMESPACE::http::headers::PredefinedHeader& header_name); - } // namespace server::request USERVER_NAMESPACE_END diff --git a/core/src/server/http/http_request_handler.cpp b/core/src/server/http/http_request_handler.cpp index a5786923bf86..1c6224c3e11a 100644 --- a/core/src/server/http/http_request_handler.cpp +++ b/core/src/server/http/http_request_handler.cpp @@ -5,7 +5,6 @@ #include #include -#include #include #include #include @@ -79,6 +78,28 @@ namespace { utils::statistics::MetricTag> kCcStatusCodeIsCustom{ "congestion-control.rps.is-custom-status-activated"}; +class HttpRequestContainer final : public server::request::RequestContainer { + public: + explicit HttpRequestContainer(std::shared_ptr request) + : request_(std::move(request)) { + UASSERT_MSG( + request_, + "Should not be called outside of the task that does HTTP handling"); + } + std::string_view GetHeader(std::string_view header_name) const override { + if (!request_) { + return {}; + } + return request_->GetHeader(header_name); + } + bool HasHeader(std::string_view header_name) const override { + return request_ && request_->HasHeader(header_name); + } + + private: + std::shared_ptr request_; +}; + } // namespace engine::TaskWithResult HttpRequestHandler::StartRequestTask( @@ -165,8 +186,9 @@ engine::TaskWithResult HttpRequestHandler::StartRequestTask( } auto payload = [request = std::move(request), handler] { - server::request::kTaskInheritedRequest.Set( + HttpRequestContainer request_container( std::static_pointer_cast(request)); + SetTaskInheritedRequest(request_container); request->SetTaskStartTime(); diff --git a/core/src/server/http/http_response.cpp b/core/src/server/http/http_response.cpp index ac36d543b692..34c7a6f81452 100644 --- a/core/src/server/http/http_response.cpp +++ b/core/src/server/http/http_response.cpp @@ -160,6 +160,10 @@ bool HttpResponse::SetHeader(std::string_view name, std::string value) { return SetHeader(std::string{name}, std::move(value)); } +bool HttpResponse::SetHeader(std::string_view name, std::string_view value) { + return SetHeader(std::string{name}, std::string{value}); +} + bool HttpResponse::SetHeader( const USERVER_NAMESPACE::http::headers::PredefinedHeader& header, std::string value) { diff --git a/core/src/server/request/task_inherited_request.cpp b/core/src/server/request/task_inherited_request.cpp index 14e701f33173..64d91ef5a101 100644 --- a/core/src/server/request/task_inherited_request.cpp +++ b/core/src/server/request/task_inherited_request.cpp @@ -1,53 +1,38 @@ #include -#include -#include +#include + +#include USERVER_NAMESPACE_BEGIN namespace server::request { namespace { -const std::string kEmptyHeader{}; +inline engine::TaskInheritedVariable< + std::reference_wrapper> + kTaskInheritedRequest; +} // namespace -template -const std::string& DoGetTaskInheritedHeader(const Header& header_name) { +void SetTaskInheritedRequest(const RequestContainer& request) { + kTaskInheritedRequest.Set(request); +} + +std::string_view GetTaskInheritedHeader(std::string_view header_name) { const auto* request = kTaskInheritedRequest.GetOptional(); if (request == nullptr) { - return kEmptyHeader; + return {}; } - return (*request)->GetHeader(header_name); + return (*request).get().GetHeader(header_name); } -template -bool DoHasTaskInheritedHeader(const Header& header_name) { +bool HasTaskInheritedHeader(std::string_view header_name) { const auto* request = kTaskInheritedRequest.GetOptional(); if (request == nullptr) { return false; } - return (*request)->HasHeader(header_name); -} - -} // namespace - -const std::string& GetTaskInheritedHeader(std::string_view header_name) { - return DoGetTaskInheritedHeader(header_name); + return (*request).get().HasHeader(header_name); } - -const std::string& GetTaskInheritedHeader( - const USERVER_NAMESPACE::http::headers::PredefinedHeader& header_name) { - return DoGetTaskInheritedHeader(header_name); -} - -bool HasTaskInheritedHeader(std::string_view header_name) { - return DoHasTaskInheritedHeader(header_name); -} - -bool HasTaskInheritedHeader( - const USERVER_NAMESPACE::http::headers::PredefinedHeader& header_name) { - return DoHasTaskInheritedHeader(header_name); -} - } // namespace server::request USERVER_NAMESPACE_END diff --git a/core/src/server/request/task_inherited_request_impl.hpp b/core/src/server/request/task_inherited_request_impl.hpp deleted file mode 100644 index 194d9b33e983..000000000000 --- a/core/src/server/request/task_inherited_request_impl.hpp +++ /dev/null @@ -1,18 +0,0 @@ -#pragma once - -#include - -USERVER_NAMESPACE_BEGIN - -namespace server::http { -class HttpRequestImpl; -} // namespace server::http - -namespace server::request { - -inline engine::TaskInheritedVariable> - kTaskInheritedRequest; - -} // namespace server::request - -USERVER_NAMESPACE_END diff --git a/grpc/functional_tests/basic_server/grpc_service.cpp b/grpc/functional_tests/basic_server/grpc_service.cpp index 8a1aa7c21c88..238ac63d871b 100644 --- a/grpc/functional_tests/basic_server/grpc_service.cpp +++ b/grpc/functional_tests/basic_server/grpc_service.cpp @@ -4,10 +4,14 @@ #include +#include +#include #include +#include #include #include #include +#include #include #include @@ -25,9 +29,29 @@ class GreeterServiceComponent final GreeterServiceComponent(const components::ComponentConfig& config, const components::ComponentContext& context) - : api::GreeterServiceBase::Component(config, context) {} + : api::GreeterServiceBase::Component(config, context), + echo_url_{config["echo-url"].As()}, + http_client_( + context.FindComponent().GetHttpClient()) {} void SayHello(SayHelloCall& call, api::GreetingRequest&& request) override; + void CallEchoNobody(SayHelloCall& call, + api::GreetingRequest&& request) override; + static yaml_config::Schema GetStaticConfigSchema() { + return yaml_config::MergeSchemas(R"( + type: object + description: HTTP echo without body component + additionalProperties: false + properties: + echo-url: + type: string + description: some other microservice listens on this URL + )"); + } + + private: + const std::string echo_url_; + clients::http::Client& http_client_; }; void GreeterServiceComponent::SayHello( @@ -38,6 +62,20 @@ void GreeterServiceComponent::SayHello( call.Finish(response); } +void GreeterServiceComponent::CallEchoNobody( + api::GreeterServiceBase::SayHelloCall& call, + samples::api::GreetingRequest&&) { + api::GreetingResponse response; + response.set_greeting("Call Echo Nobody"); + auto http_response = http_client_.CreateRequest() + .get(echo_url_) + .retry(1) + .timeout(std::chrono::seconds{5}) + .perform(); + http_response->raise_for_status(); + call.Finish(response); +} + } // namespace samples int main(int argc, char* argv[]) { @@ -46,6 +84,9 @@ int main(int argc, char* argv[]) { .Append() .Append() .Append() - .Append(); + .Append() + .Append() + .Append() + .Append(); return utils::DaemonMain(argc, argv, component_list); } diff --git a/grpc/functional_tests/basic_server/proto/samples/greeter.proto b/grpc/functional_tests/basic_server/proto/samples/greeter.proto index b239fa823b85..ab4702ff61f1 100644 --- a/grpc/functional_tests/basic_server/proto/samples/greeter.proto +++ b/grpc/functional_tests/basic_server/proto/samples/greeter.proto @@ -4,6 +4,7 @@ package samples.api; service GreeterService { rpc SayHello(GreetingRequest) returns(GreetingResponse) {} + rpc CallEchoNobody(GreetingRequest) returns(GreetingResponse) {} } message GreetingRequest { diff --git a/grpc/functional_tests/basic_server/static_config.yaml b/grpc/functional_tests/basic_server/static_config.yaml index 6a10a570482f..b0e8203fbb56 100644 --- a/grpc/functional_tests/basic_server/static_config.yaml +++ b/grpc/functional_tests/basic_server/static_config.yaml @@ -27,9 +27,19 @@ components_manager: # Our GreeterService implementation greeter-service: + echo-url: 'mockserver/v1/translations' # Some other microservice listens on this URL task-processor: main-task-processor middlewares: [] + http-client: + fs-task-processor: main-task-processor + + dns-client: + fs-task-processor: fs-task-processor + + headers-propagator: + headers: ["custom-header-1", "custom-header-2"] + server: listener: port: 8092 diff --git a/grpc/functional_tests/basic_server/tests-unix-socket/conftest.py b/grpc/functional_tests/basic_server/tests-unix-socket/conftest.py index 38040af039ac..cc4bfa037c88 100644 --- a/grpc/functional_tests/basic_server/tests-unix-socket/conftest.py +++ b/grpc/functional_tests/basic_server/tests-unix-socket/conftest.py @@ -5,7 +5,18 @@ pytest_plugins = ['pytest_userver.plugins.grpc'] -USERVER_CONFIG_HOOKS = ['prepare_service_config'] +USERVER_CONFIG_HOOKS = ['prepare_service_config', 'config_echo_url'] + + +@pytest.fixture(scope='session') +def config_echo_url(mockserver_info): + def _do_patch(config_yaml, config_vars): + components = config_yaml['components_manager']['components'] + components['greeter-service']['echo-url'] = mockserver_info.url( + '/test-service/echo-no-body', + ) + + return _do_patch @pytest.fixture(scope='session') diff --git a/grpc/functional_tests/basic_server/tests-unix-socket/test_header_propagation.py b/grpc/functional_tests/basic_server/tests-unix-socket/test_header_propagation.py new file mode 100644 index 000000000000..98c3520826b9 --- /dev/null +++ b/grpc/functional_tests/basic_server/tests-unix-socket/test_header_propagation.py @@ -0,0 +1,39 @@ +from samples import greeter_pb2 + + +HEADERS_TO_PROPAGATE = ['custom-header-1', 'custom-header-2'] +HEADERS_NOT_TO_PROPAGATE = [ + 'custom-header-1-non-prop', + 'custom-header-2-non-prop', +] + + +async def test_propagation_headers_mixed_from_grpc(grpc_client, mockserver): + headers = { + HEADERS_NOT_TO_PROPAGATE[0]: '1', + HEADERS_NOT_TO_PROPAGATE[1]: '2', + HEADERS_TO_PROPAGATE[0]: '3', + HEADERS_TO_PROPAGATE[1]: '4', + } + + @mockserver.json_handler('/test-service/echo-no-body') + async def nobody_handler(request): + assert ( + request.headers[HEADERS_TO_PROPAGATE[0]] + == headers[HEADERS_TO_PROPAGATE[0]] + ) + assert ( + request.headers[HEADERS_TO_PROPAGATE[1]] + == headers[HEADERS_TO_PROPAGATE[1]] + ) + assert HEADERS_NOT_TO_PROPAGATE[0] not in request.headers + assert HEADERS_NOT_TO_PROPAGATE[1] not in request.headers + return mockserver.make_response() + + metadata = [] + for key, value in headers.items(): + metadata.append((key, value)) + request = greeter_pb2.GreetingRequest(name='Python') + response = await grpc_client.CallEchoNobody(request, metadata=metadata) + assert response.greeting == 'Call Echo Nobody' + assert nobody_handler.times_called == 1 diff --git a/grpc/include/userver/ugrpc/server/impl/request_container.hpp b/grpc/include/userver/ugrpc/server/impl/request_container.hpp new file mode 100644 index 000000000000..e162cbfe01e6 --- /dev/null +++ b/grpc/include/userver/ugrpc/server/impl/request_container.hpp @@ -0,0 +1,37 @@ +#pragma once + +#include + +#include + +#include + +USERVER_NAMESPACE_BEGIN + +namespace ugrpc::server::impl { + +class GrpcRequestContainer final + : public USERVER_NAMESPACE::server::request::RequestContainer { + public: + GrpcRequestContainer(const grpc::ServerContext& server_context) + : server_context_(server_context){}; + std::string_view GetHeader(std::string_view header_name) const override { + const auto& header_value = utils::FindOrDefault( + server_context_.client_metadata(), + grpc::string_ref{header_name.data(), header_name.size()}); + return {header_value.data(), header_value.size()}; + }; + bool HasHeader(std::string_view header_name) const override { + return nullptr != + utils::FindOrNullptr( + server_context_.client_metadata(), + grpc::string_ref{header_name.data(), header_name.size()}); + }; + + private: + const grpc::ServerContext& server_context_; +}; + +} // namespace ugrpc::server::impl + +USERVER_NAMESPACE_END diff --git a/grpc/include/userver/ugrpc/server/impl/service_worker_impl.hpp b/grpc/include/userver/ugrpc/server/impl/service_worker_impl.hpp index 01a54338f185..1db6ffb895e7 100644 --- a/grpc/include/userver/ugrpc/server/impl/service_worker_impl.hpp +++ b/grpc/include/userver/ugrpc/server/impl/service_worker_impl.hpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -33,6 +34,7 @@ #include #include #include +#include #include #include #include @@ -197,6 +199,9 @@ class CallData final { if constexpr (!std::is_same_v) { initial_request = &initial_request_; } + GrpcRequestContainer request_container(context_); + USERVER_NAMESPACE::server::request::SetTaskInheritedRequest( + request_container); MiddlewareCallContext middleware_context( middlewares, responder, do_call, From 9bc5fe851dfa6df4ca835e9fe441a7dc3fdc1f88 Mon Sep 17 00:00:00 2001 From: Vasily Kulikov Date: Mon, 15 Jul 2024 13:17:46 +0300 Subject: [PATCH 026/629] bug otlp: build fix for gh CI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tests: протестировано CI b7f5fa76c619a091328b1ca930fb9e04eabd1059 Pull Request resolved: https://github.com/userver-framework/userver/pull/646 --- .github/workflows/docker.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index 6bf7e6a93e7a..9892557d79fd 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -21,6 +21,7 @@ jobs: -DUSERVER_FEATURE_REDIS=0 -DUSERVER_FEATURE_CLICKHOUSE=0 -DUSERVER_FEATURE_GRPC=0 + -DUSERVER_FEATURE_OTLP=0 -DUSERVER_FEATURE_KAFKA=0 -DUSERVER_FEATURE_RABBITMQ=0 -DUSERVER_FEATURE_ROCKS=0 From 4542a8ddc997c5757daab44bc4486783f441865d Mon Sep 17 00:00:00 2001 From: dvirnyak Date: Mon, 15 Jul 2024 15:17:32 +0300 Subject: [PATCH 027/629] feat userver: remove pg_last_xlog_replay_location() for AWS Aurora remove pg_last_xlog_replay_location() for AWS Aurora 8f242ae5a3af2f24ccc53f68c6458876d8ce74b1 --- .../postgres/detail/topology/hot_standby.cpp | 42 +++++++++++++++---- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/postgresql/src/storages/postgres/detail/topology/hot_standby.cpp b/postgresql/src/storages/postgres/detail/topology/hot_standby.cpp index badeb47cde74..23aa27634059 100644 --- a/postgresql/src/storages/postgres/detail/topology/hot_standby.cpp +++ b/postgresql/src/storages/postgres/detail/topology/hot_standby.cpp @@ -39,31 +39,58 @@ using ReplicationLag = std::chrono::milliseconds; constexpr const char* kDiscoveryTaskName = "pg_topology"; const std::string kShowSyncStandbyNames = "SHOW synchronous_standby_names"; +const std::string kCheckAuroraQuery = + "SELECT 'aws_commons' = ANY(string_to_array(setting, ', ')) " + "FROM pg_settings " + "WHERE name = 'rds.extensions';"; + +bool IsAuroraConnection(std::unique_ptr& connection) { + const auto res = connection->Execute(kCheckAuroraQuery); + + if (res.IsEmpty()) { + return false; + } + bool is_aurora = false; + res.Front().To(is_aurora); + + return is_aurora; +} struct WalInfoStatements { int min_version; + bool is_aurora; std::string master; std::string slave; }; -const WalInfoStatements& GetWalInfoStatementsForVersion(int version) { +const WalInfoStatements& GetWalInfoStatementsForVersion(int version, + bool is_aurora) { // must be in descending min_version order static const WalInfoStatements kKnownStatements[]{ // Master doesn't provide last xact timestamp without // `track_commit_timestamp` enabled, use current server time // as an approximation. // Also note that all times here are sourced from the master. - {100000, "SELECT pg_current_wal_lsn(), now()", + {100000, /* is_aurora */ false, "SELECT pg_current_wal_lsn(), now()", "SELECT pg_last_wal_replay_lsn(), pg_last_xact_replay_timestamp()"}, // Versions for 9.4-9.x servers. // (Functions were actually available since 8.2 but returned TEXT.) - {90400, "SELECT pg_current_xlog_location(), now()", + {90400, /* is_aurora */ false, "SELECT pg_current_xlog_location(), now()", "SELECT pg_last_xlog_replay_location(), " - "pg_last_xact_replay_timestamp()"}}; + "pg_last_xact_replay_timestamp()"}, + + // AWS Aurora doesn't support pg functions above, so use aurora's + // alternatives instead + {90400, /* is_aurora */ true, + "SELECT highest_lsn_rcvd, now() " + "FROM aurora_replica_status() WHERE session_id != 'MASTER_SESSION_ID'", + "SELECT current_read_lsn, " + "now() - (replica_lag_in_msec || ' milliseconds')::interval " + "FROM aurora_replica_status() WHERE session_id != 'MASTER_SESSION_ID'"}}; for (const auto& cand : kKnownStatements) { - if (version >= cand.min_version) return cand; + if (is_aurora == cand.is_aurora && version >= cand.min_version) return cand; } throw PoolError{fmt::format("Unsupported database version: {}", version)}; } @@ -311,8 +338,9 @@ void HotStandby::RunCheck(DsnIndex idx) { state.roundtrip_time = std::chrono::duration_cast( std::chrono::steady_clock::now() - start); - const auto& wal_info_stmts = - GetWalInfoStatementsForVersion(state.connection->GetServerVersion()); + const bool is_aurora = IsAuroraConnection(state.connection); + const auto& wal_info_stmts = GetWalInfoStatementsForVersion( + state.connection->GetServerVersion(), is_aurora); std::optional current_xact_timestamp; const auto wal_info = state.connection->Execute( From 4d1d055f2e6232c6ef8fdf8c21cad7b439631144 Mon Sep 17 00:00:00 2001 From: segoon Date: Mon, 15 Jul 2024 17:21:03 +0300 Subject: [PATCH 028/629] docs otlp: more docs cb27b8a0a2bddab10ead7800b8f5daca06dfdba7 --- .mapping.json | 1 - samples/otlp_service/config_vars.yaml | 18 ----------------- samples/otlp_service/static_config.yaml | 15 ++++---------- scripts/docs/en/userver/logging.md | 26 +++++++++++++++++++++++++ 4 files changed, 30 insertions(+), 30 deletions(-) delete mode 100644 samples/otlp_service/config_vars.yaml diff --git a/.mapping.json b/.mapping.json index 7c931937e77a..8d97f0e3608f 100644 --- a/.mapping.json +++ b/.mapping.json @@ -2942,7 +2942,6 @@ "samples/mysql_service/tests/conftest.py":"taxi/uservices/userver/samples/mysql_service/tests/conftest.py", "samples/mysql_service/tests/test_mysql.py":"taxi/uservices/userver/samples/mysql_service/tests/test_mysql.py", "samples/otlp_service/CMakeLists.txt":"taxi/uservices/userver/samples/otlp_service/CMakeLists.txt", - "samples/otlp_service/config_vars.yaml":"taxi/uservices/userver/samples/otlp_service/config_vars.yaml", "samples/otlp_service/otlp_service.cpp":"taxi/uservices/userver/samples/otlp_service/otlp_service.cpp", "samples/otlp_service/secure_data.json":"taxi/uservices/userver/samples/otlp_service/secure_data.json", "samples/otlp_service/static_config.yaml":"taxi/uservices/userver/samples/otlp_service/static_config.yaml", diff --git a/samples/otlp_service/config_vars.yaml b/samples/otlp_service/config_vars.yaml deleted file mode 100644 index f5e3d5fc236f..000000000000 --- a/samples/otlp_service/config_vars.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# yaml -server-name: sample-otlp-service 1.0 -service-name: sample-otlp-service -logger-level: info - -config-server-url: http://localhost:8083/ -server-port: 8085 -monitor-server-port: 8086 - -testsuite-enabled: false - -userver-dumps-root: /var/cache/otlp_service/userver-dumps/ -access-log-path: /var/log/otlp_service/access.log -access-tskv-log-path: /var/log/otlp_service/access_tskv.log -default-log-path: /var/log/otlp_service/server.log -secdist-path: /etc/otlp_service/secure_data.json - -config-cache: /var/cache/otlp_service/config_cache.json diff --git a/samples/otlp_service/static_config.yaml b/samples/otlp_service/static_config.yaml index 56541843be37..7a81a9dacf83 100644 --- a/samples/otlp_service/static_config.yaml +++ b/samples/otlp_service/static_config.yaml @@ -5,13 +5,7 @@ components_manager: logging: fs-task-processor: fs-task-processor loggers: {} - # default: - # file_path: '@stderr' - # level: debug - # overflow_behavior: discard -# /// [gRPC sample - static config client] -# yaml # Creates gRPC clients grpc-client-factory: # The TaskProcessor for blocking connection initiation @@ -21,10 +15,12 @@ components_manager: # https://grpc.github.io/grpc/core/group__grpc__arg__keys.html channel-args: {} +# /// [otlp logger] otlp-logger: endpoint: '0.0.0.0:4317' - service-name: 'otlp-example' - log-level: 'debug' + service-name: otlp-example + log-level: debug +# /// [otlp logger] handler-server-monitor: path: /metrics @@ -50,13 +46,10 @@ components_manager: default_task_processor: main-task-processor # Task processor in which components start. -# /// [gRPC sample - task processor] -# yaml task_processors: grpc-blocking-task-processor: # For blocking gRPC channel creation worker_threads: 2 thread_name: grpc-worker -# /// [gRPC sample - task processor] main-task-processor: # For non-blocking operations worker_threads: 8 fs-task-processor: # For blocking filesystem operations diff --git a/scripts/docs/en/userver/logging.md b/scripts/docs/en/userver/logging.md index c16779c2b3b6..70d138cc04d7 100644 --- a/scripts/docs/en/userver/logging.md +++ b/scripts/docs/en/userver/logging.md @@ -229,6 +229,32 @@ For example, this is how you can disable logging of all Span for MongoDB (that i ``` +## OpenTelemetry protocol + +It is possible to use OpenTelemetry protocol for logs and tracing exporting. +To use it, register all prerequisites for gRPC client and register `otlp::LoggerComponent` in component list in your `main` function. +The static config file should contain a component section with OTLP logger. +Also remove `default` logger from `logging.loggers` as the default logger is handled by otlp component from now on. + +The related part of static config: + +@snippet samples/otlp_service/static_config.yaml otlp logger + +`endpoint` contains OTLP collector host and port, +`service-name` contains your service name, +and `log-level` contains service log level. + +After the service start you'll see the following in stderr: + +``` + OTLP logger has started +``` + +It means the logger is successfully initialized and is ready to process the logs. + +If somethings goes wrong (e.g. OTLP collector agent is not available), you'll see errors in stderr. +The service buffers not-yet-sent logs and traces in memory, but drops them on overflow. + ---------- @htmlonly
@endhtmlonly From 4fe95a1f5ce73cd9a547fc609224dcfd0c485ad6 Mon Sep 17 00:00:00 2001 From: segoon Date: Mon, 15 Jul 2024 18:33:00 +0300 Subject: [PATCH 029/629] feat otlp: do not copy opentelemetry proto schemas 2bc242ed301cedffef77107e83985e3b280bb140 --- .mapping.json | 9 +- cmake/SetupOpentelemetryProto.cmake | 29 ++ otlp/CMakeLists.txt | 15 +- .../opentelemetry/proto/collector/README.md | 10 - .../collector/logs/v1/logs_service.proto | 79 ---- .../metrics/v1/metrics_service.proto | 79 ---- .../collector/trace/v1/trace_service.proto | 79 ---- .../proto/common/v1/common.proto | 81 ---- .../opentelemetry/proto/logs/v1/logs.proto | 211 ----------- .../proto/resource/v1/resource.proto | 37 -- .../opentelemetry/proto/trace/v1/trace.proto | 355 ------------------ scripts/docker/base-ubuntu-22.04.dockerfile | 10 +- 12 files changed, 42 insertions(+), 952 deletions(-) create mode 100644 cmake/SetupOpentelemetryProto.cmake delete mode 100644 otlp/proto/opentelemetry/proto/collector/README.md delete mode 100644 otlp/proto/opentelemetry/proto/collector/logs/v1/logs_service.proto delete mode 100644 otlp/proto/opentelemetry/proto/collector/metrics/v1/metrics_service.proto delete mode 100644 otlp/proto/opentelemetry/proto/collector/trace/v1/trace_service.proto delete mode 100644 otlp/proto/opentelemetry/proto/common/v1/common.proto delete mode 100644 otlp/proto/opentelemetry/proto/logs/v1/logs.proto delete mode 100644 otlp/proto/opentelemetry/proto/resource/v1/resource.proto delete mode 100644 otlp/proto/opentelemetry/proto/trace/v1/trace.proto diff --git a/.mapping.json b/.mapping.json index 8d97f0e3608f..443efa11e794 100644 --- a/.mapping.json +++ b/.mapping.json @@ -317,6 +317,7 @@ "cmake/SetupGrpc.cmake":"taxi/uservices/userver/cmake/SetupGrpc.cmake", "cmake/SetupLTO.cmake":"taxi/uservices/userver/cmake/SetupLTO.cmake", "cmake/SetupLinker.cmake":"taxi/uservices/userver/cmake/SetupLinker.cmake", + "cmake/SetupOpentelemetryProto.cmake":"taxi/uservices/userver/cmake/SetupOpentelemetryProto.cmake", "cmake/SetupPostgresqlDeps.cmake":"taxi/uservices/userver/cmake/SetupPostgresqlDeps.cmake", "cmake/SetupProtobuf.cmake":"taxi/uservices/userver/cmake/SetupProtobuf.cmake", "cmake/SetupRdKafka.cmake":"taxi/uservices/userver/cmake/SetupRdKafka.cmake", @@ -2234,14 +2235,6 @@ "mysql/src/storages/tests/utils_mysqltest.hpp":"taxi/uservices/userver/mysql/src/storages/tests/utils_mysqltest.hpp", "otlp/CMakeLists.txt":"taxi/uservices/userver/otlp/CMakeLists.txt", "otlp/include/userver/otlp/logs/component.hpp":"taxi/uservices/userver/otlp/include/userver/otlp/logs/component.hpp", - "otlp/proto/opentelemetry/proto/collector/README.md":"taxi/uservices/userver/otlp/proto/opentelemetry/proto/collector/README.md", - "otlp/proto/opentelemetry/proto/collector/logs/v1/logs_service.proto":"taxi/uservices/userver/otlp/proto/opentelemetry/proto/collector/logs/v1/logs_service.proto", - "otlp/proto/opentelemetry/proto/collector/metrics/v1/metrics_service.proto":"taxi/uservices/userver/otlp/proto/opentelemetry/proto/collector/metrics/v1/metrics_service.proto", - "otlp/proto/opentelemetry/proto/collector/trace/v1/trace_service.proto":"taxi/uservices/userver/otlp/proto/opentelemetry/proto/collector/trace/v1/trace_service.proto", - "otlp/proto/opentelemetry/proto/common/v1/common.proto":"taxi/uservices/userver/otlp/proto/opentelemetry/proto/common/v1/common.proto", - "otlp/proto/opentelemetry/proto/logs/v1/logs.proto":"taxi/uservices/userver/otlp/proto/opentelemetry/proto/logs/v1/logs.proto", - "otlp/proto/opentelemetry/proto/resource/v1/resource.proto":"taxi/uservices/userver/otlp/proto/opentelemetry/proto/resource/v1/resource.proto", - "otlp/proto/opentelemetry/proto/trace/v1/trace.proto":"taxi/uservices/userver/otlp/proto/opentelemetry/proto/trace/v1/trace.proto", "otlp/src/otlp/logs/component.cpp":"taxi/uservices/userver/otlp/src/otlp/logs/component.cpp", "otlp/src/otlp/logs/logger.cpp":"taxi/uservices/userver/otlp/src/otlp/logs/logger.cpp", "otlp/src/otlp/logs/logger.hpp":"taxi/uservices/userver/otlp/src/otlp/logs/logger.hpp", diff --git a/cmake/SetupOpentelemetryProto.cmake b/cmake/SetupOpentelemetryProto.cmake new file mode 100644 index 000000000000..f50520f2f8d1 --- /dev/null +++ b/cmake/SetupOpentelemetryProto.cmake @@ -0,0 +1,29 @@ +include_guard(GLOBAL) + + +set(USERVER_OPENTELEMETRY_PROTO "" CACHE PATH "Path to the folder with opentelemetry proto files") +if (NOT USERVER_OPENTELEMETRY_PROTO) + include(DownloadUsingCPM) + CPMAddPackage( + NAME opentelemetry_proto + GITHUB_REPOSITORY open-telemetry/opentelemetry-proto + VERSION 1.3.2 + DOWNLOAD_ONLY + ) +else() + set(opentelemetry_proto_SOURCE_DIR ${USERVER_OPENTELEMETRY_PROTO}) +endif() +message(STATUS "Opentelemetry proto path: ${opentelemetry_proto_SOURCE_DIR}") + +userver_add_grpc_library(userver-otlp-proto + SOURCE_PATH + ${opentelemetry_proto_SOURCE_DIR} + PROTOS + ${opentelemetry_proto_SOURCE_DIR}/opentelemetry/proto/collector/trace/v1/trace_service.proto + ${opentelemetry_proto_SOURCE_DIR}/opentelemetry/proto/collector/logs/v1/logs_service.proto + ${opentelemetry_proto_SOURCE_DIR}/opentelemetry/proto/common/v1/common.proto + ${opentelemetry_proto_SOURCE_DIR}/opentelemetry/proto/logs/v1/logs.proto + ${opentelemetry_proto_SOURCE_DIR}/opentelemetry/proto/resource/v1/resource.proto + ${opentelemetry_proto_SOURCE_DIR}/opentelemetry/proto/trace/v1/trace.proto +) + diff --git a/otlp/CMakeLists.txt b/otlp/CMakeLists.txt index 69b6c29b3ef6..5e84f64b6640 100644 --- a/otlp/CMakeLists.txt +++ b/otlp/CMakeLists.txt @@ -1,6 +1,7 @@ project(userver-otlp CXX) include(GrpcTargets) +include(SetupOpentelemetryProto) find_package(Boost REQUIRED regex) @@ -18,16 +19,6 @@ file(GLOB_RECURSE BENCH_SOURCES ) list (REMOVE_ITEM SOURCES ${BENCH_SOURCES}) -userver_add_grpc_library(${PROJECT_NAME}-proto - PROTOS - opentelemetry/proto/collector/trace/v1/trace_service.proto - opentelemetry/proto/collector/logs/v1/logs_service.proto - opentelemetry/proto/common/v1/common.proto - opentelemetry/proto/logs/v1/logs.proto - opentelemetry/proto/resource/v1/resource.proto - opentelemetry/proto/trace/v1/trace.proto -) - add_library(${PROJECT_NAME} STATIC ${SOURCES}) set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE CXX) @@ -44,14 +35,14 @@ target_include_directories (${PROJECT_NAME} PRIVATE target_link_libraries(${PROJECT_NAME} PUBLIC userver-core - ${PROJECT_NAME}-proto + userver-otlp-proto ) _userver_directory_install(COMPONENT otlp DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/.. ) -_userver_install_targets(COMPONENT otlp TARGETS ${PROJECT_NAME} ${PROJECT_NAME}-proto) +_userver_install_targets(COMPONENT otlp TARGETS ${PROJECT_NAME} userver-otlp-proto) _userver_directory_install(COMPONENT otlp FILES "${USERVER_ROOT_DIR}/cmake/install/userver-otlp-config.cmake" diff --git a/otlp/proto/opentelemetry/proto/collector/README.md b/otlp/proto/opentelemetry/proto/collector/README.md deleted file mode 100644 index f82dbb0278b0..000000000000 --- a/otlp/proto/opentelemetry/proto/collector/README.md +++ /dev/null @@ -1,10 +0,0 @@ -# OpenTelemetry Collector Proto - -This package describes the OpenTelemetry collector protocol. - -## Packages - -1. `common` package contains the common messages shared between different services. -2. `trace` package contains the Trace Service protos. -3. `metrics` package contains the Metrics Service protos. -4. `logs` package contains the Logs Service protos. diff --git a/otlp/proto/opentelemetry/proto/collector/logs/v1/logs_service.proto b/otlp/proto/opentelemetry/proto/collector/logs/v1/logs_service.proto deleted file mode 100644 index 8260d8aaeb82..000000000000 --- a/otlp/proto/opentelemetry/proto/collector/logs/v1/logs_service.proto +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright 2020, OpenTelemetry Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package opentelemetry.proto.collector.logs.v1; - -import "opentelemetry/proto/logs/v1/logs.proto"; - -option csharp_namespace = "OpenTelemetry.Proto.Collector.Logs.V1"; -option java_multiple_files = true; -option java_package = "io.opentelemetry.proto.collector.logs.v1"; -option java_outer_classname = "LogsServiceProto"; -option go_package = "go.opentelemetry.io/proto/otlp/collector/logs/v1"; - -// Service that can be used to push logs between one Application instrumented with -// OpenTelemetry and an collector, or between an collector and a central collector (in this -// case logs are sent/received to/from multiple Applications). -service LogsService { - // For performance reasons, it is recommended to keep this RPC - // alive for the entire life of the application. - rpc Export(ExportLogsServiceRequest) returns (ExportLogsServiceResponse) {} -} - -message ExportLogsServiceRequest { - // An array of ResourceLogs. - // For data coming from a single resource this array will typically contain one - // element. Intermediary nodes (such as OpenTelemetry Collector) that receive - // data from multiple origins typically batch the data before forwarding further and - // in that case this array will contain multiple elements. - repeated opentelemetry.proto.logs.v1.ResourceLogs resource_logs = 1; -} - -message ExportLogsServiceResponse { - // The details of a partially successful export request. - // - // If the request is only partially accepted - // (i.e. when the server accepts only parts of the data and rejects the rest) - // the server MUST initialize the `partial_success` field and MUST - // set the `rejected_` with the number of items it rejected. - // - // Servers MAY also make use of the `partial_success` field to convey - // warnings/suggestions to senders even when the request was fully accepted. - // In such cases, the `rejected_` MUST have a value of `0` and - // the `error_message` MUST be non-empty. - // - // A `partial_success` message with an empty value (rejected_ = 0 and - // `error_message` = "") is equivalent to it not being set/present. Senders - // SHOULD interpret it the same way as in the full success case. - ExportLogsPartialSuccess partial_success = 1; -} - -message ExportLogsPartialSuccess { - // The number of rejected log records. - // - // A `rejected_` field holding a `0` value indicates that the - // request was fully accepted. - int64 rejected_log_records = 1; - - // A developer-facing human-readable message in English. It should be used - // either to explain why the server rejected parts of the data during a partial - // success or to convey warnings/suggestions during a full success. The message - // should offer guidance on how users can address such issues. - // - // error_message is an optional field. An error_message with an empty value - // is equivalent to it not being set. - string error_message = 2; -} diff --git a/otlp/proto/opentelemetry/proto/collector/metrics/v1/metrics_service.proto b/otlp/proto/opentelemetry/proto/collector/metrics/v1/metrics_service.proto deleted file mode 100644 index dd48f1ad3a16..000000000000 --- a/otlp/proto/opentelemetry/proto/collector/metrics/v1/metrics_service.proto +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright 2019, OpenTelemetry Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package opentelemetry.proto.collector.metrics.v1; - -import "opentelemetry/proto/metrics/v1/metrics.proto"; - -option csharp_namespace = "OpenTelemetry.Proto.Collector.Metrics.V1"; -option java_multiple_files = true; -option java_package = "io.opentelemetry.proto.collector.metrics.v1"; -option java_outer_classname = "MetricsServiceProto"; -option go_package = "go.opentelemetry.io/proto/otlp/collector/metrics/v1"; - -// Service that can be used to push metrics between one Application -// instrumented with OpenTelemetry and a collector, or between a collector and a -// central collector. -service MetricsService { - // For performance reasons, it is recommended to keep this RPC - // alive for the entire life of the application. - rpc Export(ExportMetricsServiceRequest) returns (ExportMetricsServiceResponse) {} -} - -message ExportMetricsServiceRequest { - // An array of ResourceMetrics. - // For data coming from a single resource this array will typically contain one - // element. Intermediary nodes (such as OpenTelemetry Collector) that receive - // data from multiple origins typically batch the data before forwarding further and - // in that case this array will contain multiple elements. - repeated opentelemetry.proto.metrics.v1.ResourceMetrics resource_metrics = 1; -} - -message ExportMetricsServiceResponse { - // The details of a partially successful export request. - // - // If the request is only partially accepted - // (i.e. when the server accepts only parts of the data and rejects the rest) - // the server MUST initialize the `partial_success` field and MUST - // set the `rejected_` with the number of items it rejected. - // - // Servers MAY also make use of the `partial_success` field to convey - // warnings/suggestions to senders even when the request was fully accepted. - // In such cases, the `rejected_` MUST have a value of `0` and - // the `error_message` MUST be non-empty. - // - // A `partial_success` message with an empty value (rejected_ = 0 and - // `error_message` = "") is equivalent to it not being set/present. Senders - // SHOULD interpret it the same way as in the full success case. - ExportMetricsPartialSuccess partial_success = 1; -} - -message ExportMetricsPartialSuccess { - // The number of rejected data points. - // - // A `rejected_` field holding a `0` value indicates that the - // request was fully accepted. - int64 rejected_data_points = 1; - - // A developer-facing human-readable message in English. It should be used - // either to explain why the server rejected parts of the data during a partial - // success or to convey warnings/suggestions during a full success. The message - // should offer guidance on how users can address such issues. - // - // error_message is an optional field. An error_message with an empty value - // is equivalent to it not being set. - string error_message = 2; -} diff --git a/otlp/proto/opentelemetry/proto/collector/trace/v1/trace_service.proto b/otlp/proto/opentelemetry/proto/collector/trace/v1/trace_service.proto deleted file mode 100644 index d6fe67f9e553..000000000000 --- a/otlp/proto/opentelemetry/proto/collector/trace/v1/trace_service.proto +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright 2019, OpenTelemetry Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package opentelemetry.proto.collector.trace.v1; - -import "opentelemetry/proto/trace/v1/trace.proto"; - -option csharp_namespace = "OpenTelemetry.Proto.Collector.Trace.V1"; -option java_multiple_files = true; -option java_package = "io.opentelemetry.proto.collector.trace.v1"; -option java_outer_classname = "TraceServiceProto"; -option go_package = "go.opentelemetry.io/proto/otlp/collector/trace/v1"; - -// Service that can be used to push spans between one Application instrumented with -// OpenTelemetry and a collector, or between a collector and a central collector (in this -// case spans are sent/received to/from multiple Applications). -service TraceService { - // For performance reasons, it is recommended to keep this RPC - // alive for the entire life of the application. - rpc Export(ExportTraceServiceRequest) returns (ExportTraceServiceResponse) {} -} - -message ExportTraceServiceRequest { - // An array of ResourceSpans. - // For data coming from a single resource this array will typically contain one - // element. Intermediary nodes (such as OpenTelemetry Collector) that receive - // data from multiple origins typically batch the data before forwarding further and - // in that case this array will contain multiple elements. - repeated opentelemetry.proto.trace.v1.ResourceSpans resource_spans = 1; -} - -message ExportTraceServiceResponse { - // The details of a partially successful export request. - // - // If the request is only partially accepted - // (i.e. when the server accepts only parts of the data and rejects the rest) - // the server MUST initialize the `partial_success` field and MUST - // set the `rejected_` with the number of items it rejected. - // - // Servers MAY also make use of the `partial_success` field to convey - // warnings/suggestions to senders even when the request was fully accepted. - // In such cases, the `rejected_` MUST have a value of `0` and - // the `error_message` MUST be non-empty. - // - // A `partial_success` message with an empty value (rejected_ = 0 and - // `error_message` = "") is equivalent to it not being set/present. Senders - // SHOULD interpret it the same way as in the full success case. - ExportTracePartialSuccess partial_success = 1; -} - -message ExportTracePartialSuccess { - // The number of rejected spans. - // - // A `rejected_` field holding a `0` value indicates that the - // request was fully accepted. - int64 rejected_spans = 1; - - // A developer-facing human-readable message in English. It should be used - // either to explain why the server rejected parts of the data during a partial - // success or to convey warnings/suggestions during a full success. The message - // should offer guidance on how users can address such issues. - // - // error_message is an optional field. An error_message with an empty value - // is equivalent to it not being set. - string error_message = 2; -} diff --git a/otlp/proto/opentelemetry/proto/common/v1/common.proto b/otlp/proto/opentelemetry/proto/common/v1/common.proto deleted file mode 100644 index ff8a21a1fa0e..000000000000 --- a/otlp/proto/opentelemetry/proto/common/v1/common.proto +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright 2019, OpenTelemetry Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package opentelemetry.proto.common.v1; - -option csharp_namespace = "OpenTelemetry.Proto.Common.V1"; -option java_multiple_files = true; -option java_package = "io.opentelemetry.proto.common.v1"; -option java_outer_classname = "CommonProto"; -option go_package = "go.opentelemetry.io/proto/otlp/common/v1"; - -// AnyValue is used to represent any type of attribute value. AnyValue may contain a -// primitive value such as a string or integer or it may contain an arbitrary nested -// object containing arrays, key-value lists and primitives. -message AnyValue { - // The value is one of the listed fields. It is valid for all values to be unspecified - // in which case this AnyValue is considered to be "empty". - oneof value { - string string_value = 1; - bool bool_value = 2; - int64 int_value = 3; - double double_value = 4; - ArrayValue array_value = 5; - KeyValueList kvlist_value = 6; - bytes bytes_value = 7; - } -} - -// ArrayValue is a list of AnyValue messages. We need ArrayValue as a message -// since oneof in AnyValue does not allow repeated fields. -message ArrayValue { - // Array of values. The array may be empty (contain 0 elements). - repeated AnyValue values = 1; -} - -// KeyValueList is a list of KeyValue messages. We need KeyValueList as a message -// since `oneof` in AnyValue does not allow repeated fields. Everywhere else where we need -// a list of KeyValue messages (e.g. in Span) we use `repeated KeyValue` directly to -// avoid unnecessary extra wrapping (which slows down the protocol). The 2 approaches -// are semantically equivalent. -message KeyValueList { - // A collection of key/value pairs of key-value pairs. The list may be empty (may - // contain 0 elements). - // The keys MUST be unique (it is not allowed to have more than one - // value with the same key). - repeated KeyValue values = 1; -} - -// KeyValue is a key-value pair that is used to store Span attributes, Link -// attributes, etc. -message KeyValue { - string key = 1; - AnyValue value = 2; -} - -// InstrumentationScope is a message representing the instrumentation scope information -// such as the fully qualified name and version. -message InstrumentationScope { - // An empty instrumentation scope name means the name is unknown. - string name = 1; - string version = 2; - - // Additional attributes that describe the scope. [Optional]. - // Attribute keys MUST be unique (it is not allowed to have more than one - // attribute with the same key). - repeated KeyValue attributes = 3; - uint32 dropped_attributes_count = 4; -} diff --git a/otlp/proto/opentelemetry/proto/logs/v1/logs.proto b/otlp/proto/opentelemetry/proto/logs/v1/logs.proto deleted file mode 100644 index f9b97dd74510..000000000000 --- a/otlp/proto/opentelemetry/proto/logs/v1/logs.proto +++ /dev/null @@ -1,211 +0,0 @@ -// Copyright 2020, OpenTelemetry Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package opentelemetry.proto.logs.v1; - -import "opentelemetry/proto/common/v1/common.proto"; -import "opentelemetry/proto/resource/v1/resource.proto"; - -option csharp_namespace = "OpenTelemetry.Proto.Logs.V1"; -option java_multiple_files = true; -option java_package = "io.opentelemetry.proto.logs.v1"; -option java_outer_classname = "LogsProto"; -option go_package = "go.opentelemetry.io/proto/otlp/logs/v1"; - -// LogsData represents the logs data that can be stored in a persistent storage, -// OR can be embedded by other protocols that transfer OTLP logs data but do not -// implement the OTLP protocol. -// -// The main difference between this message and collector protocol is that -// in this message there will not be any "control" or "metadata" specific to -// OTLP protocol. -// -// When new fields are added into this message, the OTLP request MUST be updated -// as well. -message LogsData { - // An array of ResourceLogs. - // For data coming from a single resource this array will typically contain - // one element. Intermediary nodes that receive data from multiple origins - // typically batch the data before forwarding further and in that case this - // array will contain multiple elements. - repeated ResourceLogs resource_logs = 1; -} - -// A collection of ScopeLogs from a Resource. -message ResourceLogs { - reserved 1000; - - // The resource for the logs in this message. - // If this field is not set then resource info is unknown. - opentelemetry.proto.resource.v1.Resource resource = 1; - - // A list of ScopeLogs that originate from a resource. - repeated ScopeLogs scope_logs = 2; - - // The Schema URL, if known. This is the identifier of the Schema that the resource data - // is recorded in. To learn more about Schema URL see - // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url - // This schema_url applies to the data in the "resource" field. It does not apply - // to the data in the "scope_logs" field which have their own schema_url field. - string schema_url = 3; -} - -// A collection of Logs produced by a Scope. -message ScopeLogs { - // The instrumentation scope information for the logs in this message. - // Semantically when InstrumentationScope isn't set, it is equivalent with - // an empty instrumentation scope name (unknown). - opentelemetry.proto.common.v1.InstrumentationScope scope = 1; - - // A list of log records. - repeated LogRecord log_records = 2; - - // The Schema URL, if known. This is the identifier of the Schema that the log data - // is recorded in. To learn more about Schema URL see - // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url - // This schema_url applies to all logs in the "logs" field. - string schema_url = 3; -} - -// Possible values for LogRecord.SeverityNumber. -enum SeverityNumber { - // UNSPECIFIED is the default SeverityNumber, it MUST NOT be used. - SEVERITY_NUMBER_UNSPECIFIED = 0; - SEVERITY_NUMBER_TRACE = 1; - SEVERITY_NUMBER_TRACE2 = 2; - SEVERITY_NUMBER_TRACE3 = 3; - SEVERITY_NUMBER_TRACE4 = 4; - SEVERITY_NUMBER_DEBUG = 5; - SEVERITY_NUMBER_DEBUG2 = 6; - SEVERITY_NUMBER_DEBUG3 = 7; - SEVERITY_NUMBER_DEBUG4 = 8; - SEVERITY_NUMBER_INFO = 9; - SEVERITY_NUMBER_INFO2 = 10; - SEVERITY_NUMBER_INFO3 = 11; - SEVERITY_NUMBER_INFO4 = 12; - SEVERITY_NUMBER_WARN = 13; - SEVERITY_NUMBER_WARN2 = 14; - SEVERITY_NUMBER_WARN3 = 15; - SEVERITY_NUMBER_WARN4 = 16; - SEVERITY_NUMBER_ERROR = 17; - SEVERITY_NUMBER_ERROR2 = 18; - SEVERITY_NUMBER_ERROR3 = 19; - SEVERITY_NUMBER_ERROR4 = 20; - SEVERITY_NUMBER_FATAL = 21; - SEVERITY_NUMBER_FATAL2 = 22; - SEVERITY_NUMBER_FATAL3 = 23; - SEVERITY_NUMBER_FATAL4 = 24; -} - -// LogRecordFlags represents constants used to interpret the -// LogRecord.flags field, which is protobuf 'fixed32' type and is to -// be used as bit-fields. Each non-zero value defined in this enum is -// a bit-mask. To extract the bit-field, for example, use an -// expression like: -// -// (logRecord.flags & LOG_RECORD_FLAGS_TRACE_FLAGS_MASK) -// -enum LogRecordFlags { - // The zero value for the enum. Should not be used for comparisons. - // Instead use bitwise "and" with the appropriate mask as shown above. - LOG_RECORD_FLAGS_DO_NOT_USE = 0; - - // Bits 0-7 are used for trace flags. - LOG_RECORD_FLAGS_TRACE_FLAGS_MASK = 0x000000FF; - - // Bits 8-31 are reserved for future use. -} - -// A log record according to OpenTelemetry Log Data Model: -// https://github.com/open-telemetry/oteps/blob/main/text/logs/0097-log-data-model.md -message LogRecord { - reserved 4; - - // time_unix_nano is the time when the event occurred. - // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January 1970. - // Value of 0 indicates unknown or missing timestamp. - fixed64 time_unix_nano = 1; - - // Time when the event was observed by the collection system. - // For events that originate in OpenTelemetry (e.g. using OpenTelemetry Logging SDK) - // this timestamp is typically set at the generation time and is equal to Timestamp. - // For events originating externally and collected by OpenTelemetry (e.g. using - // Collector) this is the time when OpenTelemetry's code observed the event measured - // by the clock of the OpenTelemetry code. This field MUST be set once the event is - // observed by OpenTelemetry. - // - // For converting OpenTelemetry log data to formats that support only one timestamp or - // when receiving OpenTelemetry log data by recipients that support only one timestamp - // internally the following logic is recommended: - // - Use time_unix_nano if it is present, otherwise use observed_time_unix_nano. - // - // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January 1970. - // Value of 0 indicates unknown or missing timestamp. - fixed64 observed_time_unix_nano = 11; - - // Numerical value of the severity, normalized to values described in Log Data Model. - // [Optional]. - SeverityNumber severity_number = 2; - - // The severity text (also known as log level). The original string representation as - // it is known at the source. [Optional]. - string severity_text = 3; - - // A value containing the body of the log record. Can be for example a human-readable - // string message (including multi-line) describing the event in a free form or it can - // be a structured data composed of arrays and maps of other values. [Optional]. - opentelemetry.proto.common.v1.AnyValue body = 5; - - // Additional attributes that describe the specific event occurrence. [Optional]. - // Attribute keys MUST be unique (it is not allowed to have more than one - // attribute with the same key). - repeated opentelemetry.proto.common.v1.KeyValue attributes = 6; - uint32 dropped_attributes_count = 7; - - // Flags, a bit field. 8 least significant bits are the trace flags as - // defined in W3C Trace Context specification. 24 most significant bits are reserved - // and must be set to 0. Readers must not assume that 24 most significant bits - // will be zero and must correctly mask the bits when reading 8-bit trace flag (use - // flags & LOG_RECORD_FLAGS_TRACE_FLAGS_MASK). [Optional]. - fixed32 flags = 8; - - // A unique identifier for a trace. All logs from the same trace share - // the same `trace_id`. The ID is a 16-byte array. An ID with all zeroes OR - // of length other than 16 bytes is considered invalid (empty string in OTLP/JSON - // is zero-length and thus is also invalid). - // - // This field is optional. - // - // The receivers SHOULD assume that the log record is not associated with a - // trace if any of the following is true: - // - the field is not present, - // - the field contains an invalid value. - bytes trace_id = 9; - - // A unique identifier for a span within a trace, assigned when the span - // is created. The ID is an 8-byte array. An ID with all zeroes OR of length - // other than 8 bytes is considered invalid (empty string in OTLP/JSON - // is zero-length and thus is also invalid). - // - // This field is optional. If the sender specifies a valid span_id then it SHOULD also - // specify a valid trace_id. - // - // The receivers SHOULD assume that the log record is not associated with a - // span if any of the following is true: - // - the field is not present, - // - the field contains an invalid value. - bytes span_id = 10; -} diff --git a/otlp/proto/opentelemetry/proto/resource/v1/resource.proto b/otlp/proto/opentelemetry/proto/resource/v1/resource.proto deleted file mode 100644 index 6637560bc354..000000000000 --- a/otlp/proto/opentelemetry/proto/resource/v1/resource.proto +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2019, OpenTelemetry Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package opentelemetry.proto.resource.v1; - -import "opentelemetry/proto/common/v1/common.proto"; - -option csharp_namespace = "OpenTelemetry.Proto.Resource.V1"; -option java_multiple_files = true; -option java_package = "io.opentelemetry.proto.resource.v1"; -option java_outer_classname = "ResourceProto"; -option go_package = "go.opentelemetry.io/proto/otlp/resource/v1"; - -// Resource information. -message Resource { - // Set of attributes that describe the resource. - // Attribute keys MUST be unique (it is not allowed to have more than one - // attribute with the same key). - repeated opentelemetry.proto.common.v1.KeyValue attributes = 1; - - // dropped_attributes_count is the number of dropped attributes. If the value is 0, then - // no attributes were dropped. - uint32 dropped_attributes_count = 2; -} diff --git a/otlp/proto/opentelemetry/proto/trace/v1/trace.proto b/otlp/proto/opentelemetry/proto/trace/v1/trace.proto deleted file mode 100644 index 5cb2f3ce1cd2..000000000000 --- a/otlp/proto/opentelemetry/proto/trace/v1/trace.proto +++ /dev/null @@ -1,355 +0,0 @@ -// Copyright 2019, OpenTelemetry Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package opentelemetry.proto.trace.v1; - -import "opentelemetry/proto/common/v1/common.proto"; -import "opentelemetry/proto/resource/v1/resource.proto"; - -option csharp_namespace = "OpenTelemetry.Proto.Trace.V1"; -option java_multiple_files = true; -option java_package = "io.opentelemetry.proto.trace.v1"; -option java_outer_classname = "TraceProto"; -option go_package = "go.opentelemetry.io/proto/otlp/trace/v1"; - -// TracesData represents the traces data that can be stored in a persistent storage, -// OR can be embedded by other protocols that transfer OTLP traces data but do -// not implement the OTLP protocol. -// -// The main difference between this message and collector protocol is that -// in this message there will not be any "control" or "metadata" specific to -// OTLP protocol. -// -// When new fields are added into this message, the OTLP request MUST be updated -// as well. -message TracesData { - // An array of ResourceSpans. - // For data coming from a single resource this array will typically contain - // one element. Intermediary nodes that receive data from multiple origins - // typically batch the data before forwarding further and in that case this - // array will contain multiple elements. - repeated ResourceSpans resource_spans = 1; -} - -// A collection of ScopeSpans from a Resource. -message ResourceSpans { - reserved 1000; - - // The resource for the spans in this message. - // If this field is not set then no resource info is known. - opentelemetry.proto.resource.v1.Resource resource = 1; - - // A list of ScopeSpans that originate from a resource. - repeated ScopeSpans scope_spans = 2; - - // The Schema URL, if known. This is the identifier of the Schema that the resource data - // is recorded in. To learn more about Schema URL see - // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url - // This schema_url applies to the data in the "resource" field. It does not apply - // to the data in the "scope_spans" field which have their own schema_url field. - string schema_url = 3; -} - -// A collection of Spans produced by an InstrumentationScope. -message ScopeSpans { - // The instrumentation scope information for the spans in this message. - // Semantically when InstrumentationScope isn't set, it is equivalent with - // an empty instrumentation scope name (unknown). - opentelemetry.proto.common.v1.InstrumentationScope scope = 1; - - // A list of Spans that originate from an instrumentation scope. - repeated Span spans = 2; - - // The Schema URL, if known. This is the identifier of the Schema that the span data - // is recorded in. To learn more about Schema URL see - // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url - // This schema_url applies to all spans and span events in the "spans" field. - string schema_url = 3; -} - -// A Span represents a single operation performed by a single component of the system. -// -// The next available field id is 17. -message Span { - // A unique identifier for a trace. All spans from the same trace share - // the same `trace_id`. The ID is a 16-byte array. An ID with all zeroes OR - // of length other than 16 bytes is considered invalid (empty string in OTLP/JSON - // is zero-length and thus is also invalid). - // - // This field is required. - bytes trace_id = 1; - - // A unique identifier for a span within a trace, assigned when the span - // is created. The ID is an 8-byte array. An ID with all zeroes OR of length - // other than 8 bytes is considered invalid (empty string in OTLP/JSON - // is zero-length and thus is also invalid). - // - // This field is required. - bytes span_id = 2; - - // trace_state conveys information about request position in multiple distributed tracing graphs. - // It is a trace_state in w3c-trace-context format: https://www.w3.org/TR/trace-context/#tracestate-header - // See also https://github.com/w3c/distributed-tracing for more details about this field. - string trace_state = 3; - - // The `span_id` of this span's parent span. If this is a root span, then this - // field must be empty. The ID is an 8-byte array. - bytes parent_span_id = 4; - - // Flags, a bit field. - // - // Bits 0-7 (8 least significant bits) are the trace flags as defined in W3C Trace - // Context specification. To read the 8-bit W3C trace flag, use - // `flags & SPAN_FLAGS_TRACE_FLAGS_MASK`. - // - // See https://www.w3.org/TR/trace-context-2/#trace-flags for the flag definitions. - // - // Bits 8 and 9 represent the 3 states of whether a span's parent - // is remote. The states are (unknown, is not remote, is remote). - // To read whether the value is known, use `(flags & SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK) != 0`. - // To read whether the span is remote, use `(flags & SPAN_FLAGS_CONTEXT_IS_REMOTE_MASK) != 0`. - // - // When creating span messages, if the message is logically forwarded from another source - // with an equivalent flags fields (i.e., usually another OTLP span message), the field SHOULD - // be copied as-is. If creating from a source that does not have an equivalent flags field - // (such as a runtime representation of an OpenTelemetry span), the high 22 bits MUST - // be set to zero. - // Readers MUST NOT assume that bits 10-31 (22 most significant bits) will be zero. - // - // [Optional]. - fixed32 flags = 16; - - // A description of the span's operation. - // - // For example, the name can be a qualified method name or a file name - // and a line number where the operation is called. A best practice is to use - // the same display name at the same call point in an application. - // This makes it easier to correlate spans in different traces. - // - // This field is semantically required to be set to non-empty string. - // Empty value is equivalent to an unknown span name. - // - // This field is required. - string name = 5; - - // SpanKind is the type of span. Can be used to specify additional relationships between spans - // in addition to a parent/child relationship. - enum SpanKind { - // Unspecified. Do NOT use as default. - // Implementations MAY assume SpanKind to be INTERNAL when receiving UNSPECIFIED. - SPAN_KIND_UNSPECIFIED = 0; - - // Indicates that the span represents an internal operation within an application, - // as opposed to an operation happening at the boundaries. Default value. - SPAN_KIND_INTERNAL = 1; - - // Indicates that the span covers server-side handling of an RPC or other - // remote network request. - SPAN_KIND_SERVER = 2; - - // Indicates that the span describes a request to some remote service. - SPAN_KIND_CLIENT = 3; - - // Indicates that the span describes a producer sending a message to a broker. - // Unlike CLIENT and SERVER, there is often no direct critical path latency relationship - // between producer and consumer spans. A PRODUCER span ends when the message was accepted - // by the broker while the logical processing of the message might span a much longer time. - SPAN_KIND_PRODUCER = 4; - - // Indicates that the span describes consumer receiving a message from a broker. - // Like the PRODUCER kind, there is often no direct critical path latency relationship - // between producer and consumer spans. - SPAN_KIND_CONSUMER = 5; - } - - // Distinguishes between spans generated in a particular context. For example, - // two spans with the same name may be distinguished using `CLIENT` (caller) - // and `SERVER` (callee) to identify queueing latency associated with the span. - SpanKind kind = 6; - - // start_time_unix_nano is the start time of the span. On the client side, this is the time - // kept by the local machine where the span execution starts. On the server side, this - // is the time when the server's application handler starts running. - // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January 1970. - // - // This field is semantically required and it is expected that end_time >= start_time. - fixed64 start_time_unix_nano = 7; - - // end_time_unix_nano is the end time of the span. On the client side, this is the time - // kept by the local machine where the span execution ends. On the server side, this - // is the time when the server application handler stops running. - // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January 1970. - // - // This field is semantically required and it is expected that end_time >= start_time. - fixed64 end_time_unix_nano = 8; - - // attributes is a collection of key/value pairs. Note, global attributes - // like server name can be set using the resource API. Examples of attributes: - // - // "/http/user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36" - // "/http/server_latency": 300 - // "example.com/myattribute": true - // "example.com/score": 10.239 - // - // The OpenTelemetry API specification further restricts the allowed value types: - // https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/common/README.md#attribute - // Attribute keys MUST be unique (it is not allowed to have more than one - // attribute with the same key). - repeated opentelemetry.proto.common.v1.KeyValue attributes = 9; - - // dropped_attributes_count is the number of attributes that were discarded. Attributes - // can be discarded because their keys are too long or because there are too many - // attributes. If this value is 0, then no attributes were dropped. - uint32 dropped_attributes_count = 10; - - // Event is a time-stamped annotation of the span, consisting of user-supplied - // text description and key-value pairs. - message Event { - // time_unix_nano is the time the event occurred. - fixed64 time_unix_nano = 1; - - // name of the event. - // This field is semantically required to be set to non-empty string. - string name = 2; - - // attributes is a collection of attribute key/value pairs on the event. - // Attribute keys MUST be unique (it is not allowed to have more than one - // attribute with the same key). - repeated opentelemetry.proto.common.v1.KeyValue attributes = 3; - - // dropped_attributes_count is the number of dropped attributes. If the value is 0, - // then no attributes were dropped. - uint32 dropped_attributes_count = 4; - } - - // events is a collection of Event items. - repeated Event events = 11; - - // dropped_events_count is the number of dropped events. If the value is 0, then no - // events were dropped. - uint32 dropped_events_count = 12; - - // A pointer from the current span to another span in the same trace or in a - // different trace. For example, this can be used in batching operations, - // where a single batch handler processes multiple requests from different - // traces or when the handler receives a request from a different project. - message Link { - // A unique identifier of a trace that this linked span is part of. The ID is a - // 16-byte array. - bytes trace_id = 1; - - // A unique identifier for the linked span. The ID is an 8-byte array. - bytes span_id = 2; - - // The trace_state associated with the link. - string trace_state = 3; - - // attributes is a collection of attribute key/value pairs on the link. - // Attribute keys MUST be unique (it is not allowed to have more than one - // attribute with the same key). - repeated opentelemetry.proto.common.v1.KeyValue attributes = 4; - - // dropped_attributes_count is the number of dropped attributes. If the value is 0, - // then no attributes were dropped. - uint32 dropped_attributes_count = 5; - - // Flags, a bit field. - // - // Bits 0-7 (8 least significant bits) are the trace flags as defined in W3C Trace - // Context specification. To read the 8-bit W3C trace flag, use - // `flags & SPAN_FLAGS_TRACE_FLAGS_MASK`. - // - // See https://www.w3.org/TR/trace-context-2/#trace-flags for the flag definitions. - // - // Bits 8 and 9 represent the 3 states of whether the link is remote. - // The states are (unknown, is not remote, is remote). - // To read whether the value is known, use `(flags & SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK) != 0`. - // To read whether the link is remote, use `(flags & SPAN_FLAGS_CONTEXT_IS_REMOTE_MASK) != 0`. - // - // Readers MUST NOT assume that bits 10-31 (22 most significant bits) will be zero. - // When creating new spans, bits 10-31 (most-significant 22-bits) MUST be zero. - // - // [Optional]. - fixed32 flags = 6; - } - - // links is a collection of Links, which are references from this span to a span - // in the same or different trace. - repeated Link links = 13; - - // dropped_links_count is the number of dropped links after the maximum size was - // enforced. If this value is 0, then no links were dropped. - uint32 dropped_links_count = 14; - - // An optional final status for this span. Semantically when Status isn't set, it means - // span's status code is unset, i.e. assume STATUS_CODE_UNSET (code = 0). - Status status = 15; -} - -// The Status type defines a logical error model that is suitable for different -// programming environments, including REST APIs and RPC APIs. -message Status { - reserved 1; - - // A developer-facing human readable error message. - string message = 2; - - // For the semantics of status codes see - // https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/api.md#set-status - enum StatusCode { - // The default status. - STATUS_CODE_UNSET = 0; - // The Span has been validated by an Application developer or Operator to - // have completed successfully. - STATUS_CODE_OK = 1; - // The Span contains an error. - STATUS_CODE_ERROR = 2; - }; - - // The status code. - StatusCode code = 3; -} - -// SpanFlags represents constants used to interpret the -// Span.flags field, which is protobuf 'fixed32' type and is to -// be used as bit-fields. Each non-zero value defined in this enum is -// a bit-mask. To extract the bit-field, for example, use an -// expression like: -// -// (span.flags & SPAN_FLAGS_TRACE_FLAGS_MASK) -// -// See https://www.w3.org/TR/trace-context-2/#trace-flags for the flag definitions. -// -// Note that Span flags were introduced in version 1.1 of the -// OpenTelemetry protocol. Older Span producers do not set this -// field, consequently consumers should not rely on the absence of a -// particular flag bit to indicate the presence of a particular feature. -enum SpanFlags { - // The zero value for the enum. Should not be used for comparisons. - // Instead use bitwise "and" with the appropriate mask as shown above. - SPAN_FLAGS_DO_NOT_USE = 0; - - // Bits 0-7 are used for trace flags. - SPAN_FLAGS_TRACE_FLAGS_MASK = 0x000000FF; - - // Bits 8 and 9 are used to indicate that the parent span or link span is remote. - // Bit 8 (`HAS_IS_REMOTE`) indicates whether the value is known. - // Bit 9 (`IS_REMOTE`) indicates whether the span or link is remote. - SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK = 0x00000100; - SPAN_FLAGS_CONTEXT_IS_REMOTE_MASK = 0x00000200; - - // Bits 10-31 are reserved for future use. -} diff --git a/scripts/docker/base-ubuntu-22.04.dockerfile b/scripts/docker/base-ubuntu-22.04.dockerfile index ac53e11dc8a8..19c7ff495d21 100644 --- a/scripts/docker/base-ubuntu-22.04.dockerfile +++ b/scripts/docker/base-ubuntu-22.04.dockerfile @@ -21,9 +21,17 @@ RUN ( \ cd /userver_tmp \ && ./setup-base-ubuntu-22.04-env.sh \ && ./pip-install.sh \ - && mkdir /app && cd /app && git clone --depth 1 -b 1.50.0 https://github.com/googleapis/api-common-protos.git && rm -rf /app/api-common-protos/.git \ + && mkdir /app \ + && cd /app \ + && git clone --depth 1 -b 1.50.0 https://github.com/googleapis/api-common-protos.git \ + && rm -rf /app/api-common-protos/.git \ && rm -rf /userver_tmp \ ) +RUN ( \ + cd /app \ + && git clone --depth 1 -b v1.3.2 https://github.com/open-telemetry/opentelemetry-proto \ + && rm -rf /app/opentelemetry-proto/.git \ +) # add expose ports EXPOSE 8080-8100 From 382f0434b8256aa7d509304d233584bdcc99f97f Mon Sep 17 00:00:00 2001 From: antonyzhilin Date: Mon, 15 Jul 2024 18:48:51 +0300 Subject: [PATCH 030/629] bug core: revert "grpc header propagation" 764356d037212dc94f7e08169a570acbe0b76f1b --- .mapping.json | 3 +- .../userver/server/http/http_response.hpp | 5 -- .../server/request/task_inherited_request.hpp | 34 +++++++------- core/src/server/http/http_request_handler.cpp | 26 +--------- core/src/server/http/http_response.cpp | 4 -- .../server/request/task_inherited_request.cpp | 47 ++++++++++++------- .../request/task_inherited_request_impl.hpp | 18 +++++++ .../basic_server/grpc_service.cpp | 45 +----------------- .../basic_server/proto/samples/greeter.proto | 1 - .../basic_server/static_config.yaml | 10 ---- .../tests-unix-socket/conftest.py | 13 +---- .../test_header_propagation.py | 39 --------------- .../ugrpc/server/impl/request_container.hpp | 37 --------------- .../ugrpc/server/impl/service_worker_impl.hpp | 5 -- 14 files changed, 72 insertions(+), 215 deletions(-) create mode 100644 core/src/server/request/task_inherited_request_impl.hpp delete mode 100644 grpc/functional_tests/basic_server/tests-unix-socket/test_header_propagation.py delete mode 100644 grpc/include/userver/ugrpc/server/impl/request_container.hpp diff --git a/.mapping.json b/.mapping.json index 443efa11e794..313b618306b8 100644 --- a/.mapping.json +++ b/.mapping.json @@ -1450,6 +1450,7 @@ "core/src/server/request/response_base.cpp":"taxi/uservices/userver/core/src/server/request/response_base.cpp", "core/src/server/request/task_inherited_data.cpp":"taxi/uservices/userver/core/src/server/request/task_inherited_data.cpp", "core/src/server/request/task_inherited_request.cpp":"taxi/uservices/userver/core/src/server/request/task_inherited_request.cpp", + "core/src/server/request/task_inherited_request_impl.hpp":"taxi/uservices/userver/core/src/server/request/task_inherited_request_impl.hpp", "core/src/server/requests_view.cpp":"taxi/uservices/userver/core/src/server/requests_view.cpp", "core/src/server/requests_view.hpp":"taxi/uservices/userver/core/src/server/requests_view.hpp", "core/src/server/server.cpp":"taxi/uservices/userver/core/src/server/server.cpp", @@ -1717,7 +1718,6 @@ "grpc/functional_tests/basic_server/static_config.yaml":"taxi/uservices/userver/grpc/functional_tests/basic_server/static_config.yaml", "grpc/functional_tests/basic_server/tests-unix-socket/conftest.py":"taxi/uservices/userver/grpc/functional_tests/basic_server/tests-unix-socket/conftest.py", "grpc/functional_tests/basic_server/tests-unix-socket/test_grpc.py":"taxi/uservices/userver/grpc/functional_tests/basic_server/tests-unix-socket/test_grpc.py", - "grpc/functional_tests/basic_server/tests-unix-socket/test_header_propagation.py":"taxi/uservices/userver/grpc/functional_tests/basic_server/tests-unix-socket/test_header_propagation.py", "grpc/functional_tests/metrics/CMakeLists.txt":"taxi/uservices/userver/grpc/functional_tests/metrics/CMakeLists.txt", "grpc/functional_tests/metrics/config_vars.yaml":"taxi/uservices/userver/grpc/functional_tests/metrics/config_vars.yaml", "grpc/functional_tests/metrics/proto/samples/greeter.proto":"taxi/uservices/userver/grpc/functional_tests/metrics/proto/samples/greeter.proto", @@ -1796,7 +1796,6 @@ "grpc/include/userver/ugrpc/server/impl/codegen_definitions.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/server/impl/codegen_definitions.hpp", "grpc/include/userver/ugrpc/server/impl/error_code.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/server/impl/error_code.hpp", "grpc/include/userver/ugrpc/server/impl/queue_holder.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/server/impl/queue_holder.hpp", - "grpc/include/userver/ugrpc/server/impl/request_container.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/server/impl/request_container.hpp", "grpc/include/userver/ugrpc/server/impl/service_worker.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/server/impl/service_worker.hpp", "grpc/include/userver/ugrpc/server/impl/service_worker_impl.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/server/impl/service_worker_impl.hpp", "grpc/include/userver/ugrpc/server/middlewares/baggage/component.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/server/middlewares/baggage/component.hpp", diff --git a/core/include/userver/server/http/http_response.hpp b/core/include/userver/server/http/http_response.hpp index 0a6c1dd1f260..fa4f7614237a 100644 --- a/core/include/userver/server/http/http_response.hpp +++ b/core/include/userver/server/http/http_response.hpp @@ -65,11 +65,6 @@ class HttpResponse final : public request::ResponseBase { /// were already sent for stream'ed response and the new header was not set. bool SetHeader(std::string_view name, std::string value); - /// @brief Add a new response header or rewrite an existing one. - /// @returns true if the header was set. Returns false if headers - /// were already sent for stream'ed response and the new header was not set. - bool SetHeader(std::string_view name, std::string_view value); - /// @overload bool SetHeader( const USERVER_NAMESPACE::http::headers::PredefinedHeader& header, diff --git a/core/include/userver/server/request/task_inherited_request.hpp b/core/include/userver/server/request/task_inherited_request.hpp index 7bc493c6e926..ff2cd0cf4224 100644 --- a/core/include/userver/server/request/task_inherited_request.hpp +++ b/core/include/userver/server/request/task_inherited_request.hpp @@ -1,38 +1,38 @@ #pragma once /// @file userver/server/request/task_inherited_request.hpp -/// @brief Functions that provide access to incoming request stored in +/// @brief Functions that provide access to HttpRequest stored in /// TaskInheritedVariable. -#include +#include #include USERVER_NAMESPACE_BEGIN -namespace server::request { - -/// @brief Class for representing incoming request that is handled by the -/// current task hierarchy. -class RequestContainer { - public: - virtual std::string_view GetHeader(std::string_view header_name) const = 0; - virtual bool HasHeader(std::string_view header_name) const = 0; - virtual ~RequestContainer() = default; -}; +namespace http::headers { +class PredefinedHeader; +} -/// @brief Set a request that is handled by the current task hierarchy. -void SetTaskInheritedRequest(const RequestContainer& request); +namespace server::request { -/// @brief Get a header from incoming request that is handled by the +/// @brief Get a header from server::http::HttpRequest that is handled by the /// current task hierarchy. /// @return Header value or an empty string, if none such -std::string_view GetTaskInheritedHeader(std::string_view header_name); +const std::string& GetTaskInheritedHeader(std::string_view header_name); + +/// @overload +const std::string& GetTaskInheritedHeader( + const USERVER_NAMESPACE::http::headers::PredefinedHeader& header_name); -/// @brief Checks whether specified header exists in incoming request +/// @brief Checks whether specified header exists in server::http::HttpRequest /// that is handled by the current task hierarchy. /// @return `true` if the header exists, `false` otherwise bool HasTaskInheritedHeader(std::string_view header_name); +/// @overload +bool HasTaskInheritedHeader( + const USERVER_NAMESPACE::http::headers::PredefinedHeader& header_name); + } // namespace server::request USERVER_NAMESPACE_END diff --git a/core/src/server/http/http_request_handler.cpp b/core/src/server/http/http_request_handler.cpp index 1c6224c3e11a..a5786923bf86 100644 --- a/core/src/server/http/http_request_handler.cpp +++ b/core/src/server/http/http_request_handler.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -78,28 +79,6 @@ namespace { utils::statistics::MetricTag> kCcStatusCodeIsCustom{ "congestion-control.rps.is-custom-status-activated"}; -class HttpRequestContainer final : public server::request::RequestContainer { - public: - explicit HttpRequestContainer(std::shared_ptr request) - : request_(std::move(request)) { - UASSERT_MSG( - request_, - "Should not be called outside of the task that does HTTP handling"); - } - std::string_view GetHeader(std::string_view header_name) const override { - if (!request_) { - return {}; - } - return request_->GetHeader(header_name); - } - bool HasHeader(std::string_view header_name) const override { - return request_ && request_->HasHeader(header_name); - } - - private: - std::shared_ptr request_; -}; - } // namespace engine::TaskWithResult HttpRequestHandler::StartRequestTask( @@ -186,9 +165,8 @@ engine::TaskWithResult HttpRequestHandler::StartRequestTask( } auto payload = [request = std::move(request), handler] { - HttpRequestContainer request_container( + server::request::kTaskInheritedRequest.Set( std::static_pointer_cast(request)); - SetTaskInheritedRequest(request_container); request->SetTaskStartTime(); diff --git a/core/src/server/http/http_response.cpp b/core/src/server/http/http_response.cpp index 34c7a6f81452..ac36d543b692 100644 --- a/core/src/server/http/http_response.cpp +++ b/core/src/server/http/http_response.cpp @@ -160,10 +160,6 @@ bool HttpResponse::SetHeader(std::string_view name, std::string value) { return SetHeader(std::string{name}, std::move(value)); } -bool HttpResponse::SetHeader(std::string_view name, std::string_view value) { - return SetHeader(std::string{name}, std::string{value}); -} - bool HttpResponse::SetHeader( const USERVER_NAMESPACE::http::headers::PredefinedHeader& header, std::string value) { diff --git a/core/src/server/request/task_inherited_request.cpp b/core/src/server/request/task_inherited_request.cpp index 64d91ef5a101..14e701f33173 100644 --- a/core/src/server/request/task_inherited_request.cpp +++ b/core/src/server/request/task_inherited_request.cpp @@ -1,38 +1,53 @@ #include -#include - -#include +#include +#include USERVER_NAMESPACE_BEGIN namespace server::request { namespace { -inline engine::TaskInheritedVariable< - std::reference_wrapper> - kTaskInheritedRequest; -} // namespace +const std::string kEmptyHeader{}; -void SetTaskInheritedRequest(const RequestContainer& request) { - kTaskInheritedRequest.Set(request); -} - -std::string_view GetTaskInheritedHeader(std::string_view header_name) { +template +const std::string& DoGetTaskInheritedHeader(const Header& header_name) { const auto* request = kTaskInheritedRequest.GetOptional(); if (request == nullptr) { - return {}; + return kEmptyHeader; } - return (*request).get().GetHeader(header_name); + return (*request)->GetHeader(header_name); } -bool HasTaskInheritedHeader(std::string_view header_name) { +template +bool DoHasTaskInheritedHeader(const Header& header_name) { const auto* request = kTaskInheritedRequest.GetOptional(); if (request == nullptr) { return false; } - return (*request).get().HasHeader(header_name); + return (*request)->HasHeader(header_name); +} + +} // namespace + +const std::string& GetTaskInheritedHeader(std::string_view header_name) { + return DoGetTaskInheritedHeader(header_name); } + +const std::string& GetTaskInheritedHeader( + const USERVER_NAMESPACE::http::headers::PredefinedHeader& header_name) { + return DoGetTaskInheritedHeader(header_name); +} + +bool HasTaskInheritedHeader(std::string_view header_name) { + return DoHasTaskInheritedHeader(header_name); +} + +bool HasTaskInheritedHeader( + const USERVER_NAMESPACE::http::headers::PredefinedHeader& header_name) { + return DoHasTaskInheritedHeader(header_name); +} + } // namespace server::request USERVER_NAMESPACE_END diff --git a/core/src/server/request/task_inherited_request_impl.hpp b/core/src/server/request/task_inherited_request_impl.hpp new file mode 100644 index 000000000000..194d9b33e983 --- /dev/null +++ b/core/src/server/request/task_inherited_request_impl.hpp @@ -0,0 +1,18 @@ +#pragma once + +#include + +USERVER_NAMESPACE_BEGIN + +namespace server::http { +class HttpRequestImpl; +} // namespace server::http + +namespace server::request { + +inline engine::TaskInheritedVariable> + kTaskInheritedRequest; + +} // namespace server::request + +USERVER_NAMESPACE_END diff --git a/grpc/functional_tests/basic_server/grpc_service.cpp b/grpc/functional_tests/basic_server/grpc_service.cpp index 238ac63d871b..8a1aa7c21c88 100644 --- a/grpc/functional_tests/basic_server/grpc_service.cpp +++ b/grpc/functional_tests/basic_server/grpc_service.cpp @@ -4,14 +4,10 @@ #include -#include -#include #include -#include #include #include #include -#include #include #include @@ -29,29 +25,9 @@ class GreeterServiceComponent final GreeterServiceComponent(const components::ComponentConfig& config, const components::ComponentContext& context) - : api::GreeterServiceBase::Component(config, context), - echo_url_{config["echo-url"].As()}, - http_client_( - context.FindComponent().GetHttpClient()) {} + : api::GreeterServiceBase::Component(config, context) {} void SayHello(SayHelloCall& call, api::GreetingRequest&& request) override; - void CallEchoNobody(SayHelloCall& call, - api::GreetingRequest&& request) override; - static yaml_config::Schema GetStaticConfigSchema() { - return yaml_config::MergeSchemas(R"( - type: object - description: HTTP echo without body component - additionalProperties: false - properties: - echo-url: - type: string - description: some other microservice listens on this URL - )"); - } - - private: - const std::string echo_url_; - clients::http::Client& http_client_; }; void GreeterServiceComponent::SayHello( @@ -62,20 +38,6 @@ void GreeterServiceComponent::SayHello( call.Finish(response); } -void GreeterServiceComponent::CallEchoNobody( - api::GreeterServiceBase::SayHelloCall& call, - samples::api::GreetingRequest&&) { - api::GreetingResponse response; - response.set_greeting("Call Echo Nobody"); - auto http_response = http_client_.CreateRequest() - .get(echo_url_) - .retry(1) - .timeout(std::chrono::seconds{5}) - .perform(); - http_response->raise_for_status(); - call.Finish(response); -} - } // namespace samples int main(int argc, char* argv[]) { @@ -84,9 +46,6 @@ int main(int argc, char* argv[]) { .Append() .Append() .Append() - .Append() - .Append() - .Append() - .Append(); + .Append(); return utils::DaemonMain(argc, argv, component_list); } diff --git a/grpc/functional_tests/basic_server/proto/samples/greeter.proto b/grpc/functional_tests/basic_server/proto/samples/greeter.proto index ab4702ff61f1..b239fa823b85 100644 --- a/grpc/functional_tests/basic_server/proto/samples/greeter.proto +++ b/grpc/functional_tests/basic_server/proto/samples/greeter.proto @@ -4,7 +4,6 @@ package samples.api; service GreeterService { rpc SayHello(GreetingRequest) returns(GreetingResponse) {} - rpc CallEchoNobody(GreetingRequest) returns(GreetingResponse) {} } message GreetingRequest { diff --git a/grpc/functional_tests/basic_server/static_config.yaml b/grpc/functional_tests/basic_server/static_config.yaml index b0e8203fbb56..6a10a570482f 100644 --- a/grpc/functional_tests/basic_server/static_config.yaml +++ b/grpc/functional_tests/basic_server/static_config.yaml @@ -27,19 +27,9 @@ components_manager: # Our GreeterService implementation greeter-service: - echo-url: 'mockserver/v1/translations' # Some other microservice listens on this URL task-processor: main-task-processor middlewares: [] - http-client: - fs-task-processor: main-task-processor - - dns-client: - fs-task-processor: fs-task-processor - - headers-propagator: - headers: ["custom-header-1", "custom-header-2"] - server: listener: port: 8092 diff --git a/grpc/functional_tests/basic_server/tests-unix-socket/conftest.py b/grpc/functional_tests/basic_server/tests-unix-socket/conftest.py index cc4bfa037c88..38040af039ac 100644 --- a/grpc/functional_tests/basic_server/tests-unix-socket/conftest.py +++ b/grpc/functional_tests/basic_server/tests-unix-socket/conftest.py @@ -5,18 +5,7 @@ pytest_plugins = ['pytest_userver.plugins.grpc'] -USERVER_CONFIG_HOOKS = ['prepare_service_config', 'config_echo_url'] - - -@pytest.fixture(scope='session') -def config_echo_url(mockserver_info): - def _do_patch(config_yaml, config_vars): - components = config_yaml['components_manager']['components'] - components['greeter-service']['echo-url'] = mockserver_info.url( - '/test-service/echo-no-body', - ) - - return _do_patch +USERVER_CONFIG_HOOKS = ['prepare_service_config'] @pytest.fixture(scope='session') diff --git a/grpc/functional_tests/basic_server/tests-unix-socket/test_header_propagation.py b/grpc/functional_tests/basic_server/tests-unix-socket/test_header_propagation.py deleted file mode 100644 index 98c3520826b9..000000000000 --- a/grpc/functional_tests/basic_server/tests-unix-socket/test_header_propagation.py +++ /dev/null @@ -1,39 +0,0 @@ -from samples import greeter_pb2 - - -HEADERS_TO_PROPAGATE = ['custom-header-1', 'custom-header-2'] -HEADERS_NOT_TO_PROPAGATE = [ - 'custom-header-1-non-prop', - 'custom-header-2-non-prop', -] - - -async def test_propagation_headers_mixed_from_grpc(grpc_client, mockserver): - headers = { - HEADERS_NOT_TO_PROPAGATE[0]: '1', - HEADERS_NOT_TO_PROPAGATE[1]: '2', - HEADERS_TO_PROPAGATE[0]: '3', - HEADERS_TO_PROPAGATE[1]: '4', - } - - @mockserver.json_handler('/test-service/echo-no-body') - async def nobody_handler(request): - assert ( - request.headers[HEADERS_TO_PROPAGATE[0]] - == headers[HEADERS_TO_PROPAGATE[0]] - ) - assert ( - request.headers[HEADERS_TO_PROPAGATE[1]] - == headers[HEADERS_TO_PROPAGATE[1]] - ) - assert HEADERS_NOT_TO_PROPAGATE[0] not in request.headers - assert HEADERS_NOT_TO_PROPAGATE[1] not in request.headers - return mockserver.make_response() - - metadata = [] - for key, value in headers.items(): - metadata.append((key, value)) - request = greeter_pb2.GreetingRequest(name='Python') - response = await grpc_client.CallEchoNobody(request, metadata=metadata) - assert response.greeting == 'Call Echo Nobody' - assert nobody_handler.times_called == 1 diff --git a/grpc/include/userver/ugrpc/server/impl/request_container.hpp b/grpc/include/userver/ugrpc/server/impl/request_container.hpp deleted file mode 100644 index e162cbfe01e6..000000000000 --- a/grpc/include/userver/ugrpc/server/impl/request_container.hpp +++ /dev/null @@ -1,37 +0,0 @@ -#pragma once - -#include - -#include - -#include - -USERVER_NAMESPACE_BEGIN - -namespace ugrpc::server::impl { - -class GrpcRequestContainer final - : public USERVER_NAMESPACE::server::request::RequestContainer { - public: - GrpcRequestContainer(const grpc::ServerContext& server_context) - : server_context_(server_context){}; - std::string_view GetHeader(std::string_view header_name) const override { - const auto& header_value = utils::FindOrDefault( - server_context_.client_metadata(), - grpc::string_ref{header_name.data(), header_name.size()}); - return {header_value.data(), header_value.size()}; - }; - bool HasHeader(std::string_view header_name) const override { - return nullptr != - utils::FindOrNullptr( - server_context_.client_metadata(), - grpc::string_ref{header_name.data(), header_name.size()}); - }; - - private: - const grpc::ServerContext& server_context_; -}; - -} // namespace ugrpc::server::impl - -USERVER_NAMESPACE_END diff --git a/grpc/include/userver/ugrpc/server/impl/service_worker_impl.hpp b/grpc/include/userver/ugrpc/server/impl/service_worker_impl.hpp index 1db6ffb895e7..01a54338f185 100644 --- a/grpc/include/userver/ugrpc/server/impl/service_worker_impl.hpp +++ b/grpc/include/userver/ugrpc/server/impl/service_worker_impl.hpp @@ -17,7 +17,6 @@ #include #include #include -#include #include #include #include @@ -34,7 +33,6 @@ #include #include #include -#include #include #include #include @@ -199,9 +197,6 @@ class CallData final { if constexpr (!std::is_same_v) { initial_request = &initial_request_; } - GrpcRequestContainer request_container(context_); - USERVER_NAMESPACE::server::request::SetTaskInheritedRequest( - request_container); MiddlewareCallContext middleware_context( middlewares, responder, do_call, From 63463fd36afb50779fff587b9710c85cd303431f Mon Sep 17 00:00:00 2001 From: segoon Date: Mon, 15 Jul 2024 20:42:10 +0300 Subject: [PATCH 031/629] feat otlp: add metrics 906c8c8a351357295dd163da8c5aa3815f35aab8 --- .mapping.json | 4 ++-- .../userver/logging/impl}/log_stats.hpp | 4 ++-- .../{statistics => impl}/log_stats.cpp | 9 ++++---- core/src/logging/tp_logger.cpp | 2 +- core/src/logging/tp_logger.hpp | 6 ++--- otlp/include/userver/otlp/logs/component.hpp | 2 ++ otlp/src/otlp/logs/component.cpp | 11 +++++++++ otlp/src/otlp/logs/logger.cpp | 23 ++++++++++++------- otlp/src/otlp/logs/logger.hpp | 4 ++++ 9 files changed, 45 insertions(+), 20 deletions(-) rename core/{src/logging/statistics => include/userver/logging/impl}/log_stats.hpp (88%) rename core/src/logging/{statistics => impl}/log_stats.cpp (72%) diff --git a/.mapping.json b/.mapping.json index 313b618306b8..7efe7b0f8e63 100644 --- a/.mapping.json +++ b/.mapping.json @@ -637,6 +637,7 @@ "core/include/userver/fs/temp_file.hpp":"taxi/uservices/userver/core/include/userver/fs/temp_file.hpp", "core/include/userver/fs/write.hpp":"taxi/uservices/userver/core/include/userver/fs/write.hpp", "core/include/userver/logging/component.hpp":"taxi/uservices/userver/core/include/userver/logging/component.hpp", + "core/include/userver/logging/impl/log_stats.hpp":"taxi/uservices/userver/core/include/userver/logging/impl/log_stats.hpp", "core/include/userver/logging/level_serialization.hpp":"taxi/uservices/userver/core/include/userver/logging/level_serialization.hpp", "core/include/userver/logging/logger.hpp":"taxi/uservices/userver/core/include/userver/logging/logger.hpp", "core/include/userver/moodycamel/concurrentqueue_fwd.h":"taxi/uservices/userver/core/include/userver/moodycamel/concurrentqueue_fwd.h", @@ -1253,6 +1254,7 @@ "core/src/logging/impl/file_sink.hpp":"taxi/uservices/userver/core/src/logging/impl/file_sink.hpp", "core/src/logging/impl/file_sinks_benchmark.cpp":"taxi/uservices/userver/core/src/logging/impl/file_sinks_benchmark.cpp", "core/src/logging/impl/file_sinks_test.cpp":"taxi/uservices/userver/core/src/logging/impl/file_sinks_test.cpp", + "core/src/logging/impl/log_stats.cpp":"taxi/uservices/userver/core/src/logging/impl/log_stats.cpp", "core/src/logging/impl/null_sink.hpp":"taxi/uservices/userver/core/src/logging/impl/null_sink.hpp", "core/src/logging/impl/open_file_helper.hpp":"taxi/uservices/userver/core/src/logging/impl/open_file_helper.hpp", "core/src/logging/impl/open_file_helper_test.cpp":"taxi/uservices/userver/core/src/logging/impl/open_file_helper_test.cpp", @@ -1279,8 +1281,6 @@ "core/src/logging/split_location.hpp":"taxi/uservices/userver/core/src/logging/split_location.hpp", "core/src/logging/split_location_test.cpp":"taxi/uservices/userver/core/src/logging/split_location_test.cpp", "core/src/logging/stacktrace_cache_test.cpp":"taxi/uservices/userver/core/src/logging/stacktrace_cache_test.cpp", - "core/src/logging/statistics/log_stats.cpp":"taxi/uservices/userver/core/src/logging/statistics/log_stats.cpp", - "core/src/logging/statistics/log_stats.hpp":"taxi/uservices/userver/core/src/logging/statistics/log_stats.hpp", "core/src/logging/tp_logger.cpp":"taxi/uservices/userver/core/src/logging/tp_logger.cpp", "core/src/logging/tp_logger.hpp":"taxi/uservices/userver/core/src/logging/tp_logger.hpp", "core/src/logging/tp_logger_benchmark.cpp":"taxi/uservices/userver/core/src/logging/tp_logger_benchmark.cpp", diff --git a/core/src/logging/statistics/log_stats.hpp b/core/include/userver/logging/impl/log_stats.hpp similarity index 88% rename from core/src/logging/statistics/log_stats.hpp rename to core/include/userver/logging/impl/log_stats.hpp index 005bd6d1b9a0..4084087ca0e0 100644 --- a/core/src/logging/statistics/log_stats.hpp +++ b/core/include/userver/logging/impl/log_stats.hpp @@ -9,7 +9,7 @@ USERVER_NAMESPACE_BEGIN -namespace logging::statistics { +namespace logging::impl { using Counter = utils::statistics::RateCounter; @@ -22,6 +22,6 @@ struct LogStatistics final { void DumpMetric(utils::statistics::Writer& writer, const LogStatistics& stats); -} // namespace logging::statistics +} // namespace logging::impl USERVER_NAMESPACE_END diff --git a/core/src/logging/statistics/log_stats.cpp b/core/src/logging/impl/log_stats.cpp similarity index 72% rename from core/src/logging/statistics/log_stats.cpp rename to core/src/logging/impl/log_stats.cpp index 93bd27572299..8c751619799b 100644 --- a/core/src/logging/statistics/log_stats.cpp +++ b/core/src/logging/impl/log_stats.cpp @@ -1,12 +1,13 @@ -#include "log_stats.hpp" +#include #include USERVER_NAMESPACE_BEGIN -namespace logging::statistics { +namespace logging::impl { -void DumpMetric(utils::statistics::Writer& writer, const LogStatistics& stats) { +void DumpMetric(utils::statistics::Writer& writer, + const impl::LogStatistics& stats) { writer["dropped"].ValueWithLabels(stats.dropped, {"version", "2"}); utils::statistics::Rate total; @@ -22,6 +23,6 @@ void DumpMetric(utils::statistics::Writer& writer, const LogStatistics& stats) { writer["has_reopening_error"] = stats.has_reopening_error.load(); } -} // namespace logging::statistics +} // namespace logging::impl USERVER_NAMESPACE_END diff --git a/core/src/logging/tp_logger.cpp b/core/src/logging/tp_logger.cpp index 05005b4af0b6..62fd7739b65c 100644 --- a/core/src/logging/tp_logger.cpp +++ b/core/src/logging/tp_logger.cpp @@ -124,7 +124,7 @@ void TpLogger::Flush() { } } -statistics::LogStatistics& TpLogger::GetStatistics() noexcept { return stats_; } +impl::LogStatistics& TpLogger::GetStatistics() noexcept { return stats_; } void TpLogger::Log(Level level, std::string_view msg) { ++stats_.by_level[static_cast(level)]; diff --git a/core/src/logging/tp_logger.hpp b/core/src/logging/tp_logger.hpp index bd5350f43897..d527ecda1712 100644 --- a/core/src/logging/tp_logger.hpp +++ b/core/src/logging/tp_logger.hpp @@ -21,8 +21,8 @@ #include #include #include -#include #include +#include USERVER_NAMESPACE_BEGIN @@ -83,7 +83,7 @@ class TpLogger final : public LoggerBase { std::string_view GetLoggerName() const noexcept; - statistics::LogStatistics& GetStatistics() noexcept; + impl::LogStatistics& GetStatistics() noexcept; protected: bool DoShouldLog(Level level) const noexcept override; @@ -116,7 +116,7 @@ class TpLogger final : public LoggerBase { const std::string logger_name_; std::vector sinks_; - mutable statistics::LogStatistics stats_{}; + mutable impl::LogStatistics stats_{}; engine::Mutex capacity_waiters_mutex_; engine::ConditionVariable capacity_waiters_cv_; diff --git a/otlp/include/userver/otlp/logs/component.hpp b/otlp/include/userver/otlp/logs/component.hpp index ab917003a04c..e409c18f6e73 100644 --- a/otlp/include/userver/otlp/logs/component.hpp +++ b/otlp/include/userver/otlp/logs/component.hpp @@ -8,6 +8,7 @@ #include #include +#include USERVER_NAMESPACE_BEGIN @@ -44,6 +45,7 @@ class LoggerComponent final : public components::RawComponentBase { private: std::shared_ptr logger_; + utils::statistics::Entry statistics_holder_; }; } // namespace otlp diff --git a/otlp/src/otlp/logs/component.cpp b/otlp/src/otlp/logs/component.cpp index 5092fc1a8eef..2dbd23ab0aad 100644 --- a/otlp/src/otlp/logs/component.cpp +++ b/otlp/src/otlp/logs/component.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -49,6 +50,16 @@ LoggerComponent::LoggerComponent(const components::ComponentConfig& config, context.FindComponent(); logging::impl::SetDefaultLoggerRef(*logger_); + + auto* const statistics_storage = + context.FindComponentOptional(); + if (statistics_storage) { + statistics_holder_ = statistics_storage->GetStorage().RegisterWriter( + "logger", [this](utils::statistics::Writer& writer) { + writer.ValueWithLabels(logger_->GetStatistics(), + {"logger", "default"}); + }); + } } LoggerComponent::~LoggerComponent() { diff --git a/otlp/src/otlp/logs/logger.cpp b/otlp/src/otlp/logs/logger.cpp index 4295120875ab..9e205c70df52 100644 --- a/otlp/src/otlp/logs/logger.cpp +++ b/otlp/src/otlp/logs/logger.cpp @@ -50,7 +50,11 @@ Logger::~Logger() { Stop(); } void Logger::Stop() noexcept { sender_task_.SyncCancel(); } -void Logger::Log(logging::Level, std::string_view msg) { +const logging::impl::LogStatistics& Logger::GetStatistics() const { + return stats_; +} + +void Logger::Log(logging::Level level, std::string_view msg) { // Trim trailing \n if (msg.size() > 1 && msg[msg.size() - 1] == '\n') { msg = msg.substr(0, msg.size() - 1); @@ -59,6 +63,8 @@ void Logger::Log(logging::Level, std::string_view msg) { std::vector key_values = utils::text::SplitIntoStringViewVector(msg, "\t"); + ++stats_.by_level[static_cast(level)]; + if (IsTracingEntry(key_values)) { HandleTracing(key_values); } else { @@ -136,10 +142,10 @@ void Logger::HandleLog(const std::vector& key_values) { log_records.set_time_unix_nano(nanoseconds.count()); // Drop a log if overflown - [[maybe_unused]] auto ok = - queue_producer_.PushNoblock(std::move(log_records)); - - // TODO: count drops + auto ok = queue_producer_.PushNoblock(std::move(log_records)); + if (!ok) { + ++stats_.dropped; + } } void Logger::HandleTracing(const std::vector& key_values) { @@ -208,9 +214,10 @@ void Logger::HandleTracing(const std::vector& key_values) { 1'000'000'000); // Drop a trace if overflown - [[maybe_unused]] auto ok = queue_producer_.PushNoblock(std::move(span)); - - // TODO: count drops + auto ok = queue_producer_.PushNoblock(std::move(span)); + if (!ok) { + ++stats_.dropped; + } } void Logger::SendingLoop(Queue::Consumer& consumer, LogClient& log_client, diff --git a/otlp/src/otlp/logs/logger.hpp b/otlp/src/otlp/logs/logger.hpp index 9f460aec20a5..cc4c8c2bd732 100644 --- a/otlp/src/otlp/logs/logger.hpp +++ b/otlp/src/otlp/logs/logger.hpp @@ -8,6 +8,7 @@ #include #include +#include #include USERVER_NAMESPACE_BEGIN @@ -41,6 +42,8 @@ class Logger final : public logging::impl::LoggerBase { void Stop() noexcept; + const logging::impl::LogStatistics& GetStatistics() const; + protected: bool DoShouldLog(logging::Level level) const noexcept override; @@ -69,6 +72,7 @@ class Logger final : public logging::impl::LoggerBase { ExportTraceServiceRequest& request, TraceClient& trace_client); + logging::impl::LogStatistics stats_; const LoggerConfig config_; std::shared_ptr queue_; Queue::MultiProducer queue_producer_; From e0968ce1a5f021259bcf9fa03e7a0679dbf77be1 Mon Sep 17 00:00:00 2001 From: segoon Date: Mon, 15 Jul 2024 21:59:27 +0300 Subject: [PATCH 032/629] feat otlp: allow attributes renaming 8c129b80db70a222ed78f4fc63cf97735839447f --- otlp/src/otlp/logs/component.cpp | 19 ++++++++++++++++--- otlp/src/otlp/logs/logger.cpp | 13 ++++++++++--- otlp/src/otlp/logs/logger.hpp | 5 ++++- samples/otlp_service/static_config.yaml | 3 +++ 4 files changed, 33 insertions(+), 7 deletions(-) diff --git a/otlp/src/otlp/logs/component.cpp b/otlp/src/otlp/logs/component.cpp index 2dbd23ab0aad..4390122d8d5c 100644 --- a/otlp/src/otlp/logs/component.cpp +++ b/otlp/src/otlp/logs/component.cpp @@ -43,6 +43,12 @@ LoggerComponent::LoggerComponent(const components::ComponentConfig& config, config["service-name"].As("unknown_service"); logger_config.log_level = config["log-level"].As(); + logger_config.extra_attributes = + config["extra-attributes"] + .As>({}); + logger_config.attributes_mapping = + config["attributes-mapping"] + .As>({}); logger_ = std::make_shared(std::move(client), std::move(trace_client), std::move(logger_config)); @@ -95,13 +101,20 @@ additionalProperties: false service-name: type: string description: service name - attributes: + attributes-mapping: + type: object + description: rename rules for OTLP attributes + properties: {} + additionalProperties: + type: string + description: new attribute name + extra-attributes: type: object description: extra OTLP attributes properties: {} additionalProperties: - type: string - description: attribute value + type: string + description: attribute value )"); } diff --git a/otlp/src/otlp/logs/logger.cpp b/otlp/src/otlp/logs/logger.cpp index 9e205c70df52..2ae76e5717f4 100644 --- a/otlp/src/otlp/logs/logger.cpp +++ b/otlp/src/otlp/logs/logger.cpp @@ -128,7 +128,7 @@ void Logger::HandleLog(const std::vector& key_values) { } auto attributes = log_records.add_attributes(); - attributes->set_key(std::string{key}); + attributes->set_key(std::string{MapAttribute(key)}); attributes->mutable_value()->set_string_value(std::string{value}); } @@ -199,7 +199,7 @@ void Logger::HandleTracing(const std::vector& key_values) { } auto attributes = span.add_attributes(); - attributes->set_key(std::string{key}); + attributes->set_key(std::string{MapAttribute(key)}); attributes->mutable_value()->set_string_value(std::string{value}); } @@ -286,7 +286,7 @@ void Logger::FillAttributes( attr->mutable_value()->set_string_value(std::string{config_.service_name}); } - for (const auto& [key, value] : config_.attributes) { + for (const auto& [key, value] : config_.extra_attributes) { auto* attr = resource.add_attributes(); attr->set_key(std::string{key}); attr->mutable_value()->set_string_value(std::string{value}); @@ -327,6 +327,13 @@ void Logger::DoTrace( // TODO: count exceptions } +std::string_view Logger::MapAttribute(std::string_view attr) const { + for (const auto& [key, value] : config_.attributes_mapping) { + if (key == attr) return value; + } + return attr; +} + } // namespace otlp USERVER_NAMESPACE_END diff --git a/otlp/src/otlp/logs/logger.hpp b/otlp/src/otlp/logs/logger.hpp index cc4c8c2bd732..6bcd020ad872 100644 --- a/otlp/src/otlp/logs/logger.hpp +++ b/otlp/src/otlp/logs/logger.hpp @@ -20,7 +20,8 @@ struct LoggerConfig { std::chrono::milliseconds max_batch_delay{}; std::string service_name; - std::unordered_map attributes; + std::unordered_map extra_attributes; + std::unordered_map attributes_mapping; logging::Level log_level{logging::Level::kInfo}; }; @@ -72,6 +73,8 @@ class Logger final : public logging::impl::LoggerBase { ExportTraceServiceRequest& request, TraceClient& trace_client); + std::string_view MapAttribute(std::string_view attr) const; + logging::impl::LogStatistics stats_; const LoggerConfig config_; std::shared_ptr queue_; diff --git a/samples/otlp_service/static_config.yaml b/samples/otlp_service/static_config.yaml index 7a81a9dacf83..979e1e6d37a9 100644 --- a/samples/otlp_service/static_config.yaml +++ b/samples/otlp_service/static_config.yaml @@ -20,6 +20,9 @@ components_manager: endpoint: '0.0.0.0:4317' service-name: otlp-example log-level: debug + # How to rename attributes, if you really need to + attributes-mapping: + module: line # /// [otlp logger] handler-server-monitor: From f3ba41ffe01dce32b7f9eb7269be86198e31b532 Mon Sep 17 00:00:00 2001 From: segoon Date: Tue, 16 Jul 2024 09:57:41 +0300 Subject: [PATCH 033/629] feat logging: do not drop initial logs 95373a9c17f6e65548075737b804835ac0ec200d --- .mapping.json | 1 + core/src/components/run.cpp | 10 +-- otlp/src/otlp/logs/component.cpp | 2 + .../userver/logging/impl/logger_base.hpp | 2 + .../userver/logging/impl/mem_logger.hpp | 64 +++++++++++++++++++ universal/src/logging/impl/logger_base.cpp | 2 + universal/src/logging/log.cpp | 5 +- 7 files changed, 80 insertions(+), 6 deletions(-) create mode 100644 universal/include/userver/logging/impl/mem_logger.hpp diff --git a/.mapping.json b/.mapping.json index 7efe7b0f8e63..1953bc925176 100644 --- a/.mapping.json +++ b/.mapping.json @@ -3664,6 +3664,7 @@ "universal/include/userver/logging/format.hpp":"taxi/uservices/userver/universal/include/userver/logging/format.hpp", "universal/include/userver/logging/fwd.hpp":"taxi/uservices/userver/universal/include/userver/logging/fwd.hpp", "universal/include/userver/logging/impl/logger_base.hpp":"taxi/uservices/userver/universal/include/userver/logging/impl/logger_base.hpp", + "universal/include/userver/logging/impl/mem_logger.hpp":"taxi/uservices/userver/universal/include/userver/logging/impl/mem_logger.hpp", "universal/include/userver/logging/impl/tag_writer.hpp":"taxi/uservices/userver/universal/include/userver/logging/impl/tag_writer.hpp", "universal/include/userver/logging/level.hpp":"taxi/uservices/userver/universal/include/userver/logging/level.hpp", "universal/include/userver/logging/log.hpp":"taxi/uservices/userver/universal/include/userver/logging/log.hpp", diff --git a/core/src/components/run.cpp b/core/src/components/run.cpp index 0212d4ab8392..c8a320d0c15a 100644 --- a/core/src/components/run.cpp +++ b/core/src/components/run.cpp @@ -17,7 +17,6 @@ #include #include #include -#include #include #include #include @@ -60,6 +59,8 @@ class LogScope final { logger_new_ = std::move(logger); } + logging::LoggerRef GetPrevLogger() { return logger_prev_; } + private: logging::LoggerPtr logger_new_; logging::LoggerRef logger_prev_; @@ -172,9 +173,10 @@ ManagerConfig ParseManagerConfigAndSetupLogging( if (default_logger_config) { auto default_logger = logging::impl::MakeTpLogger(*default_logger_config); - // This line enables basic logging. Any LOG_XXX before it is meaningless, - // because it would typically go straight to a NullLogger. - log_scope.SetLogger(std::move(default_logger)); + // This line enables basic logging. Any logging before would go to + // MemLogger and be transferred to the logger in the cycle below. + log_scope.SetLogger(default_logger); + log_scope.GetPrevLogger().ForwardTo(*default_logger); } LOG_INFO() << "Parsed " << details; diff --git a/otlp/src/otlp/logs/component.cpp b/otlp/src/otlp/logs/component.cpp index 4390122d8d5c..d0c839a4d632 100644 --- a/otlp/src/otlp/logs/component.cpp +++ b/otlp/src/otlp/logs/component.cpp @@ -55,7 +55,9 @@ LoggerComponent::LoggerComponent(const components::ComponentConfig& config, // We must init after the default logger is initialized context.FindComponent(); + auto& old_logger = logging::GetDefaultLogger(); logging::impl::SetDefaultLoggerRef(*logger_); + old_logger.ForwardTo(*logger_); auto* const statistics_storage = context.FindComponentOptional(); diff --git a/universal/include/userver/logging/impl/logger_base.hpp b/universal/include/userver/logging/impl/logger_base.hpp index a3a39c04c862..32abb7c61eb0 100644 --- a/universal/include/userver/logging/impl/logger_base.hpp +++ b/universal/include/userver/logging/impl/logger_base.hpp @@ -39,6 +39,8 @@ class LoggerBase { void SetFlushOn(Level level); bool ShouldFlush(Level level) const; + virtual void ForwardTo(LoggerBase& logger_to); + protected: virtual bool DoShouldLog(Level level) const noexcept; diff --git a/universal/include/userver/logging/impl/mem_logger.hpp b/universal/include/userver/logging/impl/mem_logger.hpp new file mode 100644 index 000000000000..feb96a8ebc47 --- /dev/null +++ b/universal/include/userver/logging/impl/mem_logger.hpp @@ -0,0 +1,64 @@ +#pragma once + +#include +#include +#include + +#include + +USERVER_NAMESPACE_BEGIN + +namespace logging::impl { + +// kMaxLogItems is a hardcoded const and not config value as it is used before +// the main logger initialization and before yaml config is parsed. +inline constexpr auto kMaxLogItems = 10000; + +class MemLogger final : public LoggerBase { + public: + MemLogger() noexcept : LoggerBase(Format::kTskv) { SetLevel(Level::kDebug); } + MemLogger(const MemLogger&) = delete; + MemLogger(MemLogger&&) = delete; + + static MemLogger& GetMemLogger() noexcept { + static MemLogger logger; + return logger; + } + + struct Data { + Level level; + std::string msg; + }; + + void Log(Level level, std::string_view msg) override { + std::unique_lock lock(mutex_); + if (forward_logger_) { + forward_logger_->Log(level, msg); + return; + } + + if (data_.size() > kMaxLogItems) return; + data_.push_back({level, std::string{msg}}); + } + void Flush() override {} + + void ForwardTo(LoggerBase& logger_to) override { + std::unique_lock lock(mutex_); + for (const auto& log : data_) { + logger_to.Log(log.level, log.msg); + } + forward_logger_ = &logger_to; + } + + protected: + bool DoShouldLog(Level) const noexcept override { return true; } + + private: + std::mutex mutex_; + std::vector data_; + LoggerBase* forward_logger_{nullptr}; +}; + +} // namespace logging::impl + +USERVER_NAMESPACE_END diff --git a/universal/src/logging/impl/logger_base.cpp b/universal/src/logging/impl/logger_base.cpp index 25f9ea009134..11c48fd84834 100644 --- a/universal/src/logging/impl/logger_base.cpp +++ b/universal/src/logging/impl/logger_base.cpp @@ -30,6 +30,8 @@ bool LoggerBase::ShouldFlush(Level level) const { return flush_level_ <= level; } +void LoggerBase::ForwardTo(LoggerBase&) {} + bool LoggerBase::DoShouldLog(Level /*level*/) const noexcept { return true; } bool ShouldLogNoSpan(const LoggerBase& logger, Level level) noexcept { diff --git a/universal/src/logging/log.cpp b/universal/src/logging/log.cpp index 4b3f12bc4bb9..85fb62746aa6 100644 --- a/universal/src/logging/log.cpp +++ b/universal/src/logging/log.cpp @@ -5,7 +5,7 @@ #include #include #include -#include +#include #include USERVER_NAMESPACE_BEGIN @@ -14,7 +14,8 @@ namespace logging { namespace { auto& NonOwningDefaultLoggerInternal() noexcept { - static std::atomic default_logger_ptr{&GetNullLogger()}; + static std::atomic default_logger_ptr{ + &impl::MemLogger::GetMemLogger()}; return default_logger_ptr; } From 6d8d9adfa55a6d9906c6f1e5590034bc29f5004a Mon Sep 17 00:00:00 2001 From: i9kin Date: Tue, 16 Jul 2024 10:05:04 +0300 Subject: [PATCH 034/629] fix server: testsuite does not log exceptions b232ffc8395a6ccca503e3b23c7fab214894ddc3 --- .mapping.json | 2 ++ .../httpserver_with_exception_handler.hpp | 24 +++++++++++++++++++ core/functional_tests/basic_chaos/service.cpp | 2 ++ .../basic_chaos/static_config.yaml | 5 ++++ .../dynamic_configs/test_cache_updates.py | 9 ++++--- .../middlewares/test_testsuite_middleware.py | 23 ++++++++++++++++++ .../include/userver/testsuite/middlewares.hpp | 4 +++- core/src/testsuite/middlewares.cpp | 10 ++++++-- 8 files changed, 73 insertions(+), 6 deletions(-) create mode 100644 core/functional_tests/basic_chaos/httpserver_with_exception_handler.hpp create mode 100644 core/functional_tests/basic_chaos/tests-nonchaos/middlewares/test_testsuite_middleware.py diff --git a/.mapping.json b/.mapping.json index 1953bc925176..2ac443f12638 100644 --- a/.mapping.json +++ b/.mapping.json @@ -379,6 +379,7 @@ "core/functional_tests/basic_chaos/CMakeLists.txt":"taxi/uservices/userver/core/functional_tests/basic_chaos/CMakeLists.txt", "core/functional_tests/basic_chaos/httpclient_handlers.hpp":"taxi/uservices/userver/core/functional_tests/basic_chaos/httpclient_handlers.hpp", "core/functional_tests/basic_chaos/httpserver_handlers.hpp":"taxi/uservices/userver/core/functional_tests/basic_chaos/httpserver_handlers.hpp", + "core/functional_tests/basic_chaos/httpserver_with_exception_handler.hpp":"taxi/uservices/userver/core/functional_tests/basic_chaos/httpserver_with_exception_handler.hpp", "core/functional_tests/basic_chaos/resolver_handlers.hpp":"taxi/uservices/userver/core/functional_tests/basic_chaos/resolver_handlers.hpp", "core/functional_tests/basic_chaos/service.cpp":"taxi/uservices/userver/core/functional_tests/basic_chaos/service.cpp", "core/functional_tests/basic_chaos/static_config.yaml":"taxi/uservices/userver/core/functional_tests/basic_chaos/static_config.yaml", @@ -394,6 +395,7 @@ "core/functional_tests/basic_chaos/tests-nonchaos/dynamic_configs/test_fixtures.py":"taxi/uservices/userver/core/functional_tests/basic_chaos/tests-nonchaos/dynamic_configs/test_fixtures.py", "core/functional_tests/basic_chaos/tests-nonchaos/dynamic_configs/test_invalid_update.py":"taxi/uservices/userver/core/functional_tests/basic_chaos/tests-nonchaos/dynamic_configs/test_invalid_update.py", "core/functional_tests/basic_chaos/tests-nonchaos/metrics/test_has_reopening_error_metric.py":"taxi/uservices/userver/core/functional_tests/basic_chaos/tests-nonchaos/metrics/test_has_reopening_error_metric.py", + "core/functional_tests/basic_chaos/tests-nonchaos/middlewares/test_testsuite_middleware.py":"taxi/uservices/userver/core/functional_tests/basic_chaos/tests-nonchaos/middlewares/test_testsuite_middleware.py", "core/functional_tests/basic_chaos/tests-nonchaos/network/test_interfaces.py":"taxi/uservices/userver/core/functional_tests/basic_chaos/tests-nonchaos/network/test_interfaces.py", "core/functional_tests/basic_chaos/tests/conftest.py":"taxi/uservices/userver/core/functional_tests/basic_chaos/tests/conftest.py", "core/functional_tests/basic_chaos/tests/httpclient/conftest.py":"taxi/uservices/userver/core/functional_tests/basic_chaos/tests/httpclient/conftest.py", diff --git a/core/functional_tests/basic_chaos/httpserver_with_exception_handler.hpp b/core/functional_tests/basic_chaos/httpserver_with_exception_handler.hpp new file mode 100644 index 000000000000..5e8562e76810 --- /dev/null +++ b/core/functional_tests/basic_chaos/httpserver_with_exception_handler.hpp @@ -0,0 +1,24 @@ +#pragma once + +#include + +namespace chaos { + +class HttpServerWithExceptionHandler final + : public server::handlers::HttpHandlerBase { + public: + static constexpr std::string_view kName = + "handler-chaos-httpserver-with-exception"; + + HttpServerWithExceptionHandler(const components::ComponentConfig& config, + const components::ComponentContext& context) + : HttpHandlerBase(config, context) {} + + std::string HandleRequestThrow( + const server::http::HttpRequest&, + server::request::RequestContext&) const override { + throw std::runtime_error("some runtime error"); + } +}; + +} // namespace chaos diff --git a/core/functional_tests/basic_chaos/service.cpp b/core/functional_tests/basic_chaos/service.cpp index 422920256015..d104bbe9299e 100644 --- a/core/functional_tests/basic_chaos/service.cpp +++ b/core/functional_tests/basic_chaos/service.cpp @@ -15,6 +15,7 @@ #include "httpclient_handlers.hpp" #include "httpserver_handlers.hpp" +#include "httpserver_with_exception_handler.hpp" #include "resolver_handlers.hpp" int main(int argc, char* argv[]) { @@ -24,6 +25,7 @@ int main(int argc, char* argv[]) { .Append() .Append() .Append() + .Append() .Append() .Append() .Append() diff --git a/core/functional_tests/basic_chaos/static_config.yaml b/core/functional_tests/basic_chaos/static_config.yaml index fa645c87f752..a7c74452f5f8 100644 --- a/core/functional_tests/basic_chaos/static_config.yaml +++ b/core/functional_tests/basic_chaos/static_config.yaml @@ -17,6 +17,11 @@ components_manager: task_processor: main-task-processor method: GET,DELETE,POST + handler-chaos-httpserver-with-exception: + path: /chaos/httpserver-with-exception + task_processor: main-task-processor + method: GET + handler-chaos-dns-resolver: path: /chaos/resolver task_processor: main-task-processor diff --git a/core/functional_tests/basic_chaos/tests-nonchaos/dynamic_configs/test_cache_updates.py b/core/functional_tests/basic_chaos/tests-nonchaos/dynamic_configs/test_cache_updates.py index 035399324583..7e69d8a062f0 100644 --- a/core/functional_tests/basic_chaos/tests-nonchaos/dynamic_configs/test_cache_updates.py +++ b/core/functional_tests/basic_chaos/tests-nonchaos/dynamic_configs/test_cache_updates.py @@ -5,10 +5,13 @@ def tp_reset(data): await service_client.update_server_state() - assert tp_reset.times_called == 1 - assert tp_reset.next_call() == {'data': {'update_type': 'incremental'}} + assert tp_reset.times_called <= 1 + if tp_reset.times_called == 1: + assert tp_reset.next_call() == {'data': {'update_type': 'incremental'}} - await service_client.invalidate_caches() + await service_client.invalidate_caches( + cache_names=['dynamic-config-client-updater'], + ) assert tp_reset.times_called == 1 assert tp_reset.next_call() == {'data': {'update_type': 'full'}} diff --git a/core/functional_tests/basic_chaos/tests-nonchaos/middlewares/test_testsuite_middleware.py b/core/functional_tests/basic_chaos/tests-nonchaos/middlewares/test_testsuite_middleware.py new file mode 100644 index 000000000000..b1a2a7e92a19 --- /dev/null +++ b/core/functional_tests/basic_chaos/tests-nonchaos/middlewares/test_testsuite_middleware.py @@ -0,0 +1,23 @@ +import json + + +async def test_testsuite_middleware(service_client): + testsuite_response = { + 'code': '500', + 'message': 'Internal Server Error', + 'details': 'some runtime error', + } + + async with service_client.capture_logs() as capture: + response = await service_client.get('/chaos/httpserver-with-exception') + assert response.status_code == 500 + assert response.json() == testsuite_response + + logs = capture.select(level='ERROR') + assert len(logs) == 2 + assert ( + logs[0]['text'] + == 'exception in \'handler-chaos-httpserver-with-exception\' handler: some runtime error (std::runtime_error)' + ) + + assert json.loads(logs[1]['body']) == testsuite_response diff --git a/core/include/userver/testsuite/middlewares.hpp b/core/include/userver/testsuite/middlewares.hpp index 68aba54becb7..8d996470cb88 100644 --- a/core/include/userver/testsuite/middlewares.hpp +++ b/core/include/userver/testsuite/middlewares.hpp @@ -13,11 +13,13 @@ class ExceptionsHandlingMiddleware final "testsuite-exceptions-handling-middleware"}; explicit ExceptionsHandlingMiddleware( - const server::handlers::HttpHandlerBase&) {} + const server::handlers::HttpHandlerBase& handler); private: void HandleRequest(server::http::HttpRequest& request, server::request::RequestContext& context) const override; + + const server::handlers::HttpHandlerBase& handler_; }; using ExceptionsHandlingMiddlewareFactory = diff --git a/core/src/testsuite/middlewares.cpp b/core/src/testsuite/middlewares.cpp index da0c0d4ca787..06df3b992df4 100644 --- a/core/src/testsuite/middlewares.cpp +++ b/core/src/testsuite/middlewares.cpp @@ -3,14 +3,18 @@ #include #include #include -#include #include +#include #include USERVER_NAMESPACE_BEGIN namespace testsuite { +ExceptionsHandlingMiddleware::ExceptionsHandlingMiddleware( + const server::handlers::HttpHandlerBase& handler) + : handler_(handler) {} + void ExceptionsHandlingMiddleware::HandleRequest( server::http::HttpRequest& request, server::request::RequestContext& context) const { @@ -28,9 +32,11 @@ void ExceptionsHandlingMiddleware::HandleRequest( response.SetStatus(server::http::HttpStatus::kInternalServerError); response.SetContentType(http::content_type::kApplicationJson); response.SetData(formats::json::ToString(builder.ExtractValue())); + + // Log exception like ExceptionsHandling middleware + handler_.LogUnknownException(exc); } } - } // namespace testsuite USERVER_NAMESPACE_END From 8e08e53421a21a8a94f6fe9d272a405e6d9ef8a6 Mon Sep 17 00:00:00 2001 From: arudenko02 Date: Tue, 16 Jul 2024 13:59:31 +0300 Subject: [PATCH 035/629] fix postgresql: check connection state in Cleanup check connection state in cleaing aebb693646f7487715b8c06263ea9997a7c56167 --- postgresql/src/storages/postgres/detail/connection_impl.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgresql/src/storages/postgres/detail/connection_impl.cpp b/postgresql/src/storages/postgres/detail/connection_impl.cpp index 9f33d123a8fb..13de7da43d49 100644 --- a/postgresql/src/storages/postgres/detail/connection_impl.cpp +++ b/postgresql/src/storages/postgres/detail/connection_impl.cpp @@ -590,7 +590,7 @@ bool ConnectionImpl::Cleanup(TimeoutDuration timeout) { if (state > ConnectionState::kIdle) { Rollback(); } - return true; + return GetConnectionState() == ConnectionState::kIdle; } void ConnectionImpl::SetParameter(std::string_view name, std::string_view value, From 273bd0cd3a9a2a5e577433f3f4c451e55e01fbcf Mon Sep 17 00:00:00 2001 From: segoon Date: Tue, 16 Jul 2024 15:55:35 +0300 Subject: [PATCH 036/629] feat chaotic: better output 8d82b9ed0d371c5c14e190307fee6ed2d6d050e1 --- chaotic/chaotic/main.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/chaotic/chaotic/main.py b/chaotic/chaotic/main.py index 3538b284aefc..dc8a5e96e5ae 100644 --- a/chaotic/chaotic/main.py +++ b/chaotic/chaotic/main.py @@ -2,6 +2,7 @@ import dataclasses import os import re +import sys from typing import Any from typing import Callable from typing import Dict @@ -284,6 +285,10 @@ def main() -> None: os.path.join(args.output_dir, filename_rel + file.ext), file.content, ) + print( + f'Handled {len(types)} schema types in {len(outputs)} file groups.', + file=sys.stderr, + ) if __name__ == '__main__': From ab311669335ce8ac9fa133fb4675679d9be540d8 Mon Sep 17 00:00:00 2001 From: nepritula Date: Tue, 16 Jul 2024 16:18:47 +0300 Subject: [PATCH 037/629] feat docs: new bytea tests added MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tests: протестировано локально и в CI 6c38df4a938b9e49bd3afce078b08cb32652e636 --- .../storages/postgres/tests/bytea_pgtest.cpp | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/postgresql/src/storages/postgres/tests/bytea_pgtest.cpp b/postgresql/src/storages/postgres/tests/bytea_pgtest.cpp index 33744cbbfd8c..a397c2aa8ca1 100644 --- a/postgresql/src/storages/postgres/tests/bytea_pgtest.cpp +++ b/postgresql/src/storages/postgres/tests/bytea_pgtest.cpp @@ -119,6 +119,48 @@ UTEST_P(PostgreConnection, ByteaStoredMoved) { EXPECT_EQ(kFooBar, tgt_str); } +UTEST_P(PostgreConnection, ByteaRoundTripTypes) { + CheckConnection(GetConn()); + + { + /// [bytea_simple] + pg::ResultSet res = GetConn()->Execute("select 'foobar'::bytea"); + std::string s = "foobar"s; + + // reading a binary string + std::string received; + res[0][0].To(pg::Bytea(received)); + EXPECT_EQ(received, s); + /// [bytea_simple] + } + + { + /// [bytea_string] + // sending a "binary string" + std::string s = "\0\xff\x0afoobar"s; + pg::ResultSet res = GetConn()->Execute("select $1", pg::Bytea(s)); + + // reading a binary string + std::string received; + res[0][0].To(pg::Bytea(received)); + EXPECT_EQ(received, s); + /// [bytea_string] + } + + { + /// [bytea_vector] + // storing a byte array: + std::vector bin_str{1, 2, 3, 4, 5, 6, 7, 8, 9}; + auto res = GetConn()->Execute("select $1", pg::Bytea(bin_str)); + + // reading a binary string + std::vector received; + res[0][0].To(pg::Bytea(received)); + EXPECT_EQ(received, bin_str); + /// [bytea_vector] + } +} + } // namespace USERVER_NAMESPACE_END From 8c64d696e67cddd038566551a95d066594b455c9 Mon Sep 17 00:00:00 2001 From: antoshkka Date: Tue, 16 Jul 2024 18:14:58 +0300 Subject: [PATCH 038/629] feat core: allow using HTTPS with certificates without password protection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tests: протестировано локально и в CI 5cc5f823b72e5ff0939a1707bab4342d727aba3c --- .mapping.json | 9 +++ core/functional_tests/CMakeLists.txt | 3 + .../https_no_passphrase/CMakeLists.txt | 6 ++ .../https_no_passphrase/Readme.md | 9 +++ .../https_no_passphrase/cert.crt | 31 ++++++++ .../httpserver_handlers.hpp | 42 ++++++++++ .../https_no_passphrase/private_key.key | 52 +++++++++++++ .../https_no_passphrase/service.cpp | 33 ++++++++ .../https_no_passphrase/static_config.yaml | 76 +++++++++++++++++++ .../tests-deadline/conftest.py | 31 ++++++++ .../tests-deadline/test_server.py | 21 +++++ core/src/server/pph_config.hpp | 13 ++-- core/src/server/server.cpp | 13 +++- 13 files changed, 330 insertions(+), 9 deletions(-) create mode 100644 core/functional_tests/https_no_passphrase/CMakeLists.txt create mode 100644 core/functional_tests/https_no_passphrase/Readme.md create mode 100644 core/functional_tests/https_no_passphrase/cert.crt create mode 100644 core/functional_tests/https_no_passphrase/httpserver_handlers.hpp create mode 100644 core/functional_tests/https_no_passphrase/private_key.key create mode 100644 core/functional_tests/https_no_passphrase/service.cpp create mode 100644 core/functional_tests/https_no_passphrase/static_config.yaml create mode 100644 core/functional_tests/https_no_passphrase/tests-deadline/conftest.py create mode 100644 core/functional_tests/https_no_passphrase/tests-deadline/test_server.py diff --git a/.mapping.json b/.mapping.json index 2ac443f12638..5ff1fed21ef3 100644 --- a/.mapping.json +++ b/.mapping.json @@ -421,6 +421,15 @@ "core/functional_tests/https/static_config.yaml":"taxi/uservices/userver/core/functional_tests/https/static_config.yaml", "core/functional_tests/https/tests-deadline/conftest.py":"taxi/uservices/userver/core/functional_tests/https/tests-deadline/conftest.py", "core/functional_tests/https/tests-deadline/test_server.py":"taxi/uservices/userver/core/functional_tests/https/tests-deadline/test_server.py", + "core/functional_tests/https_no_passphrase/CMakeLists.txt":"taxi/uservices/userver/core/functional_tests/https_no_passphrase/CMakeLists.txt", + "core/functional_tests/https_no_passphrase/Readme.md":"taxi/uservices/userver/core/functional_tests/https_no_passphrase/Readme.md", + "core/functional_tests/https_no_passphrase/cert.crt":"taxi/uservices/userver/core/functional_tests/https_no_passphrase/cert.crt", + "core/functional_tests/https_no_passphrase/httpserver_handlers.hpp":"taxi/uservices/userver/core/functional_tests/https_no_passphrase/httpserver_handlers.hpp", + "core/functional_tests/https_no_passphrase/private_key.key":"taxi/uservices/userver/core/functional_tests/https_no_passphrase/private_key.key", + "core/functional_tests/https_no_passphrase/service.cpp":"taxi/uservices/userver/core/functional_tests/https_no_passphrase/service.cpp", + "core/functional_tests/https_no_passphrase/static_config.yaml":"taxi/uservices/userver/core/functional_tests/https_no_passphrase/static_config.yaml", + "core/functional_tests/https_no_passphrase/tests-deadline/conftest.py":"taxi/uservices/userver/core/functional_tests/https_no_passphrase/tests-deadline/conftest.py", + "core/functional_tests/https_no_passphrase/tests-deadline/test_server.py":"taxi/uservices/userver/core/functional_tests/https_no_passphrase/tests-deadline/test_server.py", "core/functional_tests/metrics/CMakeLists.txt":"taxi/uservices/userver/core/functional_tests/metrics/CMakeLists.txt", "core/functional_tests/metrics/config_vars.yaml":"taxi/uservices/userver/core/functional_tests/metrics/config_vars.yaml", "core/functional_tests/metrics/secure_data.json":"taxi/uservices/userver/core/functional_tests/metrics/secure_data.json", diff --git a/core/functional_tests/CMakeLists.txt b/core/functional_tests/CMakeLists.txt index 8f3bb179ee27..415b47be8368 100644 --- a/core/functional_tests/CMakeLists.txt +++ b/core/functional_tests/CMakeLists.txt @@ -8,6 +8,9 @@ add_dependencies(${PROJECT_NAME} ${PROJECT_NAME}-basic-chaos) add_subdirectory(https) add_dependencies(${PROJECT_NAME} ${PROJECT_NAME}-https) +add_subdirectory(https_no_passphrase) +add_dependencies(${PROJECT_NAME} ${PROJECT_NAME}-https-no-passphrase) + add_subdirectory(metrics) add_dependencies(${PROJECT_NAME} ${PROJECT_NAME}-metrics) diff --git a/core/functional_tests/https_no_passphrase/CMakeLists.txt b/core/functional_tests/https_no_passphrase/CMakeLists.txt new file mode 100644 index 000000000000..1ef2a81fd06d --- /dev/null +++ b/core/functional_tests/https_no_passphrase/CMakeLists.txt @@ -0,0 +1,6 @@ +project(userver-core-tests-https-no-passphrase CXX) + +add_executable(${PROJECT_NAME} "service.cpp") +target_link_libraries(${PROJECT_NAME} userver-core) + +userver_chaos_testsuite_add(TESTS_DIRECTORY tests-deadline) diff --git a/core/functional_tests/https_no_passphrase/Readme.md b/core/functional_tests/https_no_passphrase/Readme.md new file mode 100644 index 000000000000..3bb4cc550dc6 --- /dev/null +++ b/core/functional_tests/https_no_passphrase/Readme.md @@ -0,0 +1,9 @@ +Certificates and private keays were generated via: + +``` +openssl genrsa 4096 > private_key.key +openssl req -x509 -sha256 -nodes -new -key private_key.key -out cert.crt +``` + +Use more advanced techniques for production - properly sign with root +certificate and so forth... diff --git a/core/functional_tests/https_no_passphrase/cert.crt b/core/functional_tests/https_no_passphrase/cert.crt new file mode 100644 index 000000000000..1e68f8fcb635 --- /dev/null +++ b/core/functional_tests/https_no_passphrase/cert.crt @@ -0,0 +1,31 @@ +-----BEGIN CERTIFICATE----- +MIIFazCCA1OgAwIBAgIUbnAQ8B8pL8/3bAfuAiFwpkmil9MwDQYJKoZIhvcNAQEL +BQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yNDA3MTYxMDAwMzVaFw0yNDA4 +MTUxMDAwMzVaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw +HwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggIiMA0GCSqGSIb3DQEB +AQUAA4ICDwAwggIKAoICAQDNl2ZJUIkRMK3NUP14aGNHeIVsxUBdX8okK7rLjqU7 +k0zrIvFGSkUvL+ee1U9VOkpU+1Mj1/BCXKc03YUEB5CL9xyFEsskHOvJXp6EGOcJ +twhh+yYn1X8BhMV93qHWYvbJXC5geT5zazS/CklnzGk0Q6AJgWsRp3ks2a+2QFT+ +bQZOFZSd2Li8pphPItXAmgZoX1jLbW7dJtSM69HO1AmZ4+vLfpstdRw/YmNR21Jy +pWB4cUqJMgxXZMonGWYsm2fPMODI62mdPhmi773Bhn7/tF4cdHwie0lVcE/rB/X3 ++Z04Y3uTe3LOvQFM9R/na6PITZb7Z/V1LJihErLse/Tvcu9wxi8qIkD7Nc7MlgQS +zb2uWinkbm30178uE5UGYyfBoP7ZjByD6YkHvauWx31HEWUeHNefZyTeP+5HW38H +pVr/+baWGKQJQK1vVmg/x/RpWifEdyYzHz0Uk35K6vBLVn1FCyEud5Llhhf/05e2 +RH37qWD9N61F1+2JHXEnSJmm4BBP+Wsrm26XoApuFkyciMhJ8JFHhIhiBGAauSC8 +JDLQAHQBFTIEMdJw/meA8Ca4Rx754g2M1vxhW7RE6KyaIz9X8HcvKVtdhtHkee7H +hvIx392HJzQcZrHD1+eonnBxMjB8USwCg2XLBwt+0fT3tsmAscRRBB06NNUOe6ov +QQIDAQABo1MwUTAdBgNVHQ4EFgQUDdjzaOshB/BwfscEybLNf91DDAYwHwYDVR0j +BBgwFoAUDdjzaOshB/BwfscEybLNf91DDAYwDwYDVR0TAQH/BAUwAwEB/zANBgkq +hkiG9w0BAQsFAAOCAgEAiKZi80L7Ztp8iSNOkvEK5BfUOpqZDt7jcLg0NdNRD2aC +Yus6YfoKdz4fzd9JlgEiGUJLNQXg2lCHmTYgFykqgpgtq/ROkls/0Y2nsKg6F1hb +RnKPL9ZG6+s9nH1Cn6QB44PnGhCFCaRDxgKk6VxHxq9CQU5Gs++iLuYUwjCvOb8/ +jdrT++WGi53RU4wDHeEGf9V3iVEuYlJBkWiPgEMVgfxHv3cu8ZMhP0yBpArKwXmq ++7CqUrbXTG/YGmJNIph0aakSKlzHlUzRj+6+ua87cyL9aHb/C4BCxONBGSW9Amr9 +O2s9YCMf8FJ3TVwZ60f5KiBdIrvaaIyatmunPsMzxATZxPomGlohmd3YCKnOALlB +19kmawb8GPJacJfY+6NgJ8luD9nEKc3V4Av9dqmSIPm4IN3ISEYPbjaiIc6A/xg9 +vCIxF8YafARuvNVkMzZkXFoT5IYah+fNt/opnJxjl8K5/Q3X1Ni6iAPxqMzzXNM8 +ZYHbC+tCohJqVVK85mg0wCi+Cs+XZp+TpSrTDhj+DFi8ichmDOG9sOuXip6roiTD +mdoyCtkcCA2B7bFU5jivuGo7/IifGy/mZTolLJq5j0Fym7D7WTandbTWOqfTOOmJ +uKZoW+U7vnjTs5SdK7RsULSpJyncM5fq1teKg7hTWhocxSqROtceCReWPyhhIwU= +-----END CERTIFICATE----- diff --git a/core/functional_tests/https_no_passphrase/httpserver_handlers.hpp b/core/functional_tests/https_no_passphrase/httpserver_handlers.hpp new file mode 100644 index 000000000000..ac6a0b606519 --- /dev/null +++ b/core/functional_tests/https_no_passphrase/httpserver_handlers.hpp @@ -0,0 +1,42 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace https { + +class HttpServerHandler final : public server::handlers::HttpHandlerBase { + public: + static constexpr std::string_view kName = "handler-https-httpserver"; + + static inline const std::string kDefaultAnswer = "OK!"; + + HttpServerHandler(const components::ComponentConfig& config, + const components::ComponentContext& context) + : HttpHandlerBase(config, context) {} + + std::string HandleRequestThrow( + const server::http::HttpRequest& request, + server::request::RequestContext&) const override { + const auto& type = request.GetArg("type"); + + if (type == "cancel") { + engine::InterruptibleSleepFor(std::chrono::seconds(20)); + if (engine::current_task::IsCancelRequested()) { + engine::TaskCancellationBlocker block_cancel; + TESTPOINT("testpoint_cancel", {}); + } + return kDefaultAnswer; + } + + UINVARIANT(false, "Unexpected request type"); + } +}; + +} // namespace https diff --git a/core/functional_tests/https_no_passphrase/private_key.key b/core/functional_tests/https_no_passphrase/private_key.key new file mode 100644 index 000000000000..f614a768e66b --- /dev/null +++ b/core/functional_tests/https_no_passphrase/private_key.key @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQAIBADANBgkqhkiG9w0BAQEFAASCCSowggkmAgEAAoICAQDNl2ZJUIkRMK3N +UP14aGNHeIVsxUBdX8okK7rLjqU7k0zrIvFGSkUvL+ee1U9VOkpU+1Mj1/BCXKc0 +3YUEB5CL9xyFEsskHOvJXp6EGOcJtwhh+yYn1X8BhMV93qHWYvbJXC5geT5zazS/ +CklnzGk0Q6AJgWsRp3ks2a+2QFT+bQZOFZSd2Li8pphPItXAmgZoX1jLbW7dJtSM +69HO1AmZ4+vLfpstdRw/YmNR21JypWB4cUqJMgxXZMonGWYsm2fPMODI62mdPhmi +773Bhn7/tF4cdHwie0lVcE/rB/X3+Z04Y3uTe3LOvQFM9R/na6PITZb7Z/V1LJih +ErLse/Tvcu9wxi8qIkD7Nc7MlgQSzb2uWinkbm30178uE5UGYyfBoP7ZjByD6YkH +vauWx31HEWUeHNefZyTeP+5HW38HpVr/+baWGKQJQK1vVmg/x/RpWifEdyYzHz0U +k35K6vBLVn1FCyEud5Llhhf/05e2RH37qWD9N61F1+2JHXEnSJmm4BBP+Wsrm26X +oApuFkyciMhJ8JFHhIhiBGAauSC8JDLQAHQBFTIEMdJw/meA8Ca4Rx754g2M1vxh +W7RE6KyaIz9X8HcvKVtdhtHkee7HhvIx392HJzQcZrHD1+eonnBxMjB8USwCg2XL +Bwt+0fT3tsmAscRRBB06NNUOe6ovQQIDAQABAoIB/wxY62u7yn+dzNJn6tEwc8X3 +FwxRTGxPNHS3q6BOEN5jGluE3bi6/Sy4U9DWPs1/krdH4NZmQae6qNAj/bkgOy4s +ySm01z5z/jU52YtVc4rB018PFluOFvKgZrwIgu+9QdUWFDL3FSkZZzGSDJaPUEIz +TYlEnImS5VQsPsFTW8dsBNTrtuFEAtRRrRQTo4LNvVTHZS3ShLYDfwB3qUIiufDk +H2U5VViFkAy+e/iOk4UPKZwSMcfLmbIl+34dqGVFcBaheN6YRN+Muz1t6JeW3Em/ +K2fKFyp6/QyEkkgrUjpyY17fmqxg3aRM9acJkcD0VgHRpx+yPquq4f/Kf7R0PWGI +da8TY38V288pNZoxLd2iIIMXN0k+j1jpLtavwegiJ8ZXDS6vabSyClF1WjV9WXDN +mIqS3mFkJJMAjoLF/aInbZGcBPt6ET9i+m2bTYv9UQJcNgIsO+LLm2ee1oSaDrma +IilehLdJ6trA+PeG2+YNmXZ9MpJKtWXyf7PeQsL+pLoR3+J84Nf5rGZ0jb1gdBqY +qvWYCzepkB8N4c0OrDBf6mUE1oT1oT2kTDr4tF/MClPy6ymAkf3ym28aRfH2kXoS +ihplN7Sit5QBzN2UwisjfASJt5lomIWbzk6SGRw9k2MEyVRt2YZDOsfc0QkCfxcT +upLoX9DD2RETnJtNLGkCggEBANAnQwjwJvhRNukJtp8H6tpPjetgRIIMzONK4rE9 +CgKzABGMQApyc5070wkG+Zw9Uqg8WaZoENITmB5wj+DM+ZeExtvDNkKg8laWBc3u +5Gw+GwWIzGW0Hbasa9qxFKIXGpfKW9PEltfgukZwIRgWFoQJYSc+3OnC8sOv2OJ3 +lkGj9NksFDHYESydAgtoK+bxjsJW+X96Pn+Nbm/X4L/9olYM7utM1Ge2UYl6nDLK +gKMuPO/cqLssPXQrkHlIhP54y4s9cfNZ0CirRVJWLT2+kfKXsJmROyhlX16Ak2WB +byAF075NUUt929DowQhT3Go6i28ZbXSGWkABxe6yPF+sDSkCggEBAPzZYS6CfmSL +ibJxnM7qitArL4o10oj1e7FZPvzXeP2Zqr4MPiaU+8L/+F6NA7OlBJqAsP0jT/qp +yOtHYuTQvqKq9/kZZA91/J+RIVsAKsgijf2z4xv6eNLUF2fAgCIgwdwc1Fvos89t +LqRbWYpfe8B8qa9VJrmkjuxUKyjaQ+17X2LcqkyPqmY1enYunuud+jZ4tjrsYz7A +HVYJmKQ+ODzMD/Y5RNzzKO53GShGEwFioPu9WLbIbk5u4jRN0MrCVZ360C3jfpW/ +RFfPAg5qfg6vUQgcOUA+NGzhbMc7qn1mae3hhyxRRAvqoGUpuhxGScFWJcd4r8N3 +wHUdsCwkPFkCggEAYmVs5ti/Uzrw/hA9Z8fSWQiqSc3kFKYAZKGNr8Lu7H+5VJI7 +47ReusEA7Y+ChzzlyeldZ7Bh4yyoCBulp9q9UcbOSixphe2XIR8h90sqZ5CkLWWR +WbJfKW8wPNzgqA7L310lM2IG/F2bMJjoy/4E/r2AU5D/1PD22VJS8BsfyD+9Crwv +f7BnzfMaeFHyDuW5WlzK7FvZF2q//dbKw3ceOhc7Cbfz4GLrAFYZaFkZngnJfImO +qYwsMLI3RV3moHRxo4yURTj7kX4E8vsRv9JKI6dZL/7bPClPf9i/1IfvA/A9PARI +nasNMiAUv98EnqK2LvqSJzcrin5MFp3itEccoQKCAQBTJWymrbTAfX5PVrr48/mn +0Lu2WFKeRRNi03U6EeUwh8q06jO53j4X9rKG7RothfNIl08cPvN6+0Ad1oC3u8kk +ltIqcOYNi4ACPOhaE2j+HRbSOchSGACJW8NCk/POxTTanARdIN2YNVAkLgH41cNN +HfFyva6ZV3rETj1PFs9Flc9dJDFAMfIX4DggCt6pnm58YR+Tqs5MAULkWw193RQD +MhuBnX1+pttr53A3ZSu0xekoPzCgErLrSd1K09f4BQXAiIzSuj+TIK1prqH/eZdw +XshnwVg2Pv5gW5S4w28iA36MAiDAOw0EWihae+k+iQKR+DE+MfqibYu1N4kKSvQp +AoIBAFZQJUQ8NR0K46HY9ytTOmq/bVZqmTgms3ZnQ3pXHmKmHugBfMOvQd/bHf2w +6lG2XdECl6QbwmePf4SXzPl2MJLg43PlvssLtSMjy/7WymVHXQVEiNdtDtIKWibz +9QghjZbHPoGaszDk/UyqVMRBftr8Fv4dJ8u6Yhgnd08FGVVPoyuTa6ZTiygfg2TR +zNEYn9hIlZND5dW1zhgumtZ+KpgzUpfiJwqECWhvcV/GihbGA03NZJvKq1motqb/ +oC38fsBUwL6DmN02j4U9dUAs1q6tF3aVvHiu6SWObzI1DiPBUXY1wQXZLAErDIM+ +HCAHKiu9dnG3hG8A+Ny8cHtCviE= +-----END PRIVATE KEY----- diff --git a/core/functional_tests/https_no_passphrase/service.cpp b/core/functional_tests/https_no_passphrase/service.cpp new file mode 100644 index 000000000000..a10f3dfc59e5 --- /dev/null +++ b/core/functional_tests/https_no_passphrase/service.cpp @@ -0,0 +1,33 @@ +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "httpserver_handlers.hpp" + +int main(int argc, char* argv[]) { + const auto component_list = components::MinimalServerComponentList() + .Append() + .Append() + .Append() + .Append() + .Append() + .Append() + .Append() + .Append() + .Append(); + + return utils::DaemonMain(argc, argv, component_list); +} diff --git a/core/functional_tests/https_no_passphrase/static_config.yaml b/core/functional_tests/https_no_passphrase/static_config.yaml new file mode 100644 index 000000000000..fe6e8d1b4bce --- /dev/null +++ b/core/functional_tests/https_no_passphrase/static_config.yaml @@ -0,0 +1,76 @@ +# yaml +components_manager: + components: + handler-https-httpserver: + path: /https/httpserver + task_processor: main-task-processor + method: GET,DELETE,POST + + handler-server-monitor: + path: /service/monitor + method: GET + task_processor: main-task-processor + + handler-on-log-rotate: + path: /service/on-log-rotate/ + method: POST + task_processor: monitor-task-processor + + testsuite-support: + + http-client: + fs-task-processor: main-task-processor + + tests-control: + method: POST + path: /tests/{action} + skip-unregistered-testpoints: true + task_processor: main-task-processor + testpoint-timeout: 10s + testpoint-url: $mockserver/testpoint + throttling_enabled: false + + logging-configurator: + limited-logging-enable: false + limited-logging-interval: 10s + + server: + listener: + port: 8187 + task_processor: main-task-processor + handler-defaults: + deadline_expired_status_code: 504 + tls: + cert: cert.crt + private-key: private_key.key + + listener-monitor: + port: 8188 + task_processor: main-task-processor + logging: + fs-task-processor: fs-task-processor + loggers: + default: + file_path: '@stderr' + level: debug + overflow_behavior: discard + + dns-client: + fs-task-processor: fs-task-processor + + handler-fired-alerts: + path: /service/fired-alerts + method: GET + task_processor: main-task-processor + + task_processors: + main-task-processor: + worker_threads: 4 + fs-task-processor: + worker_threads: 2 + monitor-task-processor: + thread_name: mon-worker + worker_threads: $monitor_worker_threads + worker_threads#fallback: 1 + + default_task_processor: main-task-processor diff --git a/core/functional_tests/https_no_passphrase/tests-deadline/conftest.py b/core/functional_tests/https_no_passphrase/tests-deadline/conftest.py new file mode 100644 index 000000000000..413d276ffd67 --- /dev/null +++ b/core/functional_tests/https_no_passphrase/tests-deadline/conftest.py @@ -0,0 +1,31 @@ +import aiohttp +import pytest + +pytest_plugins = ['pytest_userver.plugins.core'] +USERVER_CONFIG_HOOKS = ['userver_config_secdist'] + + +@pytest.fixture(scope='session') +def userver_config_secdist(userver_config_http_client, service_source_dir): + def patch_config(config_yaml, config_vars) -> None: + components = config_yaml['components_manager']['components'] + tls = components['server']['listener']['tls'] + tls['cert'] = str(service_source_dir / 'cert.crt') + tls['private-key'] = str(service_source_dir / 'private_key.key') + + return patch_config + + +@pytest.fixture(scope='session') +def service_baseurl(service_port) -> str: + return f'https://localhost:{service_port}/' + + +@pytest.fixture(scope='session') +def service_client_session_factory(event_loop, service_source_dir): + def make_session(**kwargs): + kwargs.setdefault('loop', event_loop) + kwargs['connector'] = aiohttp.TCPConnector(verify_ssl=False) + return aiohttp.ClientSession(**kwargs) + + return make_session diff --git a/core/functional_tests/https_no_passphrase/tests-deadline/test_server.py b/core/functional_tests/https_no_passphrase/tests-deadline/test_server.py new file mode 100644 index 000000000000..2caf65e10c71 --- /dev/null +++ b/core/functional_tests/https_no_passphrase/tests-deadline/test_server.py @@ -0,0 +1,21 @@ +import asyncio +import pathlib + +import pytest + +SERVICE_SOURCE_DIR = pathlib.Path(__file__).parent.parent + + +async def test_cancellable(service_client, testpoint, service_port): + @testpoint('testpoint_cancel') + def cancel_testpoint(data): + pass + + less_than_handler_sleep_time = 1.0 + with pytest.raises(asyncio.TimeoutError): + await service_client.get( + '/https/httpserver', + params={'type': 'cancel'}, + timeout=less_than_handler_sleep_time, + ) + await cancel_testpoint.wait_call() diff --git a/core/src/server/pph_config.hpp b/core/src/server/pph_config.hpp index ea8b77fc5e66..248065820f36 100644 --- a/core/src/server/pph_config.hpp +++ b/core/src/server/pph_config.hpp @@ -19,12 +19,15 @@ class PassphraseConfig final { {})) {} Passphrase GetPassphrase(const std::string& name) const { - try { - return passphrases_.at(name); - } catch (const std::out_of_range& e) { - LOG_ERROR() << "No passphrase for name '" << name << "'"; - throw; + auto it = passphrases_.find(name); + if (it == passphrases_.cend()) { + auto message = fmt::format( + "No passphrase with name '{}' in secdist 'passphrases' entry", name); + LOG_ERROR() << message; + throw std::runtime_error(std::move(message)); } + + return it->second; } private: diff --git a/core/src/server/server.cpp b/core/src/server/server.cpp index 51cdac9bb23e..392a83000d57 100644 --- a/core/src/server/server.cpp +++ b/core/src/server/server.cpp @@ -152,10 +152,15 @@ ServerImpl::ServerImpl(ServerConfig config, if (config_.listener.tls) { auto contents = fs::blocking::ReadFileContents(config_.listener.tls_private_key_path); - auto pph = secdist.Get().GetPassphrase( - config_.listener.tls_private_key_passphrase_name); - config_.listener.tls_private_key = - crypto::PrivateKey::LoadFromString(contents, pph.GetUnderlying()); + if (config_.listener.tls_private_key_passphrase_name.empty()) { + config_.listener.tls_private_key = + crypto::PrivateKey::LoadFromString(contents); + } else { + auto pph = secdist.Get().GetPassphrase( + config_.listener.tls_private_key_passphrase_name); + config_.listener.tls_private_key = + crypto::PrivateKey::LoadFromString(contents, pph.GetUnderlying()); + } } main_port_info_.Init(config_, config_.listener, component_context, false); From 15686332a83f921f2f9a1644a09e864a69516b9a Mon Sep 17 00:00:00 2001 From: kpavlov00 Date: Wed, 17 Jul 2024 08:16:58 +0300 Subject: [PATCH 039/629] fix ydb: add SetupBrotli.cmake 0ec8b712e1c06398c81ff76879d6eb5294051bc8 --- .mapping.json | 1 + cmake/SetupBrotli.cmake | 40 +++++++++++++++++++++++ cmake/SetupYdbCppSDK.cmake | 11 ++----- scripts/docs/en/userver/tutorial/build.md | 1 + 4 files changed, 44 insertions(+), 9 deletions(-) create mode 100644 cmake/SetupBrotli.cmake diff --git a/.mapping.json b/.mapping.json index 5ff1fed21ef3..d26a596b2333 100644 --- a/.mapping.json +++ b/.mapping.json @@ -305,6 +305,7 @@ "cmake/Sanitizers.cmake":"taxi/uservices/userver/cmake/Sanitizers.cmake", "cmake/SetupAbseil.cmake":"taxi/uservices/userver/cmake/SetupAbseil.cmake", "cmake/SetupAmqpCPP.cmake":"taxi/uservices/userver/cmake/SetupAmqpCPP.cmake", + "cmake/SetupBrotli.cmake":"taxi/uservices/userver/cmake/SetupBrotli.cmake", "cmake/SetupCAres.cmake":"taxi/uservices/userver/cmake/SetupCAres.cmake", "cmake/SetupCCTZ.cmake":"taxi/uservices/userver/cmake/SetupCCTZ.cmake", "cmake/SetupCURL.cmake":"taxi/uservices/userver/cmake/SetupCURL.cmake", diff --git a/cmake/SetupBrotli.cmake b/cmake/SetupBrotli.cmake new file mode 100644 index 000000000000..317b5545ebf8 --- /dev/null +++ b/cmake/SetupBrotli.cmake @@ -0,0 +1,40 @@ +option(USERVER_DOWNLOAD_PACKAGE_BROTLI "Download and setup Brotli if no Brotli of matching version was found" ${USERVER_DOWNLOAD_PACKAGES}) + +set(USERVER_BROTLI_VERSION 1.1.0) + +if (NOT USERVER_FORCE_DOWNLOAD_PACKAGES) + if (USERVER_DOWNLOAD_PACKAGE_BROTLI) + find_package(Brotli ${USERVER_BROTLI_VERSION} QUIET) + else() + find_package(Brotli ${USERVER_BROTLI_VERSION} REQUIRED) + endif() + + if (Brotli_FOUND) + if(NOT TARGET Brotli::dec) + add_library(Brotli::dec ALIAS brotlidec) + endif() + if(NOT TARGET Brotli::enc) + add_library(Brotli::enc ALIAS brotlienc) + endif() + return() + endif() +endif() + +include(DownloadUsingCPM) +CPMAddPackage( + NAME Brotli + VERSION ${USERVER_BROTLI_VERSION} + GITHUB_REPOSITORY google/brotli +) + +set(Brotli_FOUND TRUE) +set(Brotli_VERSION ${USERVER_BROTLI_VERSION}) +add_custom_target(Brotli) +write_package_stub(Brotli) + +if(NOT TARGET Brotli::dec) + add_library(Brotli::dec ALIAS brotlidec) +endif() +if(NOT TARGET Brotli::enc) + add_library(Brotli::enc ALIAS brotlienc) +endif() diff --git a/cmake/SetupYdbCppSDK.cmake b/cmake/SetupYdbCppSDK.cmake index 2bbf4e4d247e..34e2391559e4 100644 --- a/cmake/SetupYdbCppSDK.cmake +++ b/cmake/SetupYdbCppSDK.cmake @@ -1,6 +1,7 @@ option(USERVER_DOWNLOAD_PACKAGE_YDBCPPSDK "Download and setup ydb-cpp-sdk" ${USERVER_DOWNLOAD_PACKAGES}) include(DownloadUsingCPM) +include(SetupBrotli) CPMAddPackage( NAME base64 @@ -10,15 +11,6 @@ CPMAddPackage( write_package_stub(base64) add_library(aklomp::base64 ALIAS base64) -CPMAddPackage( - NAME Brotli - VERSION 1.1.0 - GITHUB_REPOSITORY google/brotli -) -write_package_stub(Brotli) -add_library(Brotli::dec ALIAS brotlidec) -add_library(Brotli::enc ALIAS brotlienc) - CPMAddPackage( NAME jwt-cpp VERSION 0.7.0 @@ -43,6 +35,7 @@ CPMAddPackage( GIT_TAG main GITHUB_REPOSITORY ydb-platform/ydb-cpp-sdk OPTIONS + "Brotli_VERSION ${Brotli_VERSION}" "RAPIDJSON_INCLUDE_DIRS ${RAPIDJSON_INCLUDE_DIRS}" "YDB_SDK_GOOGLE_COMMON_PROTOS_TARGET ${YDB_SDK_GOOGLE_COMMON_PROTOS_TARGET}" ) diff --git a/scripts/docs/en/userver/tutorial/build.md b/scripts/docs/en/userver/tutorial/build.md index f2715fa5b88e..d068e1aa6fc9 100644 --- a/scripts/docs/en/userver/tutorial/build.md +++ b/scripts/docs/en/userver/tutorial/build.md @@ -106,6 +106,7 @@ The following CMake options are used by userver: | USERVER_DOWNLOAD_PACKAGES | Download missing third party packages and use the downloaded versions | ON | | USERVER_PIP_USE_SYSTEM_PACKAGES | Use system python packages inside venv | OFF | | USERVER_PIP_OPTIONS | Options for all pip calls | '' | +| USERVER_DOWNLOAD_PACKAGE_BROTLI | Download and setup Brotli if no Brotli of matching version was found | ${USERVER_DOWNLOAD_PACKAGES} | | USERVER_DOWNLOAD_PACKAGE_CARES | Download and setup c-ares if no c-ares of matching version was found | ${USERVER_DOWNLOAD_PACKAGES} | | USERVER_DOWNLOAD_PACKAGE_CCTZ | Download and setup cctz if no cctz of matching version was found | ${USERVER_DOWNLOAD_PACKAGES} | | USERVER_DOWNLOAD_PACKAGE_CLICKHOUSECPP | Download and setup clickhouse-cpp | ${USERVER_DOWNLOAD_PACKAGES} | From 42f4efef1d2f3d1fbfa415f2e693bd9763ac230a Mon Sep 17 00:00:00 2001 From: antoshkka Date: Wed, 17 Jul 2024 11:31:15 +0300 Subject: [PATCH 040/629] feat docs: release 2.2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tests: протестировано CI 8f1653c95b53e82a1d6bec5e61fd1282e1d854d4 --- cmake/GetUserverVersion.cmake | 2 +- scripts/docs/en/userver/roadmap_and_changelog.md | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/cmake/GetUserverVersion.cmake b/cmake/GetUserverVersion.cmake index 5556173da1ff..c1c1ebc73956 100644 --- a/cmake/GetUserverVersion.cmake +++ b/cmake/GetUserverVersion.cmake @@ -23,7 +23,7 @@ else() endif() set(USERVER_MAJOR_VERSION 2) -set(USERVER_MINOR_VERSION 2-rc) +set(USERVER_MINOR_VERSION 2) set(USERVER_VERSION "${USERVER_MAJOR_VERSION}.${USERVER_MINOR_VERSION}") string(REPLACE "-" "_" USERVER_VERSION_STR "${USERVER_VERSION}") diff --git a/scripts/docs/en/userver/roadmap_and_changelog.md b/scripts/docs/en/userver/roadmap_and_changelog.md index cf16730064d3..c2397eb1de97 100644 --- a/scripts/docs/en/userver/roadmap_and_changelog.md +++ b/scripts/docs/en/userver/roadmap_and_changelog.md @@ -38,8 +38,8 @@ Changelog news also go to the Many thanks to [Fedor Alekseev](https://github.com/atlz253) for the PR and to [MariaGrinchenko](https://github.com/MariaGrinchenko) for the buttons design! * Added @ref scripts/docs/en/userver/ydb.md "docs on YDB". -* Mobile header view was improved. Many thanks to - [Fedor Alekseev](https://github.com/atlz253) for the PR. +* Mobile header view and docs layout was improved. Many thanks to + [Fedor Alekseev](https://github.com/atlz253) for the PRs. * engine::subprocess::ProcessStarter::Exec now can lookup binaries via `PATH` variable. * Fixed gRPC generation for nested namespaces with repetitions. Many thanks to @@ -51,6 +51,7 @@ Changelog news also go to the * yaml_config::YamlConfig now can read files via `#file`. Now the static config of the service could refer to other files. * Added support of bit operations to Redis. +* PostgreSQL driver now works with AWS Aurora. * Added quick start for beginners to @ref scripts/docs/en/userver/tutorial/build.md. Many thanks to [Fedor Alekseev](https://github.com/atlz253) for the PR. * Improved path to sources trimming for Conan builds. Many thanks to From 112b5359b5ebf81b43b5d68820d2a462aaf60b07 Mon Sep 17 00:00:00 2001 From: antoshkka Date: Wed, 17 Jul 2024 11:45:40 +0300 Subject: [PATCH 041/629] feat docs: start 2.3-rc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tests: протестировано CI 3c08540bf3243e545bafc070d739ce5cc0e07e50 --- cmake/GetUserverVersion.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/GetUserverVersion.cmake b/cmake/GetUserverVersion.cmake index c1c1ebc73956..21ccf99842ed 100644 --- a/cmake/GetUserverVersion.cmake +++ b/cmake/GetUserverVersion.cmake @@ -23,7 +23,7 @@ else() endif() set(USERVER_MAJOR_VERSION 2) -set(USERVER_MINOR_VERSION 2) +set(USERVER_MINOR_VERSION 3-rc) set(USERVER_VERSION "${USERVER_MAJOR_VERSION}.${USERVER_MINOR_VERSION}") string(REPLACE "-" "_" USERVER_VERSION_STR "${USERVER_VERSION}") From 2b6e4cb12bf94299bda48fe2062bef6a1666451f Mon Sep 17 00:00:00 2001 From: segoon Date: Wed, 17 Jul 2024 12:01:01 +0300 Subject: [PATCH 042/629] bug logger: fix use-after-free on service stop e3ba41d900c3f41177323465f1f21e37c7ca3893 --- core/src/components/run.cpp | 7 +++++-- otlp/src/otlp/logs/component.cpp | 2 +- .../include/userver/logging/impl/logger_base.hpp | 2 +- universal/include/userver/logging/impl/mem_logger.hpp | 11 +++++++---- universal/src/logging/impl/logger_base.cpp | 2 +- 5 files changed, 15 insertions(+), 9 deletions(-) diff --git a/core/src/components/run.cpp b/core/src/components/run.cpp index c8a320d0c15a..1ca47a31b489 100644 --- a/core/src/components/run.cpp +++ b/core/src/components/run.cpp @@ -50,13 +50,17 @@ class LogScope final { : logger_prev_{logging::GetDefaultLogger()}, level_scope_{logging::GetDefaultLoggerLevel()} {} - ~LogScope() { logging::impl::SetDefaultLoggerRef(logger_prev_); } + ~LogScope() { + logger_prev_.ForwardTo(nullptr); + logging::impl::SetDefaultLoggerRef(logger_prev_); + } void SetLogger(logging::LoggerPtr logger) { UASSERT(logger); logging::impl::SetDefaultLoggerRef(*logger); // Destroys the old logger_new_ logger_new_ = std::move(logger); + logger_prev_.ForwardTo(&*logger_new_); } logging::LoggerRef GetPrevLogger() { return logger_prev_; } @@ -176,7 +180,6 @@ ManagerConfig ParseManagerConfigAndSetupLogging( // This line enables basic logging. Any logging before would go to // MemLogger and be transferred to the logger in the cycle below. log_scope.SetLogger(default_logger); - log_scope.GetPrevLogger().ForwardTo(*default_logger); } LOG_INFO() << "Parsed " << details; diff --git a/otlp/src/otlp/logs/component.cpp b/otlp/src/otlp/logs/component.cpp index d0c839a4d632..eab03e0f4aef 100644 --- a/otlp/src/otlp/logs/component.cpp +++ b/otlp/src/otlp/logs/component.cpp @@ -57,7 +57,7 @@ LoggerComponent::LoggerComponent(const components::ComponentConfig& config, auto& old_logger = logging::GetDefaultLogger(); logging::impl::SetDefaultLoggerRef(*logger_); - old_logger.ForwardTo(*logger_); + old_logger.ForwardTo(&*logger_); auto* const statistics_storage = context.FindComponentOptional(); diff --git a/universal/include/userver/logging/impl/logger_base.hpp b/universal/include/userver/logging/impl/logger_base.hpp index 32abb7c61eb0..1e200a383bae 100644 --- a/universal/include/userver/logging/impl/logger_base.hpp +++ b/universal/include/userver/logging/impl/logger_base.hpp @@ -39,7 +39,7 @@ class LoggerBase { void SetFlushOn(Level level); bool ShouldFlush(Level level) const; - virtual void ForwardTo(LoggerBase& logger_to); + virtual void ForwardTo(LoggerBase* logger_to); protected: virtual bool DoShouldLog(Level level) const noexcept; diff --git a/universal/include/userver/logging/impl/mem_logger.hpp b/universal/include/userver/logging/impl/mem_logger.hpp index feb96a8ebc47..40347c9a324d 100644 --- a/universal/include/userver/logging/impl/mem_logger.hpp +++ b/universal/include/userver/logging/impl/mem_logger.hpp @@ -42,12 +42,15 @@ class MemLogger final : public LoggerBase { } void Flush() override {} - void ForwardTo(LoggerBase& logger_to) override { + void ForwardTo(LoggerBase* logger_to) override { std::unique_lock lock(mutex_); - for (const auto& log : data_) { - logger_to.Log(log.level, log.msg); + if (logger_to) { + for (const auto& log : data_) { + logger_to->Log(log.level, log.msg); + } + data_.clear(); } - forward_logger_ = &logger_to; + forward_logger_ = logger_to; } protected: diff --git a/universal/src/logging/impl/logger_base.cpp b/universal/src/logging/impl/logger_base.cpp index 11c48fd84834..7febd1508635 100644 --- a/universal/src/logging/impl/logger_base.cpp +++ b/universal/src/logging/impl/logger_base.cpp @@ -30,7 +30,7 @@ bool LoggerBase::ShouldFlush(Level level) const { return flush_level_ <= level; } -void LoggerBase::ForwardTo(LoggerBase&) {} +void LoggerBase::ForwardTo(LoggerBase*) {} bool LoggerBase::DoShouldLog(Level /*level*/) const noexcept { return true; } From d717954eccc0d81a58937c34e71abd71d5ae759f Mon Sep 17 00:00:00 2001 From: Aleksandr Gusev <108462407+ALumad@users.noreply.github.com> Date: Wed, 17 Jul 2024 13:39:17 +0300 Subject: [PATCH 043/629] feat build: added kafka in conanfile.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tests: протестировано CI Pull Request resolved: #648 Co-authored-by: ALumad d406a1786ccdab4f7ec1491e7191cbb28100c239 --- .mapping.json | 1 + conan/test_package/CMakeLists.txt | 3 +++ conan/test_package/test_kafka.cpp | 29 +++++++++++++++++++++++++++++ conanfile.py | 22 +++++++++++++++++++++- 4 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 conan/test_package/test_kafka.cpp diff --git a/.mapping.json b/.mapping.json index d26a596b2333..08909cad0062 100644 --- a/.mapping.json +++ b/.mapping.json @@ -364,6 +364,7 @@ "conan/test_package/test_clickhouse.cpp":"taxi/uservices/userver/conan/test_package/test_clickhouse.cpp", "conan/test_package/test_core.cpp":"taxi/uservices/userver/conan/test_package/test_core.cpp", "conan/test_package/test_grpc.cpp":"taxi/uservices/userver/conan/test_package/test_grpc.cpp", + "conan/test_package/test_kafka.cpp":"taxi/uservices/userver/conan/test_package/test_kafka.cpp", "conan/test_package/test_mongo.cpp":"taxi/uservices/userver/conan/test_package/test_mongo.cpp", "conan/test_package/test_postgresql.cpp":"taxi/uservices/userver/conan/test_package/test_postgresql.cpp", "conan/test_package/test_rabbitmq.cpp":"taxi/uservices/userver/conan/test_package/test_rabbitmq.cpp", diff --git a/conan/test_package/CMakeLists.txt b/conan/test_package/CMakeLists.txt index 16e303f6575f..ff457e985b2e 100644 --- a/conan/test_package/CMakeLists.txt +++ b/conan/test_package/CMakeLists.txt @@ -43,4 +43,7 @@ target_link_libraries(${PROJECT_NAME}_clickhouse ${PROJECT_NAME}_objs userver::c add_executable(${PROJECT_NAME}_universal test_universal.cpp) target_link_libraries(${PROJECT_NAME}_universal ${PROJECT_NAME}_objs userver::universal) +add_executable(${PROJECT_NAME}_kafka test_kafka.cpp) +target_link_libraries(${PROJECT_NAME}_kafka ${PROJECT_NAME}_objs userver::kafka) + add_subdirectory(hello_service) diff --git a/conan/test_package/test_kafka.cpp b/conan/test_package/test_kafka.cpp new file mode 100644 index 000000000000..4c275db786f3 --- /dev/null +++ b/conan/test_package/test_kafka.cpp @@ -0,0 +1,29 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "hello.hpp" + +int main(int argc, char* argv[]) { + auto component_list = + userver::components::MinimalServerComponentList() + .Append() + .Append() + .Append() + .Append() + .Append("") + .Append(""); + + service_template::AppendHello(component_list); + + auto size = std::distance(component_list.begin(), component_list.end()); + std::cout << size << std::endl; + + return 0; +} diff --git a/conanfile.py b/conanfile.py index 355d31034c66..f794d4737f3d 100644 --- a/conanfile.py +++ b/conanfile.py @@ -38,6 +38,7 @@ class UserverConan(ConanFile): 'with_clickhouse': [True, False], 'with_rabbitmq': [True, False], 'with_utest': [True, False], + 'with_kafka': [True, False], 'namespace': ['ANY'], 'namespace_begin': ['ANY'], 'namespace_end': ['ANY'], @@ -56,6 +57,7 @@ class UserverConan(ConanFile): 'with_clickhouse': True, 'with_rabbitmq': True, 'with_utest': True, + 'with_kafka': True, 'namespace': 'userver', 'namespace_begin': 'namespace userver {', 'namespace_end': '}', @@ -160,6 +162,8 @@ def requirements(self): transitive_headers=True, transitive_libs=True, ) + if self.options.with_kafka: + self.requires('librdkafka/2.4.0') def validate(self): if ( @@ -210,7 +214,7 @@ def generate(self): tool_ch.variables[ 'USERVER_FEATURE_TESTSUITE' ] = self.options.with_utest - + tool_ch.variables['USERVER_FEATURE_KAFKA'] = self.options.with_kafka tool_ch.generate() CMakeDeps(self).generate() @@ -378,6 +382,9 @@ def copy_component(component): if self.options.with_clickhouse: copy_component('clickhouse') + if self.options.with_kafka: + copy_component('kafka') + @property def _userver_components(self): def abseil(): @@ -476,6 +483,9 @@ def clickhouse(): else [] ) + def librdkafka(): + return ['RdKafka::rdkafka++'] if self.options.with_kafka else [] + userver_components = [ { 'target': 'core', @@ -615,6 +625,16 @@ def clickhouse(): }, ], ) + if self.options.with_kafka: + userver_components.extend( + [ + { + 'target': 'kafka', + 'lib': 'kafka', + 'requires': ['core'] + librdkafka(), + }, + ], + ) return userver_components def package_info(self): From 2e9532a66a3fef069177227eff9b3365d4b2baee Mon Sep 17 00:00:00 2001 From: i9kin Date: Thu, 18 Jul 2024 12:38:10 +0300 Subject: [PATCH 044/629] bug clients: always use url from SetLoggedUrl in logs when specified 46d0d60fd12800c6f4c5d32cfd1644473deb774c --- core/src/clients/http/request_state.cpp | 11 ++++++++--- core/src/clients/http/request_state.hpp | 1 + 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/core/src/clients/http/request_state.cpp b/core/src/clients/http/request_state.cpp index 7df80da862ce..ddf437dd512b 100644 --- a/core/src/clients/http/request_state.cpp +++ b/core/src/clients/http/request_state.cpp @@ -18,7 +18,6 @@ #include #include #include -#include #include #include #include @@ -675,6 +674,12 @@ const std::string& RequestState::GetLoggedOriginalUrl() const noexcept { return log_url_ ? *log_url_ : easy().get_original_url(); } +std::string_view RequestState::GetLoggedEffectiveUrl() noexcept { + // If log_url_ exists, we use log_url_ with a semantic like original_url, + // instead of effective_url + return log_url_ ? *log_url_ : easy().get_effective_url(); +} + engine::Future> RequestState::async_perform( utils::impl::SourceLocation location) { data_.emplace(); @@ -913,11 +918,11 @@ void RequestState::AccountResponse(std::error_code err) { std::exception_ptr RequestState::PrepareException(std::error_code err) { if (deadline_expired_) { - return PrepareDeadlinePassedException(easy().get_effective_url(), + return PrepareDeadlinePassedException(GetLoggedEffectiveUrl(), easy().get_local_stats()); } - return http::PrepareException(err, easy().get_effective_url(), + return http::PrepareException(err, GetLoggedEffectiveUrl(), easy().get_local_stats()); } diff --git a/core/src/clients/http/request_state.hpp b/core/src/clients/http/request_state.hpp index bfc3e3b7afc9..0b24085a098c 100644 --- a/core/src/clients/http/request_state.hpp +++ b/core/src/clients/http/request_state.hpp @@ -167,6 +167,7 @@ class RequestState : public std::enable_shared_from_this { bool ShouldRetryResponse(); const std::string& GetLoggedOriginalUrl() const noexcept; + std::string_view GetLoggedEffectiveUrl() noexcept; static size_t StreamWriteFunction(char* ptr, size_t size, size_t nmemb, void* userdata); From 6a403b7c72d7ef192ea4969cad39ee545fd4c85b Mon Sep 17 00:00:00 2001 From: antoshkka Date: Thu, 18 Jul 2024 13:05:22 +0300 Subject: [PATCH 045/629] feat universal: do not leave a temporary file in case of write error MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tests: протестировано CI 93a6bf5597cc15f936f4e23cb59071bd9c0f6cb6 --- universal/src/fs/blocking/write.cpp | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/universal/src/fs/blocking/write.cpp b/universal/src/fs/blocking/write.cpp index 548b8a92bb66..34cfec2b195c 100644 --- a/universal/src/fs/blocking/write.cpp +++ b/universal/src/fs/blocking/write.cpp @@ -10,6 +10,7 @@ #include #include #include +#include USERVER_NAMESPACE_BEGIN @@ -70,9 +71,9 @@ void RewriteFileContentsFSync(const std::string& path, constexpr OpenMode flags{OpenFlag::kWrite, OpenFlag::kCreateIfNotExists, OpenFlag::kTruncate}; auto fd = FileDescriptor::Open(path, flags); - fd.Write(contents); fd.FSync(); + std::move(fd).Close(); } @@ -89,12 +90,20 @@ void Rename(const std::string& source, const std::string& destination) { void RewriteFileContentsAtomically(const std::string& path, std::string_view contents, boost::filesystem::perms perms) { - auto tmp_path = + const auto tmp_path = fmt::format("{}{}.tmp", path, utils::generators::GenerateBoostUuid()); - const auto directory = boost::filesystem::path{path}.parent_path().string(); + const boost::filesystem::path boost_tmp_path{path}; + const auto directory = boost_tmp_path.parent_path().string(); + + utils::FastScopeGuard guard{[&boost_tmp_path]() noexcept { + boost::system::error_code ignore_error; + boost::filesystem::remove(boost_tmp_path, ignore_error); + }}; fs::blocking::RewriteFileContentsFSync(tmp_path, contents); fs::blocking::Chmod(tmp_path, perms); fs::blocking::Rename(tmp_path, path); + guard.Release(); + fs::blocking::SyncDirectoryContents(directory); } From f9b3ee4c39c7714bcae42e559929b5e76e0169bc Mon Sep 17 00:00:00 2001 From: antoshkka Date: Thu, 18 Jul 2024 13:18:46 +0300 Subject: [PATCH 046/629] feat core: better balancing of events between ev threads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The service_template RPS on small workloads remains the same: ``` 2 ev 4 THREADS BEFORE Requests/sec: 151580.31 AFTER Requests/sec: 153493.57 2 ev 6 THREADS BEFORE Requests/sec: 194260.31 AFTER Requests/sec: 195998.52 2 ev 8 THREADS BEFORE Requests/sec: 236516.72 AFTER Requests/sec: 231662.19 ``` The ev load however is now better distributed: Before: ``` 18272 antoshk+ 20 0 3009228 155936 52192 R 87.6 1.0 0:16.39 event-worker_0 18273 antoshk+ 20 0 3009228 155936 52192 R 34.0 1.0 0:05.87 event-worker_1 ``` After: ``` 23495 antoshk+ 20 0 3008264 155180 52268 R 66.7 1.0 0:31.73 event-worker_1 23494 antoshk+ 20 0 3008264 155180 52268 R 61.9 1.0 0:31.07 event-worker_0 ``` Fixes: https://github.com/userver-framework/userver/issues/216 Tests: протестирорвано CI и локально a1c138152a5903449affbe2011b805ad51c6964c --- core/include/userver/engine/io/fd_poller.hpp | 12 +++++++++++- core/src/engine/io/fd_control.cpp | 12 ++++++++---- core/src/engine/io/fd_control.hpp | 4 ++-- core/src/engine/io/fd_poller.cpp | 8 +++++--- core/src/engine/io/fd_poller_test.cpp | 5 +++-- core/src/engine/io/sys_linux/inotify.cpp | 3 ++- mysql/src/storages/mysql/impl/socket.cpp | 9 ++++++--- 7 files changed, 37 insertions(+), 16 deletions(-) diff --git a/core/include/userver/engine/io/fd_poller.hpp b/core/include/userver/engine/io/fd_poller.hpp index d4e1b0cb70a9..08ebf810faa2 100644 --- a/core/include/userver/engine/io/fd_poller.hpp +++ b/core/include/userver/engine/io/fd_poller.hpp @@ -15,6 +15,10 @@ namespace engine::impl { class ContextAccessor; } +namespace engine::ev { +class ThreadControl; +} + namespace engine::io { namespace impl { @@ -34,7 +38,13 @@ class FdPoller final { kReadWrite = 3, /// < wait for either read or write availability }; - FdPoller(); + /// Constructor for FdPoller. `contol` parameter could be obtained via + /// engine::current_task::GetEventThread(). + /// + /// It is recommended to place read and write FdPoller's of the same FD to + /// the same `control` for better ev threads balancing. + explicit FdPoller(const ev::ThreadControl& control); + FdPoller(const FdPoller&) = delete; FdPoller(FdPoller&&) = delete; FdPoller& operator=(const FdPoller&) = delete; diff --git a/core/src/engine/io/fd_control.cpp b/core/src/engine/io/fd_control.cpp index 0dbcb1679fba..ea350ad70020 100644 --- a/core/src/engine/io/fd_control.cpp +++ b/core/src/engine/io/fd_control.cpp @@ -65,7 +65,8 @@ Direction::SingleUserGuard::~SingleUserGuard() { } #endif // #ifndef NDEBUG -Direction::Direction(Kind kind) : kind_(kind) {} +Direction::Direction(Kind kind, const ev::ThreadControl& control) + : poller_(control), kind_(kind) {} Direction::~Direction() = default; @@ -83,8 +84,11 @@ void Direction::Reset(int fd) { poller_.Reset(fd, kind_); } void Direction::Invalidate() { poller_.Invalidate(); } -FdControl::FdControl() - : read_(Direction::Kind::kRead), write_(Direction::Kind::kWrite) {} +// Write operations on socket usually do not block, so it makes sense to reuse +// the same ThreadControl for the sake of better balancing of ev threads. +FdControl::FdControl(const ev::ThreadControl& control) + : read_(Direction::Kind::kRead, control), + write_(Direction::Kind::kWrite, control) {} FdControl::~FdControl() { try { @@ -95,7 +99,7 @@ FdControl::~FdControl() { } FdControlHolder FdControl::Adopt(int fd) { - FdControlHolder fd_control{new FdControl()}; + FdControlHolder fd_control{new FdControl(current_task::GetEventThread())}; // TODO: add conditional CLOEXEC set SetCloexec(fd); SetNonblock(fd); diff --git a/core/src/engine/io/fd_control.hpp b/core/src/engine/io/fd_control.hpp index f72a6be2660f..b61fc5cc2aba 100644 --- a/core/src/engine/io/fd_control.hpp +++ b/core/src/engine/io/fd_control.hpp @@ -92,7 +92,7 @@ class Direction final { private: friend class FdControl; - explicit Direction(Kind kind); + Direction(Kind kind, const ev::ThreadControl& control); void Reset(int fd); void WakeupWaiters() { poller_.WakeupWaiters(); } @@ -114,7 +114,7 @@ class FdControl final { // fd will be silently forced to nonblocking mode static FdControlHolder Adopt(int fd); - FdControl(); + explicit FdControl(const ev::ThreadControl& control); ~FdControl(); explicit operator bool() const { return IsValid(); } diff --git a/core/src/engine/io/fd_poller.cpp b/core/src/engine/io/fd_poller.cpp index 2a44f525a534..01e217f83644 100644 --- a/core/src/engine/io/fd_poller.cpp +++ b/core/src/engine/io/fd_poller.cpp @@ -68,7 +68,7 @@ FdPoller::Kind GetUserMode(int ev_events) { } // namespace struct FdPoller::Impl final : public engine::impl::ContextAccessor { - Impl(); + Impl(ev::ThreadControl control); ~Impl(); @@ -117,7 +117,7 @@ struct FdPoller::Impl final : public engine::impl::ContextAccessor { void FdPoller::Impl::WakeupWaiters() { waiters_->SetSignalAndWakeupOne(); } -FdPoller::Impl::Impl() : watcher_(current_task::GetEventThread(), this) { +FdPoller::Impl::Impl(ev::ThreadControl control) : watcher_(control, this) { watcher_.Init(&IoWatcherCb); } @@ -182,7 +182,9 @@ bool FdPoller::Impl::IsValid() const noexcept { return state_ != State::kInvalid; } -FdPoller::FdPoller() { static_assert(std::atomic::is_always_lock_free); } +FdPoller::FdPoller(const ev::ThreadControl& control) : pimpl_(control) { + static_assert(std::atomic::is_always_lock_free); +} FdPoller::~FdPoller() = default; diff --git a/core/src/engine/io/fd_poller_test.cpp b/core/src/engine/io/fd_poller_test.cpp index 38c003091365..f7afcccdf5b4 100644 --- a/core/src/engine/io/fd_poller_test.cpp +++ b/core/src/engine/io/fd_poller_test.cpp @@ -1,6 +1,7 @@ #include #include +#include #include #include @@ -40,10 +41,10 @@ const auto kSmallWaitTime = std::chrono::milliseconds(100); UTEST(FdPoller, WaitAnyRead) { Pipe pipe; - engine::io::FdPoller poller_read; + engine::io::FdPoller poller_read{engine::current_task::GetEventThread()}; poller_read.Reset(pipe.In(), engine::io::FdPoller::Kind::kRead); - engine::io::FdPoller poller_write; + engine::io::FdPoller poller_write{engine::current_task::GetEventThread()}; poller_write.Reset(pipe.Out(), engine::io::FdPoller::Kind::kWrite); // Cannot read from an empty pipe diff --git a/core/src/engine/io/sys_linux/inotify.cpp b/core/src/engine/io/sys_linux/inotify.cpp index 750b38d21fa0..0211879c8310 100644 --- a/core/src/engine/io/sys_linux/inotify.cpp +++ b/core/src/engine/io/sys_linux/inotify.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -60,7 +61,7 @@ logging::LogHelper& operator<<(logging::LogHelper& lh, return lh; } -Inotify::Inotify() { +Inotify::Inotify() : fd_(engine::current_task::GetEventThread()) { fd_.Reset(inotify_init(), FdPoller::Kind::kRead); UASSERT(fd_.GetFd() != -1); } diff --git a/mysql/src/storages/mysql/impl/socket.cpp b/mysql/src/storages/mysql/impl/socket.cpp index 0205d0221c85..01ef3d479e00 100644 --- a/mysql/src/storages/mysql/impl/socket.cpp +++ b/mysql/src/storages/mysql/impl/socket.cpp @@ -1,9 +1,10 @@ #include -#include - +#include #include +#include + USERVER_NAMESPACE_BEGIN namespace storages::mysql::impl { @@ -48,7 +49,9 @@ int ToMySQLEvents(engine::io::FdPoller::Kind kind) { } // namespace Socket::Socket(int fd, int mysql_events) - : fd_{fd}, mysql_events_to_wait_on_{mysql_events} {} + : fd_{fd}, + poller_{engine::current_task::GetEventThread()}, + mysql_events_to_wait_on_{mysql_events} {} // Poller is Reset + Invalidated transactionally in Wait, so we don't need to // invalidate it in destructor. From 3a6f0bac6e59e3f375232caa63248d39c366481c Mon Sep 17 00:00:00 2001 From: segoon Date: Thu, 18 Jul 2024 13:50:32 +0300 Subject: [PATCH 047/629] feat docker: add clang-format to img 1b64a903655ef43a773011d772cbe79d490cc755 --- scripts/docker/base-ubuntu-22.04-ci.dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/docker/base-ubuntu-22.04-ci.dockerfile b/scripts/docker/base-ubuntu-22.04-ci.dockerfile index 8a0bdb16b071..a7bc5fcd8e15 100644 --- a/scripts/docker/base-ubuntu-22.04-ci.dockerfile +++ b/scripts/docker/base-ubuntu-22.04-ci.dockerfile @@ -18,8 +18,8 @@ RUN \ postgresql-14 \ rabbitmq-server \ redis-server \ - clang-16 lld-16 llvm-16 libclang-rt-16-dev\ - clang-14 lld-14 llvm \ + clang-16 lld-16 llvm-16 clang-format-16 libclang-rt-16-dev\ + clang-14 lld-14 llvm clang-format \ g++-11 gcc-11 \ g++-13 gcc-13 \ && \ From a1ac346375511329eb5c5133f7c2c14153cad7ea Mon Sep 17 00:00:00 2001 From: segoon Date: Thu, 18 Jul 2024 13:58:39 +0300 Subject: [PATCH 048/629] feat tracing: remove opentracing 7898e36dab7e06f6e81a998c583ad3ca56e2416b --- .mapping.json | 1 - .../tracing/static_config.yaml | 5 - .../tracing/tests/conftest.py | 75 ---------- .../tests/test_trace_headers_propagation.py | 20 +-- core/include/userver/tracing/tracer.hpp | 12 +- core/src/tracing/component.cpp | 19 +-- core/src/tracing/span.cpp | 2 - core/src/tracing/span_impl.hpp | 5 - core/src/tracing/span_opentracing.cpp | 140 ------------------ core/src/tracing/span_test.cpp | 116 +-------------- core/src/tracing/tracer.cpp | 13 +- core/src/tracing/tracing_benchmark.cpp | 7 +- 12 files changed, 18 insertions(+), 397 deletions(-) delete mode 100644 core/src/tracing/span_opentracing.cpp diff --git a/.mapping.json b/.mapping.json index 08909cad0062..cae426a5345c 100644 --- a/.mapping.json +++ b/.mapping.json @@ -1526,7 +1526,6 @@ "core/src/tracing/span.cpp":"taxi/uservices/userver/core/src/tracing/span.cpp", "core/src/tracing/span_builder.cpp":"taxi/uservices/userver/core/src/tracing/span_builder.cpp", "core/src/tracing/span_impl.hpp":"taxi/uservices/userver/core/src/tracing/span_impl.hpp", - "core/src/tracing/span_opentracing.cpp":"taxi/uservices/userver/core/src/tracing/span_opentracing.cpp", "core/src/tracing/span_test.cpp":"taxi/uservices/userver/core/src/tracing/span_test.cpp", "core/src/tracing/tag_scope.cpp":"taxi/uservices/userver/core/src/tracing/tag_scope.cpp", "core/src/tracing/tag_scope_test.cpp":"taxi/uservices/userver/core/src/tracing/tag_scope_test.cpp", diff --git a/core/functional_tests/tracing/static_config.yaml b/core/functional_tests/tracing/static_config.yaml index 8cc0ae70f55b..f3ea6d00cae5 100644 --- a/core/functional_tests/tracing/static_config.yaml +++ b/core/functional_tests/tracing/static_config.yaml @@ -26,11 +26,6 @@ components_manager: logging: fs-task-processor: fs-task-processor loggers: - opentracing: # Required for writing opentracing data into a separate file - file_path: /var/log/file/overwritten/by-testsuit - level: info - flush_level: info - overflow_behavior: block default: file_path: '@stderr' level: debug diff --git a/core/functional_tests/tracing/tests/conftest.py b/core/functional_tests/tracing/tests/conftest.py index 41df2dc71d76..fb3fbc44f1d7 100644 --- a/core/functional_tests/tracing/tests/conftest.py +++ b/core/functional_tests/tracing/tests/conftest.py @@ -1,5 +1,3 @@ -import asyncio - import pytest @@ -17,76 +15,3 @@ def _do_patch(config_yaml, config_vars): ) return _do_patch - - -@pytest.fixture(name='jaeger_logs_path', scope='session') -def _jaeger_logs_path(service_tmpdir): - return str(service_tmpdir / 'opentracing_log.txt') - - -# Overriding the default testsuite behavior -@pytest.fixture(name='userver_config_logging', scope='session') -def _userver_config_logging(userver_config_logging, jaeger_logs_path): - def _patch_config(config_yaml, config_vars): - userver_config_logging(config_yaml, config_vars) - logging = config_yaml['components_manager']['components']['logging'] - logging['loggers']['opentracing'] = { - 'file_path': jaeger_logs_path, - 'level': 'info', - 'flush_level': 'info', - 'overflow_behavior': 'block', - } - - return _patch_config - - -@pytest.fixture(scope='function') -async def assert_ids_in_file(service_client, jaeger_logs_path): - with open(jaeger_logs_path, 'w') as jaeger_file: - jaeger_file.truncate(0) - - trace_id = '' - - # Checking the capture only after the capturing was stopped, to make sure - # that the logs were flushed - async def _check_the_files(check_trace_id: str): - nonlocal trace_id - assert not trace_id, 'Fixture assert_ids_in_file was invoked twice' - trace_id = check_trace_id - - async with service_client.capture_logs() as capture: - yield _check_the_files - assert trace_id, 'Fixture assert_ids_in_file was not invoked' - - records = capture.select(trace_id=trace_id) - assert len(records) >= 1, capture.select() - - retries = 100 - required_data = { - f'\ttrace_id={trace_id}', - 'service_name=http-tracing-test', - 'duration=', - 'operation_name=external', - 'tags=[{"', - 'test-service/echo-no-body"', - '"key":"http.url"', - '"key":"http.status_code"', - '"value":"200"', - '}]', - } - - for _ in range(retries): - probable_lines = [] - with open(jaeger_logs_path, 'r') as jaeger_file: - for line in reversed(jaeger_file.read().split('\n')): - if trace_id in line: - probable_lines.append(line) - - if all(substr in line for substr in required_data): - return - - await asyncio.sleep(0.5) - assert False, ( - f'Missing substrings {required_data} in opentracing file ' - f'for trace id {trace_id}. Lines:\n {probable_lines}' - ) diff --git a/core/functional_tests/tracing/tests/test_trace_headers_propagation.py b/core/functional_tests/tracing/tests/test_trace_headers_propagation.py index 3af6012757e9..73f075561b64 100644 --- a/core/functional_tests/tracing/tests/test_trace_headers_propagation.py +++ b/core/functional_tests/tracing/tests/test_trace_headers_propagation.py @@ -70,9 +70,7 @@ async def _handler(request): assert response.status_code == 200 -async def test_b3_tracing_headers( - service_client, mockserver, assert_ids_in_file, -): +async def test_b3_tracing_headers(service_client, mockserver): @mockserver.json_handler('/test-service/echo-no-body') async def _handler(request): assert request.headers['X-YaTraceId'] == B3_HEADERS['X-B3-TraceId'] @@ -84,12 +82,9 @@ async def _handler(request): response = await service_client.get('/echo-no-body', headers=B3_HEADERS) assert _handler.times_called >= 1 assert response.status_code == 200 - await assert_ids_in_file(B3_HEADERS['X-B3-TraceId']) -async def test_otel_tracing_headers( - service_client, mockserver, assert_ids_in_file, -): +async def test_otel_tracing_headers(service_client, mockserver): @mockserver.json_handler('/test-service/echo-no-body') async def _handler(request): assert request.headers['X-YaTraceId'] == OPENTELEMETRY_TRACE_ID @@ -116,12 +111,9 @@ async def _handler(request): ) assert _handler.times_called >= 1 assert response.status_code == 200 - await assert_ids_in_file(OPENTELEMETRY_TRACE_ID) -async def test_taxi_tracing_headers( - service_client, mockserver, assert_ids_in_file, -): +async def test_taxi_tracing_headers(service_client, mockserver): @mockserver.json_handler('/test-service/echo-no-body') async def _handler(request): assert request.headers['X-YaTraceId'] == TAXI_HEADERS['X-YaTraceId'] @@ -132,7 +124,6 @@ async def _handler(request): response = await service_client.get('/echo-no-body', headers=TAXI_HEADERS) assert _handler.times_called >= 1 assert response.status_code == 200 - await assert_ids_in_file(TAXI_HEADERS['X-YaTraceId']) async def test_taxi_tracing_headers_ext(service_client, mockserver): @@ -168,9 +159,7 @@ async def _handler(request): assert response.status_code == 200 -async def test_yandex_tracing_headers( - service_client, mockserver, assert_ids_in_file, -): +async def test_yandex_tracing_headers(service_client, mockserver): @mockserver.json_handler('/test-service/echo-no-body') async def _handler(request): assert request.headers['X-RequestId'] == YANDEX_HEADERS['X-RequestId'] @@ -182,7 +171,6 @@ async def _handler(request): ) assert _handler.times_called >= 1 assert response.status_code == 200 - await assert_ids_in_file(YANDEX_HEADERS['X-RequestId']) async def test_priority_otel_tracing_headers(service_client, mockserver): diff --git a/core/include/userver/tracing/tracer.hpp b/core/include/userver/tracing/tracer.hpp index cfea6b8ad878..aa41f7ae8a48 100644 --- a/core/include/userver/tracing/tracer.hpp +++ b/core/include/userver/tracing/tracer.hpp @@ -37,24 +37,18 @@ class Tracer : public std::enable_shared_from_this { virtual void LogSpanContextTo(const Span::Impl& span, logging::impl::TagWriter writer) const = 0; - logging::LoggerPtr GetOptionalLogger() const { return optional_logger_; } - protected: - explicit Tracer(std::string_view service_name, - logging::LoggerPtr optional_logger) - : service_name_(service_name), - optional_logger_(std::move(optional_logger)) {} + explicit Tracer(std::string_view service_name) + : service_name_(service_name) {} virtual ~Tracer(); private: const std::string service_name_; - const logging::LoggerPtr optional_logger_; }; /// Make a tracer that could be set globally via tracing::Tracer::SetTracer -TracerPtr MakeTracer(std::string_view service_name, logging::LoggerPtr logger, - std::string_view tracer_type = "native"); +TracerPtr MakeTracer(std::string_view service_name); } // namespace tracing diff --git a/core/src/tracing/component.cpp b/core/src/tracing/component.cpp index b82091e2d9d0..46674e6370c6 100644 --- a/core/src/tracing/component.cpp +++ b/core/src/tracing/component.cpp @@ -13,26 +13,11 @@ namespace { constexpr std::string_view kNativeTrace = "native"; } -Tracer::Tracer(const ComponentConfig& config, const ComponentContext& context) { - auto& logging_component = context.FindComponent(); - auto opentracing_logger = logging_component.GetLoggerOptional("opentracing"); +Tracer::Tracer(const ComponentConfig& config, const ComponentContext&) { auto service_name = config["service-name"].As({}); const auto tracer_type = config["tracer"].As(kNativeTrace); if (tracer_type == kNativeTrace) { - if (service_name.empty() && opentracing_logger) { - throw std::runtime_error( - "Opentracing logger was set, but the `service-name` is empty. " - "Please provide a service name to use in OpenTracing"); - } - - if (opentracing_logger) { - LOG_INFO() << "Opentracing enabled."; - } else { - LOG_INFO() << "Opentracing logger is not registered"; - } - - tracing::Tracer::SetTracer(tracing::MakeTracer( - std::move(service_name), std::move(opentracing_logger), tracer_type)); + tracing::Tracer::SetTracer(tracing::MakeTracer(std::move(service_name))); } else { throw std::runtime_error("Tracer type is not supported: " + tracer_type); } diff --git a/core/src/tracing/span.cpp b/core/src/tracing/span.cpp index c449f3886548..042dca61824c 100644 --- a/core/src/tracing/span.cpp +++ b/core/src/tracing/span.cpp @@ -136,8 +136,6 @@ void Span::Impl::PutIntoLogger(logging::impl::TagWriter writer) && { log_extra_inheritable_.Extend(std::move(*log_extra_local_)); } writer.PutLogExtra(log_extra_inheritable_); - - LogOpenTracing(); } void Span::Impl::LogTo(logging::impl::TagWriter writer) { diff --git a/core/src/tracing/span_impl.hpp b/core/src/tracing/span_impl.hpp index 5c2807038d55..6e9c72cc73ea 100644 --- a/core/src/tracing/span_impl.hpp +++ b/core/src/tracing/span_impl.hpp @@ -72,11 +72,6 @@ class Span::Impl void AttachToCoroStack(); private: - void LogOpenTracing() const; - void DoLogOpenTracing(logging::impl::TagWriter writer) const; - static void AddOpentracingTags(formats::json::StringBuilder& output, - const logging::LogExtra& input); - static std::string GetParentIdForLogging(const Span::Impl* parent); bool ShouldLog() const; diff --git a/core/src/tracing/span_opentracing.cpp b/core/src/tracing/span_opentracing.cpp deleted file mode 100644 index 19c51755339c..000000000000 --- a/core/src/tracing/span_opentracing.cpp +++ /dev/null @@ -1,140 +0,0 @@ -#include "span_impl.hpp" - -#include - -#include -#include -#include -#include -#include - -#include - -USERVER_NAMESPACE_BEGIN - -namespace tracing { - -namespace { -namespace jaeger { - -struct OpentracingTag { - std::string_view opentracing_name; - std::string_view type; -}; - -constexpr utils::TrivialBiMap kGetOpentracingTags = [](auto selector) { - using Tag = OpentracingTag; - return selector() - .Case(kHttpStatusCode, Tag{"http.status_code", "int64"}) - .Case(kErrorFlag, Tag{"error", "bool"}) - .Case(kHttpMethod, Tag{"http.method", "string"}) - .Case(kHttpUrl, Tag{"http.url", "string"}) - - .Case(kDatabaseType, Tag{"db.type", "string"}) - .Case(kDatabaseStatement, Tag{"db.statement", "string"}) - .Case(kDatabaseInstance, Tag{"db.instance", "string"}) - .Case(kDatabaseStatementName, Tag{"db.statement_name", "string"}) - .Case(kDatabaseCollection, Tag{"db.collection", "string"}) - .Case(kDatabaseStatementDescription, - Tag{"db.query_description", "string"}) - - .Case(kPeerAddress, Tag{"peer.address", "string"}); -}; - -struct LogExtraValueVisitor { - std::string string_value; - - void operator()(const std::string& val) { string_value = val; } - - void operator()(int val) { string_value = std::to_string(val); } -}; - -void GetTagObject(formats::json::StringBuilder& builder, std::string_view key, - const logging::LogExtra::Value& value, - std::string_view type) { - const formats::json::StringBuilder::ObjectGuard guard(builder); - LogExtraValueVisitor visitor; - std::visit(visitor, value); - - builder.Key("value"); - builder.WriteString(visitor.string_value); - - builder.Key("type"); - builder.WriteString(type); - - builder.Key("key"); - builder.WriteString(key); -} - -constexpr std::string_view kOperationName = "operation_name"; -constexpr std::string_view kTraceId = "trace_id"; -constexpr std::string_view kParentId = "parent_id"; -constexpr std::string_view kSpanId = "span_id"; -constexpr std::string_view kServiceName = "service_name"; - -constexpr std::string_view kStartTime = "start_time"; -constexpr std::string_view kStartTimeMillis = "start_time_millis"; -constexpr std::string_view kDuration = "duration"; - -} // namespace jaeger -} // namespace - -void Span::Impl::LogOpenTracing() const { - if (!tracer_) { - return; - } - - auto logger = tracer_->GetOptionalLogger(); - if (logger) { - const DetachLocalSpansScope ignore_local_span; - logging::LogHelper lh(*logger, log_level_); - DoLogOpenTracing(lh.GetTagWriterAfterText({})); - } -} - -void Span::Impl::DoLogOpenTracing(logging::impl::TagWriter writer) const { - const auto steady_now = std::chrono::steady_clock::now(); - const auto duration = steady_now - start_steady_time_; - const auto duration_microseconds = - std::chrono::duration_cast(duration).count(); - auto start_time = std::chrono::duration_cast( - start_system_time_.time_since_epoch()) - .count(); - - if (tracer_) { - writer.PutTag(jaeger::kServiceName, tracer_->GetServiceName()); - } - writer.PutTag(jaeger::kTraceId, trace_id_); - writer.PutTag(jaeger::kParentId, parent_id_); - writer.PutTag(jaeger::kSpanId, span_id_); - writer.PutTag(jaeger::kStartTime, start_time); - writer.PutTag(jaeger::kStartTimeMillis, start_time / 1000); - writer.PutTag(jaeger::kDuration, duration_microseconds); - writer.PutTag(jaeger::kOperationName, name_); - - formats::json::StringBuilder tags; - { - const formats::json::StringBuilder::ArrayGuard guard(tags); - AddOpentracingTags(tags, log_extra_inheritable_); - if (log_extra_local_) { - AddOpentracingTags(tags, *log_extra_local_); - } - } - writer.PutTag("tags", tags.GetStringView()); -} - -void Span::Impl::AddOpentracingTags(formats::json::StringBuilder& output, - const logging::LogExtra& input) { - for (const auto& [key, value] : *input.extra_) { - const auto tag_it = jaeger::kGetOpentracingTags.TryFind(key); - if (tag_it) { - const auto& tag = *tag_it; - jaeger::GetTagObject(output, tag.opentracing_name, value.GetValue(), - tag.type); - } - } -} - -} // namespace tracing - -USERVER_NAMESPACE_END diff --git a/core/src/tracing/span_test.cpp b/core/src/tracing/span_test.cpp index becbda95098c..9d6b60b84c7d 100644 --- a/core/src/tracing/span_test.cpp +++ b/core/src/tracing/span_test.cpp @@ -19,57 +19,6 @@ USERVER_NAMESPACE_BEGIN class Span : public LoggingTest {}; -class OpentracingSpan : public Span { - protected: - OpentracingSpan() - : opentracing_logger_( - MakeNamedStreamLogger("openstracing", logging::Format::kTskv)) { - tracing::Tracer::SetTracer( - tracing::MakeTracer("service-test-name", opentracing_logger_.logger)); - - // Discard logs - logging::LogFlush(*opentracing_logger_.logger); - opentracing_logger_.stream.str({}); - } - - ~OpentracingSpan() override { - tracing::Tracer::SetTracer(tracing::MakeTracer({}, {})); - } - - // NOLINTNEXTLINE(readability-make-member-function-const) - void FlushOpentracing() { logging::LogFlush(*opentracing_logger_.logger); } - - static void CheckTagFormat(const formats::json::Value& tag) { - EXPECT_TRUE(tag.HasMember("key")); - EXPECT_TRUE(tag.HasMember("value")); - EXPECT_TRUE(tag.HasMember("type")); - } - - static void CheckTagTypeAndValue(const formats::json::Value& tag, - const std::string& type, - const std::string& value) { - EXPECT_EQ(type, tag["type"].As()); - EXPECT_EQ(value, tag["value"].As()); - } - - static formats::json::Value GetTagsJson(const std::string& log_output) { - auto tags_start = log_output.find("tags="); - if (tags_start == std::string::npos) return {}; - tags_start += 5; - auto tags_str = log_output.substr(tags_start); - auto const tags_end = tags_str.find(']'); - if (tags_end == std::string::npos) return {}; - return formats::json::FromString(tags_str.substr(0, tags_end + 1)); - } - - std::string GetOtStreamString() const { - return opentracing_logger_.stream.str(); - } - - private: - StringStreamLogger opentracing_logger_; -}; - UTEST_F(Span, Ctr) { { logging::LogFlush(); @@ -187,67 +136,6 @@ UTEST_F(Span, NonInheritTag) { EXPECT_THAT(GetStreamString(), Not(HasSubstr("k=v"))); } -UTEST_F(OpentracingSpan, Tags) { - { - tracing::Span span("span_name"); - span.AddTag("k", "v"); - span.AddTag("meta_code", 200); - span.AddTag("error", false); - span.AddTag("method", "POST"); - span.AddTag("db.type", "postgres"); - span.AddTag("db.statement", "SELECT * "); - span.AddTag("peer.address", "127.0.0.1:8080"); - span.AddTag("http.url", "http://example.com/example"); - } - FlushOpentracing(); - const auto log_str = GetOtStreamString(); - EXPECT_THAT(log_str, Not(HasSubstr("k=v"))); - EXPECT_THAT(log_str, HasSubstr("http.status_code")); - EXPECT_THAT(log_str, HasSubstr("error")); - EXPECT_THAT(log_str, HasSubstr("http.method")); - EXPECT_THAT(log_str, HasSubstr("db.type")); - EXPECT_THAT(log_str, HasSubstr("db.statement")); - EXPECT_THAT(log_str, HasSubstr("peer.address")); - EXPECT_THAT(log_str, HasSubstr("http.url")); -} - -UTEST_F(OpentracingSpan, FromTracerWithServiceName) { - auto tracer = tracing::MakeTracer( - "test_service", tracing::Tracer::GetTracer()->GetOptionalLogger()); - { - tracing::Span span(tracer, "span_name", nullptr, - tracing::ReferenceType::kChild); - } - FlushOpentracing(); - const auto log_str = GetOtStreamString(); - EXPECT_THAT(log_str, HasSubstr("service_name=test_service")); -} - -UTEST_F(OpentracingSpan, TagFormat) { - { - tracing::Span span("span_name"); - span.AddTag("meta_code", 200); - span.AddTag("error", false); - span.AddTag("method", "POST"); - } - FlushOpentracing(); - const auto tags = GetTagsJson(GetOtStreamString()); - EXPECT_EQ(3, tags.GetSize()); - for (const auto& tag : tags) { - CheckTagFormat(tag); - const auto key = tag["key"].As(); - if (key == "http.status_code") { - CheckTagTypeAndValue(tag, "int64", "200"); - } else if (key == "error") { - CheckTagTypeAndValue(tag, "bool", "0"); - } else if (key == "http.method") { - CheckTagTypeAndValue(tag, "string", "POST"); - } else { - FAIL() << "Got unknown key in tags: " << key; - } - } -} - UTEST_F(Span, ScopeTime) { { tracing::Span span("span_name"); @@ -366,7 +254,7 @@ UTEST_F(Span, LowerLocalLogLevel) { } UTEST_F(Span, ConstructFromTracer) { - auto tracer = tracing::MakeTracer("test_service", {}); + auto tracer = tracing::MakeTracer("test_service"); tracing::Span span(tracer, "name", nullptr, tracing::ReferenceType::kChild); span.SetLink("some_link"); @@ -560,7 +448,7 @@ UTEST_F(Span, NoLogWithSetLogLevel) { } UTEST_F(Span, ForeignSpan) { - auto tracer = tracing::MakeTracer("test_service", {}); + auto tracer = tracing::MakeTracer("test_service"); tracing::Span local_span(tracer, "local", nullptr, tracing::ReferenceType::kChild); diff --git a/core/src/tracing/tracer.cpp b/core/src/tracing/tracer.cpp index 4dea9fe5968b..b5acd7b7ec62 100644 --- a/core/src/tracing/tracer.cpp +++ b/core/src/tracing/tracer.cpp @@ -21,8 +21,7 @@ constexpr std::string_view kParentIdName = "parent_id"; class NoopTracer final : public Tracer { public: - NoopTracer(std::string_view service_name, logging::LoggerPtr logger) - : Tracer(service_name, std::move(logger)) {} + explicit NoopTracer(std::string_view service_name) : Tracer(service_name) {} void LogSpanContextTo(const Span::Impl&, logging::impl::TagWriter) const override; @@ -41,7 +40,7 @@ auto& GlobalNoLogSpans() { } auto& GlobalTracer() { - static rcu::Variable tracer(tracing::MakeTracer({}, {})); + static rcu::Variable tracer(tracing::MakeTracer({})); return tracer; } @@ -112,12 +111,8 @@ Span Tracer::CreateSpan(std::string name, const Span& parent, return Span(shared_from_this(), std::move(name), &parent, reference_type); } -tracing::TracerPtr MakeTracer(std::string_view service_name, - logging::LoggerPtr logger, - std::string_view tracer_type) { - UINVARIANT(tracer_type == "native", - "Only 'native' tracer type is available at the moment"); - return std::make_shared(service_name, std::move(logger)); +tracing::TracerPtr MakeTracer(std::string_view service_name) { + return std::make_shared(service_name); } } // namespace tracing diff --git a/core/src/tracing/tracing_benchmark.cpp b/core/src/tracing/tracing_benchmark.cpp index 54ae89fc38d8..f3d1f142f530 100644 --- a/core/src/tracing/tracing_benchmark.cpp +++ b/core/src/tracing/tracing_benchmark.cpp @@ -10,7 +10,7 @@ namespace { void tracing_noop_ctr(benchmark::State& state) { engine::RunStandalone([&] { - auto tracer = tracing::MakeTracer("test_service", {}); + auto tracer = tracing::MakeTracer("test_service"); for ([[maybe_unused]] auto _ : state) benchmark::DoNotOptimize(tracer->CreateSpanWithoutParent("name")); @@ -25,7 +25,7 @@ void tracing_happy_log(benchmark::State& state) { // TODO Null logger ignores log level and keeps kNone, this benchmark // measures nothing. Should use TpLogger instead. const logging::DefaultLoggerLevelScope level_scope{logging::Level::kInfo}; - auto tracer = tracing::MakeTracer("test_service", {}); + auto tracer = tracing::MakeTracer("test_service"); for ([[maybe_unused]] auto _ : state) benchmark::DoNotOptimize(tracer->CreateSpanWithoutParent("name")); @@ -42,9 +42,8 @@ tracing::Span GetSpanWithOpentracingHttpTags(tracing::TracerPtr tracer) { } void tracing_opentracing_ctr(benchmark::State& state) { - auto logger = logging::MakeNullLogger(); engine::RunStandalone([&] { - auto tracer = tracing::MakeTracer("test_service", logger); + auto tracer = tracing::MakeTracer("test_service"); for ([[maybe_unused]] auto _ : state) { benchmark::DoNotOptimize(GetSpanWithOpentracingHttpTags(tracer)); } From 69b9b1dcd7147f151b4128be1d3fdbdbd94348db Mon Sep 17 00:00:00 2001 From: segoon Date: Thu, 18 Jul 2024 14:05:19 +0300 Subject: [PATCH 049/629] bug logger: fix quit f026f98a5c2a79913efe78936a3f71147ea0a7d9 --- .../common_server_component_list_test.cpp | 29 +++++++++++++++++-- core/src/components/run.cpp | 7 ++++- otlp/include/userver/otlp/logs/component.hpp | 2 ++ otlp/src/otlp/logs/component.cpp | 20 +++++++++---- otlp/src/otlp/logs/logger.cpp | 5 +++- .../userver/logging/impl/mem_logger.hpp | 9 ++++++ universal/src/logging/log.cpp | 7 +++-- 7 files changed, 66 insertions(+), 13 deletions(-) diff --git a/core/src/components/common_server_component_list_test.cpp b/core/src/components/common_server_component_list_test.cpp index 94ceca5699f8..c2dce273f5a1 100644 --- a/core/src/components/common_server_component_list_test.cpp +++ b/core/src/components/common_server_component_list_test.cpp @@ -2,14 +2,14 @@ #include +#include #include #include #include // for fs::blocking::TempDirectory #include // for fs::blocking::RewriteFileContents -#include - -#include #include +#include +#include #include USERVER_NAMESPACE_BEGIN @@ -287,6 +287,29 @@ TEST_F(CommonServerComponentList, Smoke) { .Append()); } +TEST_F(CommonServerComponentList, Logger) { + auto& old_logger = logging::GetDefaultLogger(); + logging::impl::SetDefaultLoggerRef(logging::impl::MemLogger::GetMemLogger()); + + fs::blocking::RewriteFileContents( + GetConfigVarsPath(), + fmt::format(kConfigVarsTemplate, GetDumpsRoot(), + GetDynamicConfigCachePath(), "warning", "'@null'", "discard", + GetPorts().server_port, GetPorts().monitor_port)); + + components::RunOnce( + components::InMemoryConfig{std::string{kStaticConfig} + + GetConfigVarsPath()}, + components::CommonComponentList() + .AppendComponentList(components::CommonServerComponentList()) + .Append()); + + logging::SetDefaultLoggerLevel(logging::Level::kInfo); + LOG_CRITICAL() << "some text"; + + logging::impl::SetDefaultLoggerRef(old_logger); +} + TEST_F(CommonServerComponentList, TraceLogging) { fs::blocking::RewriteFileContents( GetConfigVarsPath(), diff --git a/core/src/components/run.cpp b/core/src/components/run.cpp index 1ca47a31b489..aa1ce9860630 100644 --- a/core/src/components/run.cpp +++ b/core/src/components/run.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -48,7 +49,10 @@ class LogScope final { public: LogScope() : logger_prev_{logging::GetDefaultLogger()}, - level_scope_{logging::GetDefaultLoggerLevel()} {} + level_scope_{logging::GetDefaultLoggerLevel()} { + logging::impl::SetDefaultLoggerRef( + logging::impl::MemLogger::GetMemLogger()); + } ~LogScope() { logger_prev_.ForwardTo(nullptr); @@ -180,6 +184,7 @@ ManagerConfig ParseManagerConfigAndSetupLogging( // This line enables basic logging. Any logging before would go to // MemLogger and be transferred to the logger in the cycle below. log_scope.SetLogger(default_logger); + log_scope.GetPrevLogger().ForwardTo(&*default_logger); } LOG_INFO() << "Parsed " << details; diff --git a/otlp/include/userver/otlp/logs/component.hpp b/otlp/include/userver/otlp/logs/component.hpp index e409c18f6e73..43a312610cb1 100644 --- a/otlp/include/userver/otlp/logs/component.hpp +++ b/otlp/include/userver/otlp/logs/component.hpp @@ -8,6 +8,7 @@ #include #include +#include #include USERVER_NAMESPACE_BEGIN @@ -45,6 +46,7 @@ class LoggerComponent final : public components::RawComponentBase { private: std::shared_ptr logger_; + logging::LoggerRef old_logger_; utils::statistics::Entry statistics_holder_; }; diff --git a/otlp/src/otlp/logs/component.cpp b/otlp/src/otlp/logs/component.cpp index eab03e0f4aef..d3cc55d49c3b 100644 --- a/otlp/src/otlp/logs/component.cpp +++ b/otlp/src/otlp/logs/component.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -21,7 +22,8 @@ USERVER_NAMESPACE_BEGIN namespace otlp { LoggerComponent::LoggerComponent(const components::ComponentConfig& config, - const components::ComponentContext& context) { + const components::ComponentContext& context) + : old_logger_(logging::GetDefaultLogger()) { auto& client_factory = context.FindComponent() .GetFactory(); @@ -53,11 +55,17 @@ LoggerComponent::LoggerComponent(const components::ComponentConfig& config, std::move(logger_config)); // We must init after the default logger is initialized - context.FindComponent(); + auto& logging_component = context.FindComponent(); + if (logging_component.GetLoggerOptional("default")) { + throw std::runtime_error( + "You have registered both 'otlp-logger' component and 'default' logger " + "in 'logging' component. Either disable default logger or otlp " + "logger."); + } + UASSERT(dynamic_cast(&old_logger_)); - auto& old_logger = logging::GetDefaultLogger(); logging::impl::SetDefaultLoggerRef(*logger_); - old_logger.ForwardTo(&*logger_); + old_logger_.ForwardTo(&*logger_); auto* const statistics_storage = context.FindComponentOptional(); @@ -71,8 +79,8 @@ LoggerComponent::LoggerComponent(const components::ComponentConfig& config, } LoggerComponent::~LoggerComponent() { - std::cerr << "Destroying default logger\n"; - logging::impl::SetDefaultLoggerRef(logging::GetNullLogger()); + old_logger_.ForwardTo(nullptr); + logging::impl::SetDefaultLoggerRef(old_logger_); logger_->Stop(); diff --git a/otlp/src/otlp/logs/logger.cpp b/otlp/src/otlp/logs/logger.cpp index 2ae76e5717f4..7e7fd9cad857 100644 --- a/otlp/src/otlp/logs/logger.cpp +++ b/otlp/src/otlp/logs/logger.cpp @@ -48,7 +48,10 @@ Logger::Logger( Logger::~Logger() { Stop(); } -void Logger::Stop() noexcept { sender_task_.SyncCancel(); } +void Logger::Stop() noexcept { + sender_task_.SyncCancel(); + sender_task_ = {}; +} const logging::impl::LogStatistics& Logger::GetStatistics() const { return stats_; diff --git a/universal/include/userver/logging/impl/mem_logger.hpp b/universal/include/userver/logging/impl/mem_logger.hpp index 40347c9a324d..e7f3cf9bc865 100644 --- a/universal/include/userver/logging/impl/mem_logger.hpp +++ b/universal/include/userver/logging/impl/mem_logger.hpp @@ -1,5 +1,7 @@ #pragma once +#include + #include #include #include @@ -20,6 +22,12 @@ class MemLogger final : public LoggerBase { MemLogger(const MemLogger&) = delete; MemLogger(MemLogger&&) = delete; + ~MemLogger() override { + for (const auto& data : data_) { + fputs(data.msg.c_str(), stderr); + } + } + static MemLogger& GetMemLogger() noexcept { static MemLogger logger; return logger; @@ -40,6 +48,7 @@ class MemLogger final : public LoggerBase { if (data_.size() > kMaxLogItems) return; data_.push_back({level, std::string{msg}}); } + void Flush() override {} void ForwardTo(LoggerBase* logger_to) override { diff --git a/universal/src/logging/log.cpp b/universal/src/logging/log.cpp index 85fb62746aa6..dd3c7b1c7ee4 100644 --- a/universal/src/logging/log.cpp +++ b/universal/src/logging/log.cpp @@ -5,7 +5,7 @@ #include #include #include -#include +#include #include USERVER_NAMESPACE_BEGIN @@ -14,8 +14,11 @@ namespace logging { namespace { auto& NonOwningDefaultLoggerInternal() noexcept { + // Initial logger should be Null logger as non-service utils + // may use userver's universal without logger. They should not suffer from any + // logger at all. static std::atomic default_logger_ptr{ - &impl::MemLogger::GetMemLogger()}; + &logging::GetNullLogger()}; return default_logger_ptr; } From 9a2ab68c7a9ad17dd5584684b10dd2b42dd394fb Mon Sep 17 00:00:00 2001 From: nepritula Date: Thu, 18 Jul 2024 14:13:45 +0300 Subject: [PATCH 050/629] feat docs: snippets in bytea.hpp and supported_types.hpp added MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tests: протестировано локально и в CI Relates: https://st.yandex-team.ru/ 2c9a886c98f6aecacb8e88b6ebfa00e3362b1f5e --- .../userver/storages/postgres/io/bytea.hpp | 18 +++++++++- .../storages/postgres/io/supported_types.hpp | 36 ++++++++----------- .../storages/postgres/tests/bytea_pgtest.cpp | 6 ++-- 3 files changed, 35 insertions(+), 25 deletions(-) diff --git a/postgresql/include/userver/storages/postgres/io/bytea.hpp b/postgresql/include/userver/storages/postgres/io/bytea.hpp index 86342737dfe6..f4acc232f9cf 100644 --- a/postgresql/include/userver/storages/postgres/io/bytea.hpp +++ b/postgresql/include/userver/storages/postgres/io/bytea.hpp @@ -57,14 +57,30 @@ struct ByteaRefWrapper { } // namespace detail -/// Helper function for writing binary data +// clang-format off +/// Helper function for reading binary data +/// +/// Example usage: +/// @snippet postgresql/src/storages/postgres/tests/bytea_pgtest.cpp bytea_simple +/// @snippet postgresql/src/storages/postgres/tests/bytea_pgtest.cpp bytea_string +/// @snippet postgresql/src/storages/postgres/tests/bytea_pgtest.cpp bytea_vector +// clang-format on + template detail::ByteaRefWrapper Bytea( const ByteContainer& bytes) { return {bytes}; } +// clang-format off /// Helper function for reading binary data +/// +/// Example usage: +/// @snippet postgresql/src/storages/postgres/tests/bytea_pgtest.cpp bytea_simple +/// @snippet postgresql/src/storages/postgres/tests/bytea_pgtest.cpp bytea_string +/// @snippet postgresql/src/storages/postgres/tests/bytea_pgtest.cpp bytea_vector +// clang-format on + template detail::ByteaRefWrapper Bytea(ByteContainer& bytes) { return {bytes}; diff --git a/postgresql/include/userver/storages/postgres/io/supported_types.hpp b/postgresql/include/userver/storages/postgres/io/supported_types.hpp index 894938c305ef..2d7243f4f771 100644 --- a/postgresql/include/userver/storages/postgres/io/supported_types.hpp +++ b/postgresql/include/userver/storages/postgres/io/supported_types.hpp @@ -1,5 +1,6 @@ #pragma once +// clang-format off /// @page pg_types uPg: Supported data types /// /// uPg provides data type support with a system of buffer parsers and @@ -124,27 +125,19 @@ /// @warning When reading to `std::string_view` the value MUST NOT be used after /// the PostgreSQL result set is destroyed. /// -/// @code{.cpp} -/// namespace pg = storages::postgres; -/// using namespace std::string_literals; -/// // using a "binary string" -/// std::string s = "\0\xff\x0afoobar"s; -/// std::vector tgt_bin_str; -/// // note: pg::Bytea(const T &) is used -/// trx.Execute("select $1", pg::Bytea(s)); -/// // storing a byte array: -/// std::vector bin_str{1, 2, 3, 4, 5, 6, 7, 8, 9}; -/// trx.Execute("INSERT INTO mytable (data) VALUES ($1)", pg::Bytea(bin_str)); -/// // note: pg::Bytea(const T &) is used -/// @endcode -/// -/// To read data from bytea field: -/// @code{.cpp} -/// // SQL: data BYTEA {NOT} NULL -/// auto res = trx.Execute("SELECT id, data FROM mytable WHERE id = $1", id); -/// std::vector tgt_bin_str; const auto& row = res.Front(); -/// row["data"].To(pg::Bytea(tgt_bin_str)); // note: pg::Bytea(T &) is used -/// @endcode +/// +/// +/// +/// Bytea() is a helper function for reading and writing binary data from/to a database. +/// +/// Example usage of Bytea(): +/// @snippet postgresql/src/storages/postgres/tests/bytea_pgtest.cpp bytea_simple +/// @snippet postgresql/src/storages/postgres/tests/bytea_pgtest.cpp bytea_string +/// @snippet postgresql/src/storages/postgres/tests/bytea_pgtest.cpp bytea_vector +/// +/// +/// +/// /// /// @par Network types /// @@ -173,6 +166,7 @@ /// @htmlonly
@endhtmlonly /// ⇦ @ref pg_process_results | @ref pg_user_row_types ⇨ /// @htmlonly
@endhtmlonly +// clang-format on //@{ /** @name Traits etc */ diff --git a/postgresql/src/storages/postgres/tests/bytea_pgtest.cpp b/postgresql/src/storages/postgres/tests/bytea_pgtest.cpp index a397c2aa8ca1..9ba04a9f88e4 100644 --- a/postgresql/src/storages/postgres/tests/bytea_pgtest.cpp +++ b/postgresql/src/storages/postgres/tests/bytea_pgtest.cpp @@ -136,7 +136,7 @@ UTEST_P(PostgreConnection, ByteaRoundTripTypes) { { /// [bytea_string] - // sending a "binary string" + // sending a binary string std::string s = "\0\xff\x0afoobar"s; pg::ResultSet res = GetConn()->Execute("select $1", pg::Bytea(s)); @@ -149,11 +149,11 @@ UTEST_P(PostgreConnection, ByteaRoundTripTypes) { { /// [bytea_vector] - // storing a byte array: + // storing a byte vector: std::vector bin_str{1, 2, 3, 4, 5, 6, 7, 8, 9}; auto res = GetConn()->Execute("select $1", pg::Bytea(bin_str)); - // reading a binary string + // reading a byte vector std::vector received; res[0][0].To(pg::Bytea(received)); EXPECT_EQ(received, bin_str); From 18afc2aa0f67fe04593af7ee569c49c5b4257eb8 Mon Sep 17 00:00:00 2001 From: segoon Date: Thu, 18 Jul 2024 15:45:22 +0300 Subject: [PATCH 051/629] feat otlp: handle special characters 38d986671360f1cc4167874ded8cc31068e83808 --- core/src/tracing/span.cpp | 1 + otlp/src/otlp/logs/logger.cpp | 204 +++++++----------- otlp/src/otlp/logs/logger.hpp | 8 +- .../userver/logging/impl/logger_base.hpp | 2 + .../include/userver/logging/log_helper.hpp | 2 + universal/src/logging/impl/logger_base.cpp | 2 + universal/src/logging/log_helper.cpp | 2 + universal/src/logging/log_helper_impl.cpp | 7 +- universal/src/logging/log_helper_impl.hpp | 3 + 9 files changed, 96 insertions(+), 135 deletions(-) diff --git a/core/src/tracing/span.cpp b/core/src/tracing/span.cpp index 042dca61824c..990d81ffbab0 100644 --- a/core/src/tracing/span.cpp +++ b/core/src/tracing/span.cpp @@ -107,6 +107,7 @@ Span::Impl::~Impl() { const DetachLocalSpansScope ignore_local_span; logging::LogHelper lh{logging::GetDefaultLogger(), log_level_, source_location_}; + lh.MarkAsTrace(logging::LogHelper::InternalTag{}); std::move(*this).PutIntoLogger(lh.GetTagWriterAfterText({})); } } diff --git a/otlp/src/otlp/logs/logger.cpp b/otlp/src/otlp/logs/logger.cpp index 7e7fd9cad857..d9a56019dd03 100644 --- a/otlp/src/otlp/logs/logger.cpp +++ b/otlp/src/otlp/logs/logger.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -21,8 +22,6 @@ constexpr std::string_view kTelemetrySdkLanguage = "telemetry.sdk.language"; constexpr std::string_view kTelemetrySdkName = "telemetry.sdk.name"; constexpr std::string_view kServiceName = "service.name"; -constexpr std::string_view kStopwatchNameEq = "stopwatch_name="; - const std::string kTimestampFormat = "%Y-%m-%dT%H:%M:%E*S"; } // namespace @@ -57,24 +56,6 @@ const logging::impl::LogStatistics& Logger::GetStatistics() const { return stats_; } -void Logger::Log(logging::Level level, std::string_view msg) { - // Trim trailing \n - if (msg.size() > 1 && msg[msg.size() - 1] == '\n') { - msg = msg.substr(0, msg.size() - 1); - } - - std::vector key_values = - utils::text::SplitIntoStringViewVector(msg, "\t"); - - ++stats_.by_level[static_cast(level)]; - - if (IsTracingEntry(key_values)) { - HandleTracing(key_values); - } else { - HandleLog(key_values); - } -} - void Logger::PrependCommonTags(logging::impl::TagWriter writer) const { logging::impl::default_::PrependCommonTags(writer); } @@ -83,133 +64,100 @@ bool Logger::DoShouldLog(logging::Level level) const noexcept { return logging::impl::default_::DoShouldLog(level); } -bool Logger::IsTracingEntry( - const std::vector& key_values) const { - for (const auto& key_value : key_values) { - if (key_value.substr(0, kStopwatchNameEq.size()) == kStopwatchNameEq) - return true; - } - return false; -} +void Logger::Log(logging::Level level, std::string_view msg) { + utils::encoding::TskvParser parser{msg}; -void Logger::HandleLog(const std::vector& key_values) { - ::opentelemetry::proto::logs::v1::LogRecord log_records; - std::string text; - std::string trace_id; - std::string span_id; - std::string level; + ::opentelemetry::proto::logs::v1::LogRecord log_record; std::chrono::system_clock::time_point timestamp; - for (const auto key_value : key_values) { - auto eq_pos = key_value.find('='); - if (eq_pos == std::string::npos) continue; - - auto key = key_value.substr(0, eq_pos); - auto value = key_value.substr(eq_pos + 1); - - if (key == "text") { - text = std::string{value}; - continue; - } - if (key == "trace_id") { - trace_id = utils::encoding::FromHex(value); - continue; - } - if (key == "span_id") { - span_id = utils::encoding::FromHex(value); - continue; - } - if (key == "timestamp") { - timestamp = utils::datetime::Stringtime(std::string{value}, - utils::datetime::kDefaultTimezone, - kTimestampFormat); - continue; - } - if (key == "level") { - level = value; - continue; - } - - auto attributes = log_records.add_attributes(); - attributes->set_key(std::string{MapAttribute(key)}); - attributes->mutable_value()->set_string_value(std::string{value}); - } + ++stats_.by_level[static_cast(level)]; - log_records.set_severity_text(grpc::string(level)); - log_records.mutable_body()->set_string_value(grpc::string(text)); - log_records.set_trace_id(std::move(trace_id)); - log_records.set_span_id(std::move(span_id)); + [[maybe_unused]] auto parse_ok = utils::encoding::TskvReadRecord( + parser, [&](std::string_view key, std::string_view value) { + if (key == "text") { + log_record.mutable_body()->set_string_value( + grpc::string(std::string{value})); + return true; + } + if (key == "trace_id") { + log_record.set_trace_id(utils::encoding::FromHex(value)); + return true; + } + if (key == "span_id") { + log_record.set_span_id(utils::encoding::FromHex(value)); + return true; + } + if (key == "timestamp") { + timestamp = utils::datetime::Stringtime( + std::string{value}, utils::datetime::kDefaultTimezone, + kTimestampFormat); + return true; + } + if (key == "level") { + log_record.set_severity_text(grpc::string(std::string{value})); + return true; + } + + auto attributes = log_record.add_attributes(); + attributes->set_key(std::string{MapAttribute(key)}); + attributes->mutable_value()->set_string_value(std::string{value}); + return true; + }); auto nanoseconds = std::chrono::duration_cast( timestamp.time_since_epoch()); - log_records.set_time_unix_nano(nanoseconds.count()); + log_record.set_time_unix_nano(nanoseconds.count()); // Drop a log if overflown - auto ok = queue_producer_.PushNoblock(std::move(log_records)); + auto ok = queue_producer_.PushNoblock(std::move(log_record)); if (!ok) { ++stats_.dropped; } } -void Logger::HandleTracing(const std::vector& key_values) { +void Logger::Trace(logging::Level, std::string_view msg) { + utils::encoding::TskvParser parser{msg}; + ::opentelemetry::proto::trace::v1::Span span; - std::string trace_id; - std::string span_id; - std::string parent_span_id; - std::string_view level; - std::string_view stopwatch_name; std::string start_timestamp; std::string total_time; - for (const auto key_value : key_values) { - auto eq_pos = key_value.find('='); - if (eq_pos == std::string::npos) continue; - - auto key = key_value.substr(0, eq_pos); - auto value = key_value.substr(eq_pos + 1); - - if (key == "trace_id") { - trace_id = utils::encoding::FromHex(value); - continue; - } - if (key == "span_id") { - span_id = utils::encoding::FromHex(value); - continue; - } - if (key == "parent_span_id") { - parent_span_id = utils::encoding::FromHex(value); - continue; - } - if (key == "stopwatch_name") { - stopwatch_name = value; - continue; - } - if (key == "total_time") { - total_time = value; - continue; - } - if (key == "start_timestamp") { - start_timestamp = value; - continue; - } - if (key == "level") { - level = value; - continue; - } - if (key == "timestamp" || key == "text") { - continue; - } - - auto attributes = span.add_attributes(); - attributes->set_key(std::string{MapAttribute(key)}); - attributes->mutable_value()->set_string_value(std::string{value}); - } - - span.set_trace_id(std::string(trace_id)); - span.set_span_id(std::string(span_id)); - span.set_parent_span_id(std::string(parent_span_id)); - span.set_name(std::string(stopwatch_name)); + [[maybe_unused]] auto parse_ok = utils::encoding::TskvReadRecord( + parser, [&](std::string_view key, std::string_view value) { + if (key == "trace_id") { + span.set_trace_id(utils::encoding::FromHex(value)); + return true; + } + if (key == "span_id") { + span.set_span_id(utils::encoding::FromHex(value)); + return true; + } + if (key == "parent_span_id") { + span.set_parent_span_id(utils::encoding::FromHex(value)); + return true; + } + if (key == "stopwatch_name") { + span.set_name(std::string(value)); + return true; + } + if (key == "total_time") { + total_time = value; + return true; + } + if (key == "start_timestamp") { + start_timestamp = value; + return true; + } + if (key == "timestamp" || key == "text") { + return true; + } + + auto attributes = span.add_attributes(); + attributes->set_key(std::string{MapAttribute(key)}); + attributes->mutable_value()->set_string_value(std::string{value}); + return true; + }); auto start_timestamp_double = std::stod(start_timestamp); span.set_start_time_unix_nano(start_timestamp_double * 1'000'000'000); diff --git a/otlp/src/otlp/logs/logger.hpp b/otlp/src/otlp/logs/logger.hpp index 6bcd020ad872..0fc3c628797b 100644 --- a/otlp/src/otlp/logs/logger.hpp +++ b/otlp/src/otlp/logs/logger.hpp @@ -39,6 +39,8 @@ class Logger final : public logging::impl::LoggerBase { void Log(logging::Level level, std::string_view msg) override; + void Trace(logging::Level level, std::string_view msg) override; + void PrependCommonTags(logging::impl::TagWriter writer) const override; void Stop() noexcept; @@ -56,12 +58,6 @@ class Logger final : public logging::impl::LoggerBase { void SendingLoop(Queue::Consumer& consumer, LogClient& log_client, TraceClient& trace_client); - bool IsTracingEntry(const std::vector& key_values) const; - - void HandleTracing(const std::vector& key_values); - - void HandleLog(const std::vector& key_values); - void FillAttributes(::opentelemetry::proto::resource::v1::Resource& resource); void DoLog( diff --git a/universal/include/userver/logging/impl/logger_base.hpp b/universal/include/userver/logging/impl/logger_base.hpp index 1e200a383bae..499a6d2435e3 100644 --- a/universal/include/userver/logging/impl/logger_base.hpp +++ b/universal/include/userver/logging/impl/logger_base.hpp @@ -26,6 +26,8 @@ class LoggerBase { virtual void Log(Level level, std::string_view msg) = 0; + virtual void Trace(Level level, std::string_view msg); + virtual void Flush(); virtual void PrependCommonTags(TagWriter writer) const; diff --git a/universal/include/userver/logging/log_helper.hpp b/universal/include/userver/logging/log_helper.hpp index 54fdc0ddd9f9..0d5c52f5145c 100644 --- a/universal/include/userver/logging/log_helper.hpp +++ b/universal/include/userver/logging/log_helper.hpp @@ -162,6 +162,8 @@ class LogHelper final { // For internal use only! impl::TagWriter GetTagWriterAfterText(InternalTag); + + void MarkAsTrace(InternalTag); /// @endcond private: diff --git a/universal/src/logging/impl/logger_base.cpp b/universal/src/logging/impl/logger_base.cpp index 7febd1508635..28c503f71f45 100644 --- a/universal/src/logging/impl/logger_base.cpp +++ b/universal/src/logging/impl/logger_base.cpp @@ -10,6 +10,8 @@ LoggerBase::LoggerBase(Format format) noexcept : format_(format) {} LoggerBase::~LoggerBase() = default; +void LoggerBase::Trace(Level level, std::string_view msg) { Log(level, msg); } + void LoggerBase::Flush() {} void LoggerBase::PrependCommonTags(TagWriter /*writer*/) const {} diff --git a/universal/src/logging/log_helper.cpp b/universal/src/logging/log_helper.cpp index 02f436358694..0da0c8bb0bf8 100644 --- a/universal/src/logging/log_helper.cpp +++ b/universal/src/logging/log_helper.cpp @@ -222,6 +222,8 @@ impl::TagWriter LogHelper::GetTagWriterAfterText(InternalTag) { return GetTagWriter(); } +void LogHelper::MarkAsTrace(InternalTag) { pimpl_->MarkAsTrace(); } + LogHelper& LogHelper::operator<<(char value) noexcept { try { Put(value); diff --git a/universal/src/logging/log_helper_impl.cpp b/universal/src/logging/log_helper_impl.cpp index 2c3bb30e91fa..2dd0a70cd062 100644 --- a/universal/src/logging/log_helper_impl.cpp +++ b/universal/src/logging/log_helper_impl.cpp @@ -181,6 +181,8 @@ void LogHelper::Impl::MarkValueEnd() noexcept { UASSERT(std::exchange(is_within_value_, false)); } +void LogHelper::Impl::MarkAsTrace() noexcept { is_trace_ = true; } + void LogHelper::Impl::StartText() { PutRawKey("text"); initial_length_ = msg_.size(); @@ -201,7 +203,10 @@ void LogHelper::Impl::LogTheMessage() const { UASSERT(logger_); const std::string_view message(msg_.data(), msg_.size()); - logger_->Log(level_, message); + if (is_trace_) + logger_->Trace(level_, message); + else + logger_->Log(level_, message); } void LogHelper::Impl::MarkAsBroken() noexcept { logger_ = nullptr; } diff --git a/universal/src/logging/log_helper_impl.hpp b/universal/src/logging/log_helper_impl.hpp index feeb25bb4311..fa0bdc1a6980 100644 --- a/universal/src/logging/log_helper_impl.hpp +++ b/universal/src/logging/log_helper_impl.hpp @@ -52,6 +52,8 @@ class LogHelper::Impl final { bool IsBroken() const noexcept; + void MarkAsTrace() noexcept; + private: class BufferStd final : public std::streambuf { public: @@ -86,6 +88,7 @@ class LogHelper::Impl final { LogExtra extra_; std::size_t initial_length_{0}; bool is_within_value_{false}; + bool is_trace_{false}; std::optional> debug_tag_keys_; }; From 822fb045555686ec04a5c2b48476fd01e62f5918 Mon Sep 17 00:00:00 2001 From: segoon Date: Thu, 18 Jul 2024 19:04:12 +0300 Subject: [PATCH 052/629] feat chaotic: add golden tests 997a03379ab9dd89ec8f3e1305f71de2fd48c8bb --- .mapping.json | 6 +++ chaotic/CMakeLists.txt | 1 + chaotic/golden_tests/CMakeLists.txt | 40 ++++++++++++++ chaotic/golden_tests/output/schemas/int.cpp | 54 +++++++++++++++++++ chaotic/golden_tests/output/schemas/int.hpp | 38 +++++++++++++ .../golden_tests/output/schemas/int_fwd.hpp | 7 +++ .../output/schemas/int_parsers.ipp | 30 +++++++++++ chaotic/golden_tests/schemas/int.yaml | 8 +++ 8 files changed, 184 insertions(+) create mode 100644 chaotic/golden_tests/CMakeLists.txt create mode 100644 chaotic/golden_tests/output/schemas/int.cpp create mode 100644 chaotic/golden_tests/output/schemas/int.hpp create mode 100644 chaotic/golden_tests/output/schemas/int_fwd.hpp create mode 100644 chaotic/golden_tests/output/schemas/int_parsers.ipp create mode 100644 chaotic/golden_tests/schemas/int.yaml diff --git a/.mapping.json b/.mapping.json index cae426a5345c..588505f65886 100644 --- a/.mapping.json +++ b/.mapping.json @@ -50,6 +50,12 @@ "chaotic/chaotic/front/ref_resolver.py":"taxi/uservices/userver/chaotic/chaotic/front/ref_resolver.py", "chaotic/chaotic/front/types.py":"taxi/uservices/userver/chaotic/chaotic/front/types.py", "chaotic/chaotic/main.py":"taxi/uservices/userver/chaotic/chaotic/main.py", + "chaotic/golden_tests/CMakeLists.txt":"taxi/uservices/userver/chaotic/golden_tests/CMakeLists.txt", + "chaotic/golden_tests/output/schemas/int.cpp":"taxi/uservices/userver/chaotic/golden_tests/output/schemas/int.cpp", + "chaotic/golden_tests/output/schemas/int.hpp":"taxi/uservices/userver/chaotic/golden_tests/output/schemas/int.hpp", + "chaotic/golden_tests/output/schemas/int_fwd.hpp":"taxi/uservices/userver/chaotic/golden_tests/output/schemas/int_fwd.hpp", + "chaotic/golden_tests/output/schemas/int_parsers.ipp":"taxi/uservices/userver/chaotic/golden_tests/output/schemas/int_parsers.ipp", + "chaotic/golden_tests/schemas/int.yaml":"taxi/uservices/userver/chaotic/golden_tests/schemas/int.yaml", "chaotic/include/userver/chaotic/array.hpp":"taxi/uservices/userver/chaotic/include/userver/chaotic/array.hpp", "chaotic/include/userver/chaotic/convert.hpp":"taxi/uservices/userver/chaotic/include/userver/chaotic/convert.hpp", "chaotic/include/userver/chaotic/convert/to.hpp":"taxi/uservices/userver/chaotic/include/userver/chaotic/convert/to.hpp", diff --git a/chaotic/CMakeLists.txt b/chaotic/CMakeLists.txt index 8f0104222d83..09845388bbe5 100644 --- a/chaotic/CMakeLists.txt +++ b/chaotic/CMakeLists.txt @@ -33,6 +33,7 @@ if(USERVER_IS_THE_ROOT_PROJECT) ) add_subdirectory(integration_tests) + add_subdirectory(golden_tests) endif() _userver_directory_install(COMPONENT chaotic diff --git a/chaotic/golden_tests/CMakeLists.txt b/chaotic/golden_tests/CMakeLists.txt new file mode 100644 index 000000000000..d99c2193300e --- /dev/null +++ b/chaotic/golden_tests/CMakeLists.txt @@ -0,0 +1,40 @@ +project(chaotic-golden-tests) + +include(GoogleTest) +include(ChaoticGen) + +file(GLOB_RECURSE SCHEMAS ${CMAKE_CURRENT_SOURCE_DIR}/schemas/*.yaml) +userver_target_generate_chaotic(${PROJECT_NAME}-chgen + ARGS + -n "/components/schemas/([^/]*)/=ns::{0}" + -f "(.*)={0}" + -I ${CMAKE_CURRENT_SOURCE_DIR}/include + -I ${CMAKE_CURRENT_SOURCE_DIR}/../include + --parse-extra-formats + --generate-serializers + OUTPUT_DIR + ${CMAKE_CURRENT_BINARY_DIR}/src + SCHEMAS + ${SCHEMAS} + RELATIVE_TO + ${CMAKE_CURRENT_SOURCE_DIR} +) + +add_test( + NAME + chaotic-golden + COMMAND + # Diff returns 0 if files are the same, 1 if they differ + diff -uNrpB ${CMAKE_CURRENT_SOURCE_DIR}/output + ${CMAKE_CURRENT_BINARY_DIR}/src +) +add_custom_target( + update-golden-tests + DEPENDS + ${PROJECT_NAME}-chgen + COMMAND + ${CMAKE_COMMAND} -E copy_directory + ${CMAKE_CURRENT_BINARY_DIR}/src + ${CMAKE_CURRENT_SOURCE_DIR}/output +) + diff --git a/chaotic/golden_tests/output/schemas/int.cpp b/chaotic/golden_tests/output/schemas/int.cpp new file mode 100644 index 000000000000..07df11fa3e48 --- /dev/null +++ b/chaotic/golden_tests/output/schemas/int.cpp @@ -0,0 +1,54 @@ +#include "int.hpp" + +#include + +#include +#include +#include +#include +#include + +#include "int_parsers.ipp" + +namespace ns { + +bool operator==(const ns::Int& lhs, const ns::Int& rhs) { + return lhs.foo == rhs.foo && true; +} + +USERVER_NAMESPACE::logging::LogHelper& operator<<( + USERVER_NAMESPACE::logging::LogHelper& lh, const ns::Int& value) { + return lh << ToString(USERVER_NAMESPACE::formats::json::ValueBuilder(value) + .ExtractValue()); +} + +Int Parse(USERVER_NAMESPACE::formats::json::Value json, + USERVER_NAMESPACE::formats::parse::To to) { + return Parse(json, to); +} + +Int Parse(USERVER_NAMESPACE::formats::yaml::Value json, + USERVER_NAMESPACE::formats::parse::To to) { + return Parse(json, to); +} + +Int Parse(USERVER_NAMESPACE::yaml_config::Value json, + USERVER_NAMESPACE::formats::parse::To to) { + return Parse(json, to); +} + +USERVER_NAMESPACE::formats::json::Value Serialize( + [[maybe_unused]] const ns::Int& value, + USERVER_NAMESPACE::formats::serialize::To< + USERVER_NAMESPACE::formats::json::Value>) { + USERVER_NAMESPACE::formats::json::ValueBuilder vb = + USERVER_NAMESPACE::formats::common::Type::kObject; + + if (value.foo) { + vb["foo"] = USERVER_NAMESPACE::chaotic::Primitive{value.foo.value()}; + } + + return vb.ExtractValue(); +} + +} // namespace ns diff --git a/chaotic/golden_tests/output/schemas/int.hpp b/chaotic/golden_tests/output/schemas/int.hpp new file mode 100644 index 000000000000..5f2a376e97a2 --- /dev/null +++ b/chaotic/golden_tests/output/schemas/int.hpp @@ -0,0 +1,38 @@ +#pragma once + +#include "int_fwd.hpp" + +#include +#include +#include + +#include + +namespace ns { +namespace impl {} +} // namespace ns +namespace ns { + +struct Int { + std::optional foo{}; +}; + +bool operator==(const ns::Int& lhs, const ns::Int& rhs); + +USERVER_NAMESPACE::logging::LogHelper& operator<<( + USERVER_NAMESPACE::logging::LogHelper& lh, const ns::Int& value); + +Int Parse(USERVER_NAMESPACE::formats::json::Value json, + USERVER_NAMESPACE::formats::parse::To); + +Int Parse(USERVER_NAMESPACE::formats::yaml::Value json, + USERVER_NAMESPACE::formats::parse::To); + +Int Parse(USERVER_NAMESPACE::yaml_config::Value json, + USERVER_NAMESPACE::formats::parse::To); + +USERVER_NAMESPACE::formats::json::Value Serialize( + const ns::Int& value, USERVER_NAMESPACE::formats::serialize::To< + USERVER_NAMESPACE::formats::json::Value>); + +} // namespace ns diff --git a/chaotic/golden_tests/output/schemas/int_fwd.hpp b/chaotic/golden_tests/output/schemas/int_fwd.hpp new file mode 100644 index 000000000000..133c60ca6ca4 --- /dev/null +++ b/chaotic/golden_tests/output/schemas/int_fwd.hpp @@ -0,0 +1,7 @@ +#pragma once + +namespace ns { + +struct Int; + +} diff --git a/chaotic/golden_tests/output/schemas/int_parsers.ipp b/chaotic/golden_tests/output/schemas/int_parsers.ipp new file mode 100644 index 000000000000..a403794d6372 --- /dev/null +++ b/chaotic/golden_tests/output/schemas/int_parsers.ipp @@ -0,0 +1,30 @@ +#pragma once + +#include "int.hpp" + +namespace ns { + +static constexpr USERVER_NAMESPACE::utils::TrivialSet kns__Int_PropertiesNames = + [](auto selector) { + return selector().template Type().Case("foo"); + }; + +template +ns::Int Parse(Value value, USERVER_NAMESPACE::formats::parse::To) { + value.CheckNotMissing(); + value.CheckObjectOrNull(); + + ns::Int res; + + res.foo = + value["foo"] + .template As< + std::optional>>(); + + USERVER_NAMESPACE::chaotic::ValidateNoAdditionalProperties( + value, kns__Int_PropertiesNames); + + return res; +} + +} // namespace ns diff --git a/chaotic/golden_tests/schemas/int.yaml b/chaotic/golden_tests/schemas/int.yaml new file mode 100644 index 000000000000..551cf5759f58 --- /dev/null +++ b/chaotic/golden_tests/schemas/int.yaml @@ -0,0 +1,8 @@ +components: + schemas: + Int: + type: object + additionalProperties: false + properties: + foo: + type: integer From c688be6858e209980b46ee454834b4c61fe11e8d Mon Sep 17 00:00:00 2001 From: nepritula Date: Thu, 18 Jul 2024 21:07:23 +0300 Subject: [PATCH 053/629] feat docs: benchmark run results added to framework_comparison.md && new image added MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tests: протестировано локально и в CI Relates: https://st.yandex-team.ru/ 73a48d43a5b0f2006ff322129cad9f3360fb3515 --- .mapping.json | 1 + scripts/docs/doxygen.conf | 1 + scripts/docs/en/userver/framework_comparison.md | 7 +++++++ scripts/docs/img/top15.png | Bin 0 -> 7377 bytes 4 files changed, 9 insertions(+) create mode 100644 scripts/docs/img/top15.png diff --git a/.mapping.json b/.mapping.json index 588505f65886..e0af039c72bf 100644 --- a/.mapping.json +++ b/.mapping.json @@ -3249,6 +3249,7 @@ "scripts/docs/img/telegram_logo.svg":"taxi/uservices/userver/scripts/docs/img/telegram_logo.svg", "scripts/docs/img/title.svg":"taxi/uservices/userver/scripts/docs/img/title.svg", "scripts/docs/img/tools.svg":"taxi/uservices/userver/scripts/docs/img/tools.svg", + "scripts/docs/img/top15.png":"taxi/uservices/userver/scripts/docs/img/top15.png", "scripts/docs/img/toy_1.jpeg":"taxi/uservices/userver/scripts/docs/img/toy_1.jpeg", "scripts/docs/img/toy_2.jpeg":"taxi/uservices/userver/scripts/docs/img/toy_2.jpeg", "scripts/docs/img/uber.svg":"taxi/uservices/userver/scripts/docs/img/uber.svg", diff --git a/scripts/docs/doxygen.conf b/scripts/docs/doxygen.conf index 46eaf3f54c30..cd35fa1db6af 100644 --- a/scripts/docs/doxygen.conf +++ b/scripts/docs/doxygen.conf @@ -1512,6 +1512,7 @@ HTML_EXTRA_FILES = scripts/docs/doxygen-awesome-css/doxygen-awesome-darkmo scripts/docs/img/telegram_logo.svg \ scripts/docs/img/title.svg \ scripts/docs/img/tools.svg \ + scripts/docs/img/top15.png \ scripts/docs/img/uber.svg \ scripts/docs/img/yago.svg \ scripts/docs/highlight.js/highlight.min.js \ diff --git a/scripts/docs/en/userver/framework_comparison.md b/scripts/docs/en/userver/framework_comparison.md index 9602af0d1a4b..f2895decbe2c 100644 --- a/scripts/docs/en/userver/framework_comparison.md +++ b/scripts/docs/en/userver/framework_comparison.md @@ -80,12 +80,19 @@ use ❌ and ❓ respectively. [poco-streams]: https://pocoproject.org/slides/100-Streams.pdf [poco-db]: https://docs.pocoproject.org/current/Poco.Data.html +The table above shows the well-developed functionality of userver. Additionally userver is highly effective which is proven by the results of the Techempower benchmark run. Userver comes ahead of its competitors from the table. @ref bnchrs "[3]", see [[↗]][techempower-run] + +![top15](scripts/docs/img/top15.png) + +[techempower-run]: https://www.techempower.com/benchmarks/#section=test&runid=3c2e9871-9c2a-4ff3-bc31-620f65da4e74&hw=ph&test=composite + @anchor fcmp1 [1]: "Dynamic Configs" stands for any out-of-the-box functionality that allows to change behavior of the service without downtime and restart. @anchor fcmp2 [2]: Functional Testing includes DB startup and initialization; mocks for other microservices; testpoints functionality. +@anchor bnchrs [3]: Techempower run № 3c2e9871-9c2a-4ff3-bc31-620f65da4e74. ---------- diff --git a/scripts/docs/img/top15.png b/scripts/docs/img/top15.png new file mode 100644 index 0000000000000000000000000000000000000000..40229b94193ac9b62bea8c9fc817b6b0d333aa37 GIT binary patch literal 7377 zcmV;?94_ODP)iP(lH2aT#&d3F&H4YU@-VAHYmGpuNo>*EMD8%MJQgZh?VXt zTXuyPU3qa=E~rqvAcDdRx{V792vVeKnYxADy6xTm6Zg%TX-hkuVhb(X!bv8iJFcjq_jD z_08C7=0tn7uvi4PB7xwNdcsXco_K3M4fp#6!w&t8d}gBSy1evCb?KRTI?@oP~e9A$mHOWp%orhAn9v^Z8cg04-- zDOHbyBzs%8H!i0jKWPIC&6Hg^z>5B?Y1)>28hI+3d3lsDjNTFN0$ri3*t8>c1LTQ% zq)Dmyf~+`JB%9w%wz#EiXr+xZXJZuso9$cj^I!96_^`Psxm#zFQa@xvOT(1~q=|{<;a{ zka7o$n_oBZme7yh`XUa^tp&N#BaIJF5Hy4SSH|Xd-IW{*kB}dE&L5^e$oEeoZ2BEU$P#68Qpbw5IH>}v;a|WX~^wM z_g8~Aq~f|k`&&m>-Xb3;PeQlLpM#0{@#S4Cge|yhBn+dS?V54DksZF>ZE5xBC%9dQG4$>!wSw@U9q^DQyfBuE8|a|C6@=a%1}02-Zss63R)M#B-5#ZKB=et(KuzR{5r!^)hLV%tXm zP&r0#Q7iUvl*Qd4_>P?TA#x9-U5g>3^kHLgL?AHBx8S%on4OFffXG)}eS*j#a`>OL zl|Cy$l|z+7l|z-AEmr3*LM{5cwD7~7-jKRayh9JU zgLm=YKc;Q|)%t9U0B7gbzg~eiDF&pl>jsl2pDpB1Cd8x{dLc+qrkTnQ`$CnY(bmJYMvV`O^(750Cq|(*% z@XaSW$k(Bo9DYYm27-p%G# z>`aeIDIG8^TsDLMw}D4K*(U#oxTH)b`Lg8n$)iV$OG>_ZqIV%-^|W@Y4A;0v(6MND zM!o>=+2^-K2Q!j4aXFi~IEw7PZZOZNvfZcc1(BOyH|&lIL$Z`w*O9vqFZf#mkK8Fn zOIJA+P-p5NTB!eAx+Nn0TrauyHq_)q@V++zl}B-ly1ZApq4^k>^SOiYMm@9KZRsRsSO&U$>|JrE9K) z8h*bcUracBD(sFN^77v*+9qLo42H#UlX(S#8@j}V+chcAcdV~=t)eg<$@C@Nl#DF zwVpR_G78t^dD{FF%HbAydDBz3^ZKXrQ0vy;$Qcbir`7VaH#BZWP>zUu;vKHZbL_LE z7tKeaKX$SK3N=Vex^}mV^*3@xLr=dT*R=>XqB&RX#+qZbIjjUpkCB*A%MWagjzrK% zc2XW`_2**OTKTMgKPwyw`x}CU14B>J;?1^X8&d6$Uc0LXxQZF4A4rqd?@;`-@>g?y zM^1)}=KD6?RV5Uj6pBO_>N-XlV(O*JAD3=LQ_HFPJXE=zm8l#uXm91QqjQy=6Gmc8 zed-g^_YEMQw`&Cw@u>3Ax&v%hj`Hc0`cxu2=c=2}pPNMdE4aU;w`D<21#aez=C=G~ z#XWa(ZZDA&!?ih)lm~CW1pX!^gxrFUF!nKjRY|CWT$2(L?38y zE&zWCjO97$G9ot^i7B=G41>F)x5y1|?|i|I{;X-dy)c$bdg@NpW`-lIjdtVJ4eZ#X zExsW)7(0s-!v4kC?@w)nj2d5`C4a>$eLEwzEEwcG}FH8KxfRkcIkdzawWb^qe4_xkX}W>6IV) zm@N2xKyDaSWTMDgyOp{@TX%pRdw`tF4R6PLx}P$7ceWSCvVi0I^n`ZWV0`?YR8mix zb?W=eQdz0x{e-^1EQJ+cq_&&#uVm_eN4}U~`GVXyeltJmXv;jC33VB{Vh4{(NX=J( zr!KguT&I?#un?3Lol<;Xr>)t`V(qORH@xi>Z;j5Zq)hpVzVcL7g5d7TFYiHtJk(2N zXpJ8tUrbpFEylFbLgWxR{7;(8;(rAu1XTG7gi5RAw6_pBME;?|9wcAWgu_^L1bkNc zqR#{m43Y>yWd4*IT( zhcgp7Fl`SygVF9C|@8sA_?KtGJKY zy+tES!S*%iE!Nq0CbOxdC>ZiXjSO@&w*2hA{{k%|mow09lJ3{0*>~d?GD*iU zCJijmL-7A9Ajla!TxVb0y;35%e6wfL>>iXQB5ym6ZfwNL--%MKX&OTEWq~R!vAW?XBC4dTAGAW0k#N8FW zdW)%%M;`o9SQy4{o^wS$)2jdmpb=;ko_tW*ZBHq;@c@gdMjm(f34U@6h6bfioZnT> z;Nkkare1QTn`ZCloG9KRe||AJDn+E$nJp&mkAlrnDHomtHM^+%r7|7e158*gR`WAi zGDaew>n+-ZrOSGOyVtw%k(Va3Q_JkY#cYf#dP3{U898YHx=*EfXEo0ZDsnLV^Hy*< z%;N#6gXXA4g4N~O@5sqmyLz~4>cu6>T6tNCxX&pST zmjU^&?Uzowlq`&HwZgdAE>@9qVw!uI(7c5%#)rvYHuH-onQcB&4~U z{iH2?bPa}+E}y{Yla2Gf_l$g|!NmY{Oqq`Ete>%Z7!O2l`L+FGNz)5ktOvg-uIi+_ z@Oz0Is47S0PmJbGrU#Ixv^V zVc8X=x2wr8@`UvCq!sp@i1f>pW2>R@Aco~C!CXE!Uc2|=TUU-Y#VjcNjm>q|qQRIsKdmwSI3cJM$jQGhv*I+S;*|HAsA$~x0 zI1NYFASi|>^Da^@>rN`S_RCUG6z6Oqi-FCupIWhBoL{jsjAzv$r{kp24XwBOm*ZOd z=dsWl`2wR2%<|1m#|w-A{%V;c0Fke#vgA$nK;#fPQ%w%`|D-XE*_(V{o%=wQLzOcr z2C94|)dQbZPWw_cd>e8R2w~(y!y}bd)s>g3hlW>$;fKF+D^!sW4Uc?v^&9Gu^zm@TbskornS4tReJ^i6x9F{JJ#^$=6JL*KOL)ffN$W3RN{BaL61 ze@nmW?n86VhJYYFR9&q-#qcwd>ZbIyt1mAR?nTEZ0wlVD=$?$ALa5IBKJRX zMng{<@>lmRNw4)1E)2+D1sJ*Ze);~S4JaCpvevT@7FpQ!Eqv|{e!=!w7MdjP2R4EA zg_&Fy*oV7AK>0-hqq=Uz(H*gCQD6qTJ|R=8rUuWn^V_<;ar=sk_oZ!3`XnxT;|@u) z-FwuY@$MJ?m$dj><<#ZT3_YpFp+@PsABLFBd%Z;NT-ZH5vfjP04_)2G+OS{I1FW9L z8;`^~kgrFQq}?`WslM{0$gQWEM+w8*p_@nIBe$0I0cWW@U$9wO<@d+H>rU}jG`AM) z?_uYhp_dIg(Q{2IZ-2#@*Iptg3`WQD%^kUi<(sx92Tdvk zItH7gunU$g&<>nHb85$Ja?~oa(4=w=LAr$s+HEV3rxn{NcYR z6O`-3`2B&*0-HN72J4vI@`$Y*zq<#73RM?3gX;Nf@;0KTy zzbnk#$U-B@75jIj!>hqa4Ak;d*=RU|ZrE{XZ-Q<4#=J8`?jvqi z*$WKJ@-1Y;LC>%$&EucQrQeuZl7hyQpWx~HgxbYIm!JVIZq4nz)-f9UIuzUCC%KiDDwRSs4DA(gMBulKIbYw=m-t20b1>8c@e zc()2H>`W1WDqnTVfXE?ow+NUz3fMODxFFAufXwA|L(jR%yLPXbtDOuUkKTL5EHau? z9g51h!l7;0UGvxBOIXpPThoz>%v3IL-IX@Ay|Qatni@r}1I+ z>6$@+oanAP*Yx5&y()Tze6FJ?E@Jm}V8{oy*A2U|#L{P0v&~%<0q&}wt2DsE(Zxk| zv?&i;gM@vAAZvinrgm7Fj&TcG$bM!z3O3=qCd%o{KB{-Ehu7f+YRZ<|)cm>4P!QO@ zw(xP%metw1Rn$Tr4v^!!kMombA`leY%$s+OT4ZJNds5mv3`k)&yBd0$-|Ks>%OwA- zt-e@&=f&cknl%+EXxc@pK5La3Se8m_vAL`E&sfc`dqv3+>6b|_Uz&@mtI9{N?ZjgE z&94p3d@hGiy0O~BLEM{UY^9E}ZY^rj&00(&syv)6 zXq&T|0rD_xAL$MExHG?-o8Re}BE!B$e`t}d#s$jz4Vnp}*NnZem4Y-95Hn!P8$*}$b$ zdu=D@nCe0%2C9pxqabdl;-#x~tEu%cM!E-+Dlh7?U6fw66WwOZ&S0ay9LP1-#Z^6| ziRb(2rgO5U$$8jRIJ`w}0XoC^t@=ko&L&~E$!eZy$XyvQW-L#1JTaa0XeL!o&gd=y zb@NxGseG z&YSDg6GnKwQ}VIsb-K}){CGB0Y}K0yGMkA$2x z5!p)0Gt|^}f{R4$ta(YE6xh6{i0^d_1M|x_YE?MF(IGpE!~Czv({~rd(8q) zOkU#@IZhyVEWLv1`75SgQL-bkNH_l+l|!|_N_G>5tcgI`iMu7h5!P%OujglQun1&L z1QwTZ_;RoNlEo)5f!jGKkO%JaV9)LmfXEk9mfYhQLM;$EM9wt(LaV(x54H$El|z+7m50LJqf4nA zA*J%?%AB=vf@ZtdkGE`G{T;kLYQ?HQD3CAkZxxqx)@yEEt|eXAeBuF-yM!ws$Xz{L zjl@LH^$Mv_EES5+tNuOVLXp&*x5yVMP>4LB$Qe9b-*j9No&Og}dR#3puDUnrG6KS2 z^cK0k>CoB)Ne_AWgj#-JYjh-vMzWI*R8pRb9(*|g^7y?+_9X(p&ghhU;Pj|h3%F>e z(rykmP@c4Q|1Hm+RqkmTx-EWJ`76!f#2aTr&zYg?XC3a$MAsG3wce#8*WSuwN9QUZ zPZ^1cZh00fI9*F;uyobzmh}|9F## zen(E?bVhD?)1eTS{_oH$gOPYW+E_0119H-8^V9=->a-UEXI7mDUrx$|Pipt{94Jph zcPPe;#7~u}k=x4#NISqR_P%;wSB3VoFLJv1%MM=#HpgbusjbOnB}w~iaAUa)%Da z@0~w;W6;g4*=OXghPuRq;(3*S&}#FcG(K{F9XV}J-^`Crl-{q;kKXI>?4_q;r~`H% z|1u(fJK9nymEU79YxWtrL0fl##mcD!bGc5V*vq!f<({oJVV^2Uz0=)+ExpC>$eBD`4X^tCRd@cZRDN3nW^}jBcjQK5LR~h$VlSUx zu`~H{c2)uleqf`&HZJohx#+Wxp;MfIpu1{-DS_fO;WL#b`C?I+&EnEKQwar_@v%UYePWXq#+e;^0Ta#m__ zi`_MS$$bthGf_0Yz&@}0D~*D_pU^=b^q~xu)c{1kn9|Hycm8Od9?c(S08a(iMba z>nPe3kzVoh?6ZsM?2HQ_4;Da9YJNoSB*Br&4AQ$30G{m}k^qxei_KltK9@t} zR(~UZ(_M8+Aoz#)ba8P}u~>e08U%gz#a$(7=y|rHzL%`_P7*g;tll99*66^q*8)7) zOVc83$j4OK>_pKclX*;)z0Pj5K7275%}{_9nMreP4~F2Rhq8;&CseuR{U87N!~7>> zHS^_jxvhXR_T#D1JLFdLbJgKw99xHT()bFcU^DObtZ7`4g(u4fDHC1o^*yA~I=PZR zlfjAMCU48gKGD#4Cgt#tCYrY^ME+MI_mm2-JrEJ_cVTy}*nt=^>~o0`fXEj!6D=|5 zf9>HAIYiD>nFGCi`)jj)wVfTR9I70uoGAh;sS^A@R7}5wja2%600000NkvXXu0mjf DztFEy literal 0 HcmV?d00001 From 9646b908a75de403aa460312593b612456808aac Mon Sep 17 00:00:00 2001 From: antonyzhilin Date: Fri, 19 Jul 2024 10:13:45 +0300 Subject: [PATCH 054/629] feat grpc: add GenericClient 45c630765c628f16cd600ab26711d67e118c8767 --- .mapping.json | 8 ++ core/include/userver/concurrent/variable.hpp | 6 + .../utils/statistics/histogram_view.hpp | 8 +- .../userver/utils/statistics/metric_value.hpp | 28 +++- .../utils/statistics/striped_rate_counter.hpp | 2 +- core/src/utils/statistics/histogram_view.cpp | 5 - .../statistics/impl/histogram_view_utils.hpp | 3 +- .../userver/utils/statistics/testing.hpp | 1 + .../userver/ugrpc/byte_buffer_utils.hpp | 38 ++++++ .../userver/ugrpc/client/client_factory.hpp | 23 ++-- grpc/include/userver/ugrpc/client/generic.hpp | 88 ++++++++++++ .../ugrpc/client/impl/async_methods.hpp | 24 +--- .../userver/ugrpc/client/impl/call_params.hpp | 32 ++--- .../userver/ugrpc/client/impl/client_data.hpp | 55 +++++--- .../userver/ugrpc/client/middlewares/base.hpp | 7 + .../client/middlewares/log/component.hpp | 8 +- grpc/include/userver/ugrpc/client/rpc.hpp | 80 ++++++----- .../userver/ugrpc/impl/maybe_owned_string.hpp | 47 +++++++ .../include/userver/ugrpc/impl/statistics.hpp | 16 ++- .../userver/ugrpc/impl/statistics_storage.hpp | 70 +++++++--- grpc/src/ugrpc/byte_buffer_utils.cpp | 40 ++++++ grpc/src/ugrpc/client/generic.cpp | 48 +++++++ grpc/src/ugrpc/client/impl/async_methods.cpp | 11 +- grpc/src/ugrpc/client/impl/call_params.cpp | 78 +++++++++-- grpc/src/ugrpc/client/impl/client_data.cpp | 34 +++++ grpc/src/ugrpc/client/middlewares/base.cpp | 14 ++ .../client/middlewares/log/component.cpp | 19 ++- .../client/middlewares/log/middleware.cpp | 2 +- .../client/middlewares/log/middleware.hpp | 19 ++- grpc/src/ugrpc/impl/statistics.cpp | 19 ++- grpc/src/ugrpc/impl/statistics_storage.cpp | 122 ++++++++++++++--- grpc/tests/src/base_test.cpp | 1 + grpc/tests/src/generic_client_test.cpp | 125 ++++++++++++++++++ grpc/tests/src/logging_test.cpp | 2 +- grpc/utest/src/ugrpc/tests/service.cpp | 3 +- scripts/grpc/templates/client.usrv.cpp.jinja | 4 +- scripts/grpc/templates/client.usrv.hpp.jinja | 2 +- .../userver/utils/impl/source_location.hpp | 2 + universal/include/userver/utils/span.hpp | 22 ++- universal/src/utils/impl/source_location.cpp | 16 +++ .../userver/utest/log_capture_fixture.hpp | 32 ++++- .../utest/src/utest/log_capture_fixture.cpp | 50 ++++--- ydb/tests/small_table.hpp | 7 +- 43 files changed, 970 insertions(+), 251 deletions(-) create mode 100644 grpc/include/userver/ugrpc/byte_buffer_utils.hpp create mode 100644 grpc/include/userver/ugrpc/client/generic.hpp create mode 100644 grpc/include/userver/ugrpc/impl/maybe_owned_string.hpp create mode 100644 grpc/src/ugrpc/byte_buffer_utils.cpp create mode 100644 grpc/src/ugrpc/client/generic.cpp create mode 100644 grpc/src/ugrpc/client/impl/client_data.cpp create mode 100644 grpc/tests/src/generic_client_test.cpp create mode 100644 universal/src/utils/impl/source_location.cpp diff --git a/.mapping.json b/.mapping.json index e0af039c72bf..e056b493e9b1 100644 --- a/.mapping.json +++ b/.mapping.json @@ -1771,11 +1771,13 @@ "grpc/handlers/proto/healthchecking/healthchecking.proto":"taxi/uservices/userver/grpc/handlers/proto/healthchecking/healthchecking.proto", "grpc/handlers/src/ugrpc/server/health/health.cpp":"taxi/uservices/userver/grpc/handlers/src/ugrpc/server/health/health.cpp", "grpc/handlers/src/ugrpc/server/health/test_test.cpp":"taxi/uservices/userver/grpc/handlers/src/ugrpc/server/health/test_test.cpp", + "grpc/include/userver/ugrpc/byte_buffer_utils.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/byte_buffer_utils.hpp", "grpc/include/userver/ugrpc/client/channels.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/channels.hpp", "grpc/include/userver/ugrpc/client/client_factory.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/client_factory.hpp", "grpc/include/userver/ugrpc/client/client_factory_component.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/client_factory_component.hpp", "grpc/include/userver/ugrpc/client/exceptions.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/exceptions.hpp", "grpc/include/userver/ugrpc/client/fwd.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/fwd.hpp", + "grpc/include/userver/ugrpc/client/generic.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/generic.hpp", "grpc/include/userver/ugrpc/client/impl/async_method_invocation.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/async_method_invocation.hpp", "grpc/include/userver/ugrpc/client/impl/async_methods.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/async_methods.hpp", "grpc/include/userver/ugrpc/client/impl/call_params.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/call_params.hpp", @@ -1797,6 +1799,7 @@ "grpc/include/userver/ugrpc/impl/completion_queues.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/impl/completion_queues.hpp", "grpc/include/userver/ugrpc/impl/deadline_timepoint.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/impl/deadline_timepoint.hpp", "grpc/include/userver/ugrpc/impl/internal_tag_fwd.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/impl/internal_tag_fwd.hpp", + "grpc/include/userver/ugrpc/impl/maybe_owned_string.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/impl/maybe_owned_string.hpp", "grpc/include/userver/ugrpc/impl/queue_runner.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/impl/queue_runner.hpp", "grpc/include/userver/ugrpc/impl/span.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/impl/span.hpp", "grpc/include/userver/ugrpc/impl/static_metadata.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/impl/static_metadata.hpp", @@ -1835,16 +1838,19 @@ "grpc/proto/tests/repeating_word_in_package_name.proto":"taxi/uservices/userver/grpc/proto/tests/repeating_word_in_package_name.proto", "grpc/proto/tests/same_service_and_method_name.proto":"taxi/uservices/userver/grpc/proto/tests/same_service_and_method_name.proto", "grpc/proto/tests/unit_test.proto":"taxi/uservices/userver/grpc/proto/tests/unit_test.proto", + "grpc/src/ugrpc/byte_buffer_utils.cpp":"taxi/uservices/userver/grpc/src/ugrpc/byte_buffer_utils.cpp", "grpc/src/ugrpc/client/channels.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/channels.cpp", "grpc/src/ugrpc/client/client_factory.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/client_factory.cpp", "grpc/src/ugrpc/client/client_factory_component.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/client_factory_component.cpp", "grpc/src/ugrpc/client/exceptions.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/exceptions.cpp", + "grpc/src/ugrpc/client/generic.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/generic.cpp", "grpc/src/ugrpc/client/impl/async_method_invocation.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/impl/async_method_invocation.cpp", "grpc/src/ugrpc/client/impl/async_methods.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/impl/async_methods.cpp", "grpc/src/ugrpc/client/impl/call_params.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/impl/call_params.cpp", "grpc/src/ugrpc/client/impl/channel_cache.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/impl/channel_cache.cpp", "grpc/src/ugrpc/client/impl/client_configs.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/impl/client_configs.cpp", "grpc/src/ugrpc/client/impl/client_configs.hpp":"taxi/uservices/userver/grpc/src/ugrpc/client/impl/client_configs.hpp", + "grpc/src/ugrpc/client/impl/client_data.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/impl/client_data.cpp", "grpc/src/ugrpc/client/impl/client_factory_config.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/impl/client_factory_config.cpp", "grpc/src/ugrpc/client/impl/client_factory_config.hpp":"taxi/uservices/userver/grpc/src/ugrpc/client/impl/client_factory_config.hpp", "grpc/src/ugrpc/client/impl/client_qos.cpp":"taxi/uservices/userver/grpc/src/ugrpc/client/impl/client_qos.cpp", @@ -1920,6 +1926,7 @@ "grpc/tests/src/deadline_metrics_test.cpp":"taxi/uservices/userver/grpc/tests/src/deadline_metrics_test.cpp", "grpc/tests/src/deadline_test.cpp":"taxi/uservices/userver/grpc/tests/src/deadline_test.cpp", "grpc/tests/src/error_test.cpp":"taxi/uservices/userver/grpc/tests/src/error_test.cpp", + "grpc/tests/src/generic_client_test.cpp":"taxi/uservices/userver/grpc/tests/src/generic_client_test.cpp", "grpc/tests/src/logging_test.cpp":"taxi/uservices/userver/grpc/tests/src/logging_test.cpp", "grpc/tests/src/middlewares_test.cpp":"taxi/uservices/userver/grpc/tests/src/middlewares_test.cpp", "grpc/tests/src/serialization_test.cpp":"taxi/uservices/userver/grpc/tests/src/serialization_test.cpp", @@ -4024,6 +4031,7 @@ "universal/src/utils/impl/disable_core_dumps.cpp":"taxi/uservices/userver/universal/src/utils/impl/disable_core_dumps.cpp", "universal/src/utils/impl/internal_tag.hpp":"taxi/uservices/userver/universal/src/utils/impl/internal_tag.hpp", "universal/src/utils/impl/projecting_view_test.cpp":"taxi/uservices/userver/universal/src/utils/impl/projecting_view_test.cpp", + "universal/src/utils/impl/source_location.cpp":"taxi/uservices/userver/universal/src/utils/impl/source_location.cpp", "universal/src/utils/impl/source_location_test.cpp":"taxi/uservices/userver/universal/src/utils/impl/source_location_test.cpp", "universal/src/utils/impl/static_registration.cpp":"taxi/uservices/userver/universal/src/utils/impl/static_registration.cpp", "universal/src/utils/impl/transparent_hash_test.cpp":"taxi/uservices/userver/universal/src/utils/impl/transparent_hash_test.cpp", diff --git a/core/include/userver/concurrent/variable.hpp b/core/include/userver/concurrent/variable.hpp index 4a5e75505e15..24c9d7b1966d 100644 --- a/core/include/userver/concurrent/variable.hpp +++ b/core/include/userver/concurrent/variable.hpp @@ -93,6 +93,12 @@ class Variable final { return {mutex_, data_}; } + /// Useful for grabbing a reference to an object in a node-based container, + /// e.g. `std::unordered_map`. Values must support concurrent modification. + LockedPtr, Data> SharedMutableLockUnsafe() { + return {mutex_, data_}; + } + LockedPtr, Data> Lock() { return {mutex_, data_}; } LockedPtr, const Data> Lock() const { diff --git a/core/include/userver/utils/statistics/histogram_view.hpp b/core/include/userver/utils/statistics/histogram_view.hpp index 4c5e53e87504..754607a10522 100644 --- a/core/include/userver/utils/statistics/histogram_view.hpp +++ b/core/include/userver/utils/statistics/histogram_view.hpp @@ -26,8 +26,8 @@ struct Access; class HistogramView final { public: // trivially copyable - HistogramView(const HistogramView&) noexcept = default; - HistogramView& operator=(const HistogramView&) noexcept = default; + constexpr HistogramView(const HistogramView&) noexcept = default; + constexpr HistogramView& operator=(const HistogramView&) noexcept = default; /// Returns the number of "normal" (non-"infinity") buckets. std::size_t GetBucketCount() const noexcept; @@ -48,7 +48,9 @@ class HistogramView final { private: friend struct impl::histogram::Access; - explicit HistogramView(const impl::histogram::Bucket* buckets) noexcept; + constexpr explicit HistogramView( + const impl::histogram::Bucket& buckets) noexcept + : buckets_(&buckets) {} const impl::histogram::Bucket* buckets_; }; diff --git a/core/include/userver/utils/statistics/metric_value.hpp b/core/include/userver/utils/statistics/metric_value.hpp index 0d718db25b61..0ee86215728d 100644 --- a/core/include/userver/utils/statistics/metric_value.hpp +++ b/core/include/userver/utils/statistics/metric_value.hpp @@ -21,16 +21,34 @@ class MetricValue final { public: using RawType = std::variant; + /// Creates an unspecified metric value. + constexpr MetricValue() noexcept : value_(std::int64_t{0}) {} + + /// Constructs MetricValue for tests. + /// @{ + constexpr /*implicit*/ MetricValue(std::int64_t value) noexcept + : value_(value) {} + + constexpr /*implicit*/ MetricValue(double value) noexcept : value_(value) {} + + constexpr /*implicit*/ MetricValue(Rate value) noexcept : value_(value) {} + + constexpr /*implicit*/ MetricValue(HistogramView value) noexcept + : value_(value) {} + /// @} + // trivially copyable MetricValue(const MetricValue&) = default; MetricValue& operator=(const MetricValue&) = default; - bool operator==(const MetricValue& other) const noexcept { - return value_ == other.value_; + friend bool operator==(const MetricValue& lhs, + const MetricValue& rhs) noexcept { + return lhs.value_ == rhs.value_; } - bool operator!=(const MetricValue& other) const noexcept { - return value_ != other.value_; + friend bool operator!=(const MetricValue& lhs, + const MetricValue& rhs) noexcept { + return lhs.value_ != rhs.value_; } /// @brief Retrieve the value of an integer metric. @@ -65,8 +83,6 @@ class MetricValue final { } /// @cond - MetricValue() noexcept : value_(std::int64_t{0}) {} - explicit MetricValue(RawType value) noexcept : value_(value) {} /// @endcond diff --git a/core/include/userver/utils/statistics/striped_rate_counter.hpp b/core/include/userver/utils/statistics/striped_rate_counter.hpp index 2b0eb2f8df54..d93e4ba1bf0d 100644 --- a/core/include/userver/utils/statistics/striped_rate_counter.hpp +++ b/core/include/userver/utils/statistics/striped_rate_counter.hpp @@ -90,7 +90,7 @@ class StripedRateCounter final { return val_.Read() + offset_.load(std::memory_order_relaxed); } - concurrent::StripedCounter val_; + USERVER_NAMESPACE::concurrent::StripedCounter val_; std::atomic offset_{0}; }; diff --git a/core/src/utils/statistics/histogram_view.cpp b/core/src/utils/statistics/histogram_view.cpp index 82b4fa0a9a95..0355e71f9dcf 100644 --- a/core/src/utils/statistics/histogram_view.cpp +++ b/core/src/utils/statistics/histogram_view.cpp @@ -16,11 +16,6 @@ static_assert(std::is_trivially_copyable_v && "HistogramView should fit in registers, because it is expected " "to be passed around by value"); -HistogramView::HistogramView(const impl::histogram::Bucket* buckets) noexcept - : buckets_(buckets) { - UASSERT(buckets); -} - std::size_t HistogramView::GetBucketCount() const noexcept { UASSERT(buckets_); return buckets_[0].upper_bound.size; diff --git a/core/src/utils/statistics/impl/histogram_view_utils.hpp b/core/src/utils/statistics/impl/histogram_view_utils.hpp index 99ef2c333fc3..4d41e7b80ed3 100644 --- a/core/src/utils/statistics/impl/histogram_view_utils.hpp +++ b/core/src/utils/statistics/impl/histogram_view_utils.hpp @@ -21,7 +21,8 @@ namespace utils::statistics::impl::histogram { struct Access final { static HistogramView MakeView(const Bucket* buckets) noexcept { - return HistogramView{buckets}; + UASSERT(buckets); + return HistogramView{*buckets}; } template diff --git a/core/utest/include/userver/utils/statistics/testing.hpp b/core/utest/include/userver/utils/statistics/testing.hpp index ccdfe2157858..1757acdfccf7 100644 --- a/core/utest/include/userver/utils/statistics/testing.hpp +++ b/core/utest/include/userver/utils/statistics/testing.hpp @@ -4,6 +4,7 @@ /// @brief Utilities for analyzing emitted metrics in unit tests #include +#include #include #include #include diff --git a/grpc/include/userver/ugrpc/byte_buffer_utils.hpp b/grpc/include/userver/ugrpc/byte_buffer_utils.hpp new file mode 100644 index 000000000000..a346032695fb --- /dev/null +++ b/grpc/include/userver/ugrpc/byte_buffer_utils.hpp @@ -0,0 +1,38 @@ +#pragma once + +/// @file userver/ugrpc/byte_buffer_utils.hpp +/// @brief Helper functions for working with `grpc::ByteBuffer` + +#include + +#include +#include + +USERVER_NAMESPACE_BEGIN + +namespace ugrpc { + +/// @see @ref SerializeToByteBuffer +inline constexpr std::size_t kDefaultSerializeBlockSize = 4096; + +/// @brief Serialize a Protobuf message to the wire format. +/// @param message the message to serialize +/// @param block_size can be used for performance tuning, too small chunk size +/// results in extra allocations, too large chunk size results in wasting memory +/// @throws std::runtime_error on serialization errors (supposedly only happens +/// on extremely rare allocation failures or proto reflection malfunctioning). +grpc::ByteBuffer SerializeToByteBuffer( + const ::google::protobuf::Message& message, + std::size_t block_size = kDefaultSerializeBlockSize); + +/// @brief Parse a Protobuf message from the wire format. +/// @param buffer the buffer that might be tempered with during deserialization +/// @param message will contain the parsing result on success +/// @returns `true` on success, `false` if @a buffer does not contain a valid +/// message, according to the derived type of @a message +bool ParseFromByteBuffer(grpc::ByteBuffer&& buffer, + ::google::protobuf::Message& message); + +} // namespace ugrpc + +USERVER_NAMESPACE_END diff --git a/grpc/include/userver/ugrpc/client/client_factory.hpp b/grpc/include/userver/ugrpc/client/client_factory.hpp index 63d88ec35639..3f4e113dcc16 100644 --- a/grpc/include/userver/ugrpc/client/client_factory.hpp +++ b/grpc/include/userver/ugrpc/client/client_factory.hpp @@ -50,7 +50,9 @@ struct ClientFactorySettings final { std::size_t channel_count{1}; }; -/// @brief Creates generated gRPC clients. Has a minimal built-in channel cache: +/// @ingroup userver_clients +/// +/// @brief Creates gRPC clients. Has a minimal built-in channel cache: /// as long as a channel to the same endpoint is used somewhere, the same /// channel is given out. class ClientFactory final { @@ -84,17 +86,16 @@ class ClientFactory final { template Client ClientFactory::MakeClient(const std::string& client_name, const std::string& endpoint) { - auto& statistics = client_statistics_storage_.GetServiceStatistics( - Client::GetMetadata(), endpoint); - - Middlewares mws; - mws.reserve(mws_.size()); - for (const auto& mw_factory : mws_) - mws.push_back(mw_factory->GetMiddleware(client_name)); - return Client(impl::ClientParams{ - client_name, std::move(mws), queue_, statistics, - GetChannel(client_name, endpoint), config_source_, testsuite_grpc_}); + client_name, + endpoint, + impl::InstantiateMiddlewares(mws_, client_name), + queue_, + client_statistics_storage_, + GetChannel(client_name, endpoint), + config_source_, + testsuite_grpc_, + }); } } // namespace ugrpc::client diff --git a/grpc/include/userver/ugrpc/client/generic.hpp b/grpc/include/userver/ugrpc/client/generic.hpp new file mode 100644 index 000000000000..7a1222f236f6 --- /dev/null +++ b/grpc/include/userver/ugrpc/client/generic.hpp @@ -0,0 +1,88 @@ +#pragma once + +/// @file userver/ugrpc/client/generic.hpp +/// @brief @copybrief ugrpc::client::GenericClient + +#include +#include + +#include +#include + +#include +#include +#include + +USERVER_NAMESPACE_BEGIN + +namespace ugrpc::client { + +struct GenericOptions { + /// Client QOS for this call. Note that there is no QOS dynamic config by + /// default, so unless a timeout is specified here, only the deadline + /// propagation mechanism will affect the gRPC deadline. + Qos qos{}; + + /// If non-`nullopt`, metrics are accounted for specified fake call name. + /// If `nullopt` (by default), writes a set of metrics per real call name. + /// If the microservice serves as a proxy and has untrusted clients, it is + /// a good idea to have this option set to non-`nullopt` to avoid + /// the situations where an upstream client can spam various RPCs with + /// non-existent names, which leads to this microservice spamming RPCs + /// with non-existent names, which leads to creating storage for infinite + /// metrics and causes OOM. + std::optional metrics_call_name{}; +}; + +/// @ingroup userver_clients +/// +/// @brief Allows to talk to gRPC services (generic and normal) using dynamic +/// method names. +/// +/// Created using @ref ClientFactory::MakeClient. +/// +/// `call_name` must be in the format `full.path.to.TheService/MethodName`. +/// Note that unlike in base grpc++, there must be no initial `/` character. +/// +/// The API is mainly intended for proxies, where the request-response body is +/// passed unchanged, with settings taken solely from the RPC metadata. +/// In cases where the code needs to operate on the actual messages, +/// serialization of requests and responses is left as an excercise to the user. +/// +/// Middlewares are customizable and are applied as usual, except that no +/// message hooks are called, meaning that there won't be any logs of messages +/// from the default middleware. +/// +/// Metrics are written per-method by default, which causes OOM in some corner +/// cases, for details see @ref GenericOptions::metrics_call_name. +/// +/// ## Example GenericClient usage with known message types +/// +/// @snippet grpc/tests/src/generic_client_test.cpp sample +class GenericClient final { + public: + GenericClient(GenericClient&&) noexcept = default; + GenericClient& operator=(GenericClient&&) noexcept = delete; + + /// Initiate a `single request -> single response` RPC with the given name. + client::UnaryCall UnaryCall( + std::string_view call_name, const grpc::ByteBuffer& request, + std::unique_ptr context = + std::make_unique(), + const GenericOptions& options = {}) const; + + /// @cond + // For internal use only. + explicit GenericClient(impl::ClientParams&&); + /// @endcond + + private: + template + friend impl::ClientData& impl::GetClientData(Client& client); + + impl::ClientData impl_; +}; + +} // namespace ugrpc::client + +USERVER_NAMESPACE_END diff --git a/grpc/include/userver/ugrpc/client/impl/async_methods.hpp b/grpc/include/userver/ugrpc/client/impl/async_methods.hpp index cbcefd4d122a..ac8ca92ca621 100644 --- a/grpc/include/userver/ugrpc/client/impl/async_methods.hpp +++ b/grpc/include/userver/ugrpc/client/impl/async_methods.hpp @@ -20,6 +20,7 @@ #include #include #include +#include #include USERVER_NAMESPACE_BEGIN @@ -45,27 +46,6 @@ using RawReaderWriter = std::unique_ptr>; /// @} -/// @{ -/// @brief Helper type aliases for stub member function pointers -template -using RawResponseReaderPreparer = RawResponseReader (Stub::*)( - grpc::ClientContext*, const Request&, grpc::CompletionQueue*); - -template -using RawReaderPreparer = RawReader (Stub::*)(grpc::ClientContext*, - const Request&, - grpc::CompletionQueue*); - -template -using RawWriterPreparer = RawWriter (Stub::*)(grpc::ClientContext*, - Response*, - grpc::CompletionQueue*); - -template -using RawReaderWriterPreparer = RawReaderWriter (Stub::*)( - grpc::ClientContext*, grpc::CompletionQueue*); -/// @} - struct RpcConfigValues final { explicit RpcConfigValues(const dynamic_config::Snapshot& config); @@ -156,7 +136,7 @@ class RpcData final { private: std::unique_ptr context_; std::string client_name_; - std::string_view call_name_; + ugrpc::impl::MaybeOwnedString call_name_; bool writes_finished_{false}; bool is_finished_{false}; bool is_deadline_propagated_{false}; diff --git a/grpc/include/userver/ugrpc/client/impl/call_params.hpp b/grpc/include/userver/ugrpc/client/impl/call_params.hpp index 659dc166f70a..5506ec1556ac 100644 --- a/grpc/include/userver/ugrpc/client/impl/call_params.hpp +++ b/grpc/include/userver/ugrpc/client/impl/call_params.hpp @@ -1,5 +1,6 @@ #pragma once +#include #include #include @@ -8,8 +9,10 @@ #include #include +#include #include #include +#include #include USERVER_NAMESPACE_BEGIN @@ -20,37 +23,22 @@ struct CallParams { std::string_view client_name; grpc::CompletionQueue& queue; dynamic_config::Snapshot config; - std::string_view call_name; + ugrpc::impl::MaybeOwnedString call_name; std::unique_ptr context; ugrpc::impl::MethodStatistics& statistics; const Middlewares& mws; }; -CallParams DoCreateCallParams(const ClientData&, std::size_t method_id, - std::unique_ptr); - -template CallParams CreateCallParams(const ClientData& client_data, std::size_t method_id, std::unique_ptr client_context, - const ClientQosConfig& client_qos, - const ugrpc::client::Qos& qos) { - const auto& metadata = client_data.GetMetadata(); - const auto& full_name = metadata.method_full_names[method_id]; - const auto& method_name = - full_name.substr(metadata.service_full_name.size() + 1); - - const auto& config = client_data.GetConfigSnapshot(); - - // User qos goes first - ApplyQos(*client_context, qos, client_data.GetTestsuiteControl()); - - // If user qos was empty update timeout from config - ApplyQos(*client_context, config[client_qos][method_name], - client_data.GetTestsuiteControl()); + const dynamic_config::Key& client_qos, + const Qos& qos); - return DoCreateCallParams(client_data, method_id, std::move(client_context)); -} +CallParams CreateGenericCallParams( + const ClientData& client_data, std::string_view call_name, + std::unique_ptr client_context, const Qos& qos, + std::optional metrics_call_name); } // namespace ugrpc::client::impl diff --git a/grpc/include/userver/ugrpc/client/impl/client_data.hpp b/grpc/include/userver/ugrpc/client/impl/client_data.hpp index a1d354b14c66..a0f4b2a89349 100644 --- a/grpc/include/userver/ugrpc/client/impl/client_data.hpp +++ b/grpc/include/userver/ugrpc/client/impl/client_data.hpp @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -18,18 +19,25 @@ USERVER_NAMESPACE_BEGIN +namespace ugrpc::impl { +class StatisticsStorage; +} // namespace ugrpc::impl + namespace ugrpc::client::impl { struct ClientParams final { std::string client_name; + std::string endpoint; Middlewares mws; grpc::CompletionQueue& queue; - ugrpc::impl::ServiceStatistics& statistics_storage; + ugrpc::impl::StatisticsStorage& statistics_storage; impl::ChannelCache::Token channel_token; const dynamic_config::Source config_source; testsuite::GrpcControl& testsuite_grpc; }; +struct GenericClientTag final {}; + /// A helper class for generated gRPC clients class ClientData final { public: @@ -41,14 +49,16 @@ class ClientData final { template ClientData(ClientParams&& params, ugrpc::impl::StaticServiceMetadata metadata, std::in_place_type_t) - : params_(std::move(params)), metadata_(metadata) { - const std::size_t channel_count = GetChannelToken().GetChannelCount(); - stubs_ = utils::GenerateFixedArray(channel_count, [&](std::size_t index) { - return StubPtr( - Service::NewStub(GetChannelToken().GetChannel(index)).release(), - &StubDeleter); - }); - } + : params_(std::move(params)), + metadata_(metadata), + service_statistics_(&GetServiceStatistics()), + stubs_(MakeStubs(params_.channel_token)) {} + + template + ClientData(ClientParams&& params, GenericClientTag, + std::in_place_type_t) + : params_(std::move(params)), + stubs_(MakeStubs(params_.channel_token)) {} ClientData(ClientData&&) noexcept = default; ClientData& operator=(ClientData&&) = delete; @@ -68,9 +78,10 @@ class ClientData final { return params_.config_source.GetSnapshot(); } - ugrpc::impl::MethodStatistics& GetStatistics(std::size_t method_id) const { - return params_.statistics_storage.GetMethodStatistics(method_id); - } + ugrpc::impl::MethodStatistics& GetStatistics(std::size_t method_id) const; + + ugrpc::impl::MethodStatistics& GetGenericStatistics( + std::string_view call_name) const; ChannelCache::Token& GetChannelToken() { return params_.channel_token; } @@ -78,9 +89,7 @@ class ClientData final { const Middlewares& GetMiddlewares() const { return params_.mws; } - const ugrpc::impl::StaticServiceMetadata& GetMetadata() const { - return metadata_; - } + const ugrpc::impl::StaticServiceMetadata& GetMetadata() const; const testsuite::GrpcControl& GetTestsuiteControl() const { return params_.testsuite_grpc; @@ -95,8 +104,22 @@ class ClientData final { delete static_cast*>(ptr); } + template + static utils::FixedArray MakeStubs( + impl::ChannelCache::Token& channel_token) { + const std::size_t channel_count = channel_token.GetChannelCount(); + return utils::GenerateFixedArray(channel_count, [&](std::size_t index) { + return StubPtr( + Service::NewStub(channel_token.GetChannel(index)).release(), + &StubDeleter); + }); + } + + ugrpc::impl::ServiceStatistics& GetServiceStatistics(); + ClientParams params_; - ugrpc::impl::StaticServiceMetadata metadata_; + std::optional metadata_{std::nullopt}; + ugrpc::impl::ServiceStatistics* service_statistics_{nullptr}; utils::FixedArray stubs_; }; diff --git a/grpc/include/userver/ugrpc/client/middlewares/base.hpp b/grpc/include/userver/ugrpc/client/middlewares/base.hpp index f19fdc8d2d45..90cb469f4853 100644 --- a/grpc/include/userver/ugrpc/client/middlewares/base.hpp +++ b/grpc/include/userver/ugrpc/client/middlewares/base.hpp @@ -94,6 +94,13 @@ class MiddlewareComponentBase : public components::ComponentBase { GetMiddlewareFactory() = 0; }; +namespace impl { + +Middlewares InstantiateMiddlewares(const MiddlewareFactories& factories, + const std::string& client_name); + +} // namespace impl + } // namespace ugrpc::client USERVER_NAMESPACE_END diff --git a/grpc/include/userver/ugrpc/client/middlewares/log/component.hpp b/grpc/include/userver/ugrpc/client/middlewares/log/component.hpp index 66cadb8678b7..6f29417fb25b 100644 --- a/grpc/include/userver/ugrpc/client/middlewares/log/component.hpp +++ b/grpc/include/userver/ugrpc/client/middlewares/log/component.hpp @@ -4,12 +4,15 @@ /// @brief @copybrief ugrpc::client::middlewares::log::Component #include +#include USERVER_NAMESPACE_BEGIN /// Client logging middleware namespace ugrpc::client::middlewares::log { +struct Settings; + // clang-format off /// @ingroup userver_components @@ -33,13 +36,14 @@ class Component final : public MiddlewareComponentBase { Component(const components::ComponentConfig& config, const components::ComponentContext& context); + ~Component() override; + std::shared_ptr GetMiddlewareFactory() override; static yaml_config::Schema GetStaticConfigSchema(); private: - std::size_t max_size_; - logging::Level log_level_; + const utils::Box settings_; }; } // namespace ugrpc::client::middlewares::log diff --git a/grpc/include/userver/ugrpc/client/rpc.hpp b/grpc/include/userver/ugrpc/client/rpc.hpp index c63f675f3ff4..a9262d62f703 100644 --- a/grpc/include/userver/ugrpc/client/rpc.hpp +++ b/grpc/include/userver/ugrpc/client/rpc.hpp @@ -180,11 +180,9 @@ class [[nodiscard]] UnaryCall final : public CallAnyBase { /// @cond // For internal use only - template - UnaryCall( - impl::CallParams&& params, Stub& stub, - impl::RawResponseReaderPreparer prepare_func, - const Request& req); + template + UnaryCall(impl::CallParams&& params, PrepareFunc prepare_func, + const Request& req); /// @endcond UnaryCall(UnaryCall&&) noexcept = default; @@ -223,9 +221,8 @@ class [[nodiscard]] InputStream final : public CallAnyBase { // For internal use only using RawStream = grpc::ClientAsyncReader; - template - InputStream(impl::CallParams&& params, Stub& stub, - impl::RawReaderPreparer prepare_func, + template + InputStream(impl::CallParams&& params, PrepareFunc prepare_func, const Request& req); /// @endcond @@ -293,9 +290,8 @@ class [[nodiscard]] OutputStream final : public CallAnyBase { // For internal use only using RawStream = grpc::ClientAsyncWriter; - template - OutputStream(impl::CallParams&& params, Stub& stub, - impl::RawWriterPreparer prepare_func); + template + OutputStream(impl::CallParams&& params, PrepareFunc prepare_func); /// @endcond OutputStream(OutputStream&&) noexcept = default; @@ -399,10 +395,8 @@ class [[nodiscard]] BidirectionalStream final : public CallAnyBase { // For internal use only using RawStream = grpc::ClientAsyncReaderWriter; - template - BidirectionalStream( - impl::CallParams&& params, Stub& stub, - impl::RawReaderWriterPreparer prepare_func); + template + BidirectionalStream(impl::CallParams&& params, PrepareFunc prepare_func); /// @endcond BidirectionalStream(BidirectionalStream&&) noexcept = default; @@ -477,20 +471,22 @@ bool StreamReadFuture::IsReady() const noexcept { } template -template -UnaryCall::UnaryCall( - impl::CallParams&& params, Stub& stub, - impl::RawResponseReaderPreparer prepare_func, - const Request& req) +template +UnaryCall::UnaryCall(impl::CallParams&& params, + PrepareFunc prepare_func, const Request& req) : CallAnyBase(std::move(params)) { + const ::google::protobuf::Message* req_message = nullptr; + if constexpr (std::is_base_of_v<::google::protobuf::Message, Request>) { + req_message = &req; + } impl::CallMiddlewares( GetData().GetMiddlewares(), *this, [&] { - reader_ = (stub.*prepare_func)(&GetData().GetContext(), req, - &GetData().GetQueue()); + reader_ = + prepare_func(&GetData().GetContext(), req, &GetData().GetQueue()); reader_->StartCall(); }, - &req); + req_message); GetData().SetWritesFinished(); } @@ -514,20 +510,22 @@ UnaryFuture UnaryCall::FinishAsync(Response& response) { } template -template -InputStream::InputStream( - impl::CallParams&& params, Stub& stub, - impl::RawReaderPreparer prepare_func, - const Request& req) +template +InputStream::InputStream(impl::CallParams&& params, + PrepareFunc prepare_func, const Request& req) : CallAnyBase(std::move(params)) { + const ::google::protobuf::Message* req_message = nullptr; + if constexpr (std::is_base_of_v<::google::protobuf::Message, Request>) { + req_message = &req; + } impl::CallMiddlewares( GetData().GetMiddlewares(), *this, [&] { - stream_ = (stub.*prepare_func)(&GetData().GetContext(), req, - &GetData().GetQueue()); + stream_ = + prepare_func(&GetData().GetContext(), req, &GetData().GetQueue()); impl::StartCall(*stream_, GetData()); }, - &req); + req_message); GetData().SetWritesFinished(); } @@ -544,19 +542,17 @@ bool InputStream::Read(Response& response) { } template -template -OutputStream::OutputStream( - impl::CallParams&& params, Stub& stub, - impl::RawWriterPreparer prepare_func) +template +OutputStream::OutputStream(impl::CallParams&& params, + PrepareFunc prepare_func) : CallAnyBase(std::move(params)), final_response_(std::make_unique()) { impl::CallMiddlewares( GetData().GetMiddlewares(), *this, [&] { // 'final_response_' will be filled upon successful 'Finish' async call - stream_ = - (stub.*prepare_func)(&GetData().GetContext(), final_response_.get(), - &GetData().GetQueue()); + stream_ = prepare_func(&GetData().GetContext(), final_response_.get(), + &GetData().GetQueue()); impl::StartCall(*stream_, GetData()); }, nullptr); @@ -596,16 +592,14 @@ Response OutputStream::Finish() { } template -template +template BidirectionalStream::BidirectionalStream( - impl::CallParams&& params, Stub& stub, - impl::RawReaderWriterPreparer prepare_func) + impl::CallParams&& params, PrepareFunc prepare_func) : CallAnyBase(std::move(params)) { impl::CallMiddlewares( GetData().GetMiddlewares(), *this, [&] { - stream_ = (stub.*prepare_func)(&GetData().GetContext(), - &GetData().GetQueue()); + stream_ = prepare_func(&GetData().GetContext(), &GetData().GetQueue()); impl::StartCall(*stream_, GetData()); }, nullptr); diff --git a/grpc/include/userver/ugrpc/impl/maybe_owned_string.hpp b/grpc/include/userver/ugrpc/impl/maybe_owned_string.hpp new file mode 100644 index 000000000000..abca68678944 --- /dev/null +++ b/grpc/include/userver/ugrpc/impl/maybe_owned_string.hpp @@ -0,0 +1,47 @@ +#pragma once + +#include +#include +#include + +USERVER_NAMESPACE_BEGIN + +namespace ugrpc::impl { + +class MaybeOwnedString final { + public: + struct Ref {}; + + MaybeOwnedString() = default; + + explicit MaybeOwnedString(std::string&& string) + : storage_(std::move(string)), view_(storage_) {} + + MaybeOwnedString(Ref, std::string_view string) : view_(string) {} + + MaybeOwnedString(MaybeOwnedString&& other) noexcept { + *this = std::move(other); + } + + MaybeOwnedString& operator=(MaybeOwnedString&& other) noexcept { + if (this == &other) return *this; + if (other.view_.data() == other.storage_.data()) { + storage_ = std::move(other.storage_); + view_ = storage_; + } else { + storage_.clear(); + view_ = other.view_; + } + return *this; + } + + std::string_view Get() const noexcept { return view_; } + + private: + std::string storage_; + std::string_view view_; +}; + +} // namespace ugrpc::impl + +USERVER_NAMESPACE_END diff --git a/grpc/include/userver/ugrpc/impl/statistics.hpp b/grpc/include/userver/ugrpc/impl/statistics.hpp index ae61e24edda4..9eeeaabc9497 100644 --- a/grpc/include/userver/ugrpc/impl/statistics.hpp +++ b/grpc/include/userver/ugrpc/impl/statistics.hpp @@ -18,6 +18,10 @@ USERVER_NAMESPACE_BEGIN +namespace utils::statistics { +class StripedRateCounter; +} // namespace utils::statistics + namespace ugrpc::impl { enum class StatisticsDomain { kClient, kServer }; @@ -26,7 +30,9 @@ std::string_view ToString(StatisticsDomain); class MethodStatistics final { public: - explicit MethodStatistics(StatisticsDomain domain); + explicit MethodStatistics( + StatisticsDomain domain, + utils::statistics::StripedRateCounter& global_started); void AccountStarted() noexcept; @@ -57,6 +63,7 @@ class MethodStatistics final { private: using Percentile = utils::statistics::Percentile<2000, std::uint32_t, 256, 100>; + using Timings = utils::statistics::RecentPeriod; using RateCounter = utils::statistics::RateCounter; // StatusCode enum cases have consecutive underlying values, starting from 0. // UNAUTHENTICATED currently has the largest value. @@ -64,9 +71,11 @@ class MethodStatistics final { static_cast(grpc::StatusCode::UNAUTHENTICATED) + 1; const StatisticsDomain domain_; + utils::statistics::StripedRateCounter& global_started_; + RateCounter started_{0}; std::array status_codes_{}; - utils::statistics::RecentPeriod timings_; + Timings timings_; RateCounter network_errors_{0}; RateCounter internal_errors_{0}; RateCounter cancelled_{0}; @@ -78,7 +87,8 @@ class MethodStatistics final { class ServiceStatistics final { public: ServiceStatistics(const StaticServiceMetadata& metadata, - StatisticsDomain domain); + StatisticsDomain domain, + utils::statistics::StripedRateCounter& global_started); ~ServiceStatistics(); diff --git a/grpc/include/userver/ugrpc/impl/statistics_storage.hpp b/grpc/include/userver/ugrpc/impl/statistics_storage.hpp index 5d9232f62646..5f24e96c3ab4 100644 --- a/grpc/include/userver/ugrpc/impl/statistics_storage.hpp +++ b/grpc/include/userver/ugrpc/impl/statistics_storage.hpp @@ -5,8 +5,11 @@ #include #include +#include #include +#include #include +#include #include @@ -14,8 +17,7 @@ USERVER_NAMESPACE_BEGIN namespace ugrpc::impl { -/// Clients are created on-the-fly, so we must use a separate stable container -/// for storing their statistics. +/// Allows to create ServiceStatistics and generic MethodStatistics on the fly. class StatisticsStorage final { public: explicit StatisticsStorage(utils::statistics::Storage& statistics_storage, @@ -23,16 +25,14 @@ class StatisticsStorage final { StatisticsStorage(const StatisticsStorage&) = delete; StatisticsStorage& operator=(const StatisticsStorage&) = delete; - ~StatisticsStorage(); - ugrpc::impl::ServiceStatistics& GetServiceStatistics( - const ugrpc::impl::StaticServiceMetadata& metadata, - std::optional endpoint); + ServiceStatistics& GetServiceStatistics(const StaticServiceMetadata& metadata, + std::optional endpoint); + + MethodStatistics& GetGenericStatistics( + std::string_view call_name, std::optional endpoint); - // Can only be called on StatisticsStorage for gRPC services (not clients). - // Can only be called strictly after all the components are loaded. - // gRPC services must not be [un]registered during GetStartedRequests(). std::uint64_t GetStartedRequests() const; private: @@ -40,27 +40,59 @@ class StatisticsStorage final { using ServiceId = const char*; struct ServiceKey { - ServiceId service_id; + ServiceId service_id{}; std::optional endpoint; }; - void ExtendStatistics(utils::statistics::Writer& writer); + struct GenericKey { + std::string call_name; + std::optional endpoint; + }; - struct ServiceKeyComparer final { + struct GenericKeyView { + GenericKey Dereference() const; + + std::string_view call_name; + std::optional endpoint; + }; + + struct ServiceKeyComparer { bool operator()(ServiceKey lhs, ServiceKey rhs) const; }; - struct ServiceKeyHasher final { - std::size_t operator()(const ServiceKey& key) const; + struct ServiceKeyHasher { + std::size_t operator()(const ServiceKey& key) const noexcept; }; - const StatisticsDomain domain_; + struct GenericKeyComparer { + using is_transparent [[maybe_unused]] = void; - std::unordered_map - service_statistics_; - engine::SharedMutex mutex_; + bool operator()(const GenericKey& lhs, const GenericKey& rhs) const; + bool operator()(const GenericKeyView& lhs, const GenericKey& rhs) const; + bool operator()(const GenericKey& lhs, const GenericKeyView& rhs) const; + }; + + struct GenericKeyHasher { + using is_transparent [[maybe_unused]] = void; + + std::size_t operator()(const GenericKey& key) const noexcept; + std::size_t operator()(const GenericKeyView& key) const noexcept; + }; + void ExtendStatistics(utils::statistics::Writer& writer); + + const StatisticsDomain domain_; + utils::statistics::StripedRateCounter global_started_; + concurrent::Variable, + engine::SharedMutex> + service_statistics_; + concurrent::Variable< + utils::impl::TransparentMap, + engine::SharedMutex> + generic_statistics_; + // statistics_holder_ must be the last field. utils::statistics::Entry statistics_holder_; }; diff --git a/grpc/src/ugrpc/byte_buffer_utils.cpp b/grpc/src/ugrpc/byte_buffer_utils.cpp new file mode 100644 index 000000000000..ab2cf1534aab --- /dev/null +++ b/grpc/src/ugrpc/byte_buffer_utils.cpp @@ -0,0 +1,40 @@ +#include + +#include +#include +#include + +#include +#include + +USERVER_NAMESPACE_BEGIN + +namespace ugrpc { + +grpc::ByteBuffer SerializeToByteBuffer( + const ::google::protobuf::Message& message, std::size_t block_size) { + grpc::ByteBuffer buffer; + // NOLINTNEXTLINE(clang-analyzer-optin.cplusplus.UninitializedObject) + grpc::ProtoBufferWriter writer{ + &buffer, + /*block_size*/ utils::numeric_cast(block_size), + /*total_size*/ utils::numeric_cast(message.ByteSizeLong()), + }; + const bool success = message.SerializeToZeroCopyStream(&writer); + if (!success) { + throw std::runtime_error( + fmt::format("Failed to serialize Protobuf message of type {}", + message.GetTypeName())); + } + return buffer; +} + +bool ParseFromByteBuffer(grpc::ByteBuffer&& buffer, + ::google::protobuf::Message& message) { + grpc::ProtoBufferReader reader{&buffer}; + return message.ParseFromZeroCopyStream(&reader); +} + +} // namespace ugrpc + +USERVER_NAMESPACE_END diff --git a/grpc/src/ugrpc/client/generic.cpp b/grpc/src/ugrpc/client/generic.cpp new file mode 100644 index 000000000000..ef652d9eb6b9 --- /dev/null +++ b/grpc/src/ugrpc/client/generic.cpp @@ -0,0 +1,48 @@ +#include + +#include + +#include + +USERVER_NAMESPACE_BEGIN + +namespace ugrpc::client { + +namespace { + +struct GenericStubService final { + using Stub = grpc::GenericStub; + + static std::unique_ptr NewStub(std::shared_ptr channel) { + return std::make_unique(channel); + } +}; + +} // namespace + +GenericClient::GenericClient(impl::ClientParams&& client_params) + : impl_(std::move(client_params), impl::GenericClientTag{}, + std::in_place_type) {} + +client::UnaryCall GenericClient::UnaryCall( + std::string_view call_name, const grpc::ByteBuffer& request, + std::unique_ptr context, + const GenericOptions& generic_options) const { + auto& stub = impl_.NextStub(); + auto grpcpp_call_name = utils::StrCat("/", call_name); + return { + impl::CreateGenericCallParams(impl_, call_name, std::move(context), + generic_options.qos, + generic_options.metrics_call_name), + [&stub, &grpcpp_call_name](grpc::ClientContext* context, + const grpc::ByteBuffer& request, + grpc::CompletionQueue* cq) { + return stub.PrepareUnaryCall(context, grpcpp_call_name, request, cq); + }, + request, + }; +} + +} // namespace ugrpc::client + +USERVER_NAMESPACE_END diff --git a/grpc/src/ugrpc/client/impl/async_methods.cpp b/grpc/src/ugrpc/client/impl/async_methods.cpp index 888afe83e294..06ceab588df3 100644 --- a/grpc/src/ugrpc/client/impl/async_methods.cpp +++ b/grpc/src/ugrpc/client/impl/async_methods.cpp @@ -88,15 +88,15 @@ void FutureImpl::ClearData() noexcept { data_ = nullptr; } RpcData::RpcData(impl::CallParams&& params) : context_(std::move(params.context)), - client_name_(params.call_name), - call_name_(params.call_name), + client_name_(params.client_name), + call_name_(std::move(params.call_name)), stats_scope_(params.statistics), queue_(params.queue), config_values_(params.config), mws_(params.mws) { UASSERT(context_); UASSERT(!client_name_.empty()); - SetupSpan(span_, *context_, call_name_); + SetupSpan(span_, *context_, call_name_.Get()); } RpcData::~RpcData() { @@ -136,7 +136,7 @@ const Middlewares& RpcData::GetMiddlewares() const noexcept { std::string_view RpcData::GetCallName() const noexcept { UASSERT(context_); - return call_name_; + return call_name_.Get(); } std::string_view RpcData::GetClientName() const noexcept { @@ -282,7 +282,8 @@ void ProcessFinishResult(RpcData& data, std::move(parsed_gstatus.gstatus_string)); } } else { - data.GetStatsScope().Flush(); + data.GetSpan().AddTag("grpc_code", + std::string{ugrpc::ToString(grpc::StatusCode::OK)}); data.ResetSpan(); } } diff --git a/grpc/src/ugrpc/client/impl/call_params.cpp b/grpc/src/ugrpc/client/impl/call_params.cpp index 82879f106258..561498106847 100644 --- a/grpc/src/ugrpc/client/impl/call_params.cpp +++ b/grpc/src/ugrpc/client/impl/call_params.cpp @@ -1,19 +1,77 @@ #include +#include +#include + USERVER_NAMESPACE_BEGIN namespace ugrpc::client::impl { -CallParams DoCreateCallParams(const ClientData& client_data, - std::size_t method_id, - std::unique_ptr context) { - return CallParams{client_data.GetClientName(), - client_data.GetQueue(), - client_data.GetConfigSnapshot(), - client_data.GetMetadata().method_full_names[method_id], - std::move(context), - client_data.GetStatistics(method_id), - client_data.GetMiddlewares()}; +namespace { + +void CheckValidCallName(std::string_view call_name) { + UASSERT_MSG(!call_name.empty(), "generic call_name must NOT be empty"); + UASSERT_MSG(call_name[0] != '/', + utils::StrCat("generic call_name must NOT start with /, given: ", + call_name)); + UASSERT_MSG( + call_name.find('/') != std::string_view::npos, + utils::StrCat("generic call_name must contain /, given: ", call_name)); +} + +} // namespace + +CallParams CreateCallParams(const ClientData& client_data, + std::size_t method_id, + std::unique_ptr client_context, + const dynamic_config::Key& client_qos, + const Qos& qos) { + const auto& metadata = client_data.GetMetadata(); + const auto call_name = metadata.method_full_names[method_id]; + const auto method_name = + call_name.substr(metadata.service_full_name.size() + 1); + + const auto& config = client_data.GetConfigSnapshot(); + + // User qos goes first + ApplyQos(*client_context, qos, client_data.GetTestsuiteControl()); + + // If user qos was empty update timeout from config + ApplyQos(*client_context, config[client_qos][method_name], + client_data.GetTestsuiteControl()); + + return CallParams{ + client_data.GetClientName(), // + client_data.GetQueue(), + client_data.GetConfigSnapshot(), + {ugrpc::impl::MaybeOwnedString::Ref{}, call_name}, + std::move(client_context), + client_data.GetStatistics(method_id), + client_data.GetMiddlewares(), + }; +} + +CallParams CreateGenericCallParams( + const ClientData& client_data, std::string_view call_name, + std::unique_ptr client_context, const Qos& qos, + std::optional metrics_call_name) { + CheckValidCallName(call_name); + if (metrics_call_name) { + CheckValidCallName(*metrics_call_name); + } + + // User qos goes first + ApplyQos(*client_context, qos, client_data.GetTestsuiteControl()); + + return CallParams{ + client_data.GetClientName(), // + client_data.GetQueue(), + client_data.GetConfigSnapshot(), + ugrpc::impl::MaybeOwnedString{std::string{call_name}}, + std::move(client_context), + client_data.GetGenericStatistics(metrics_call_name.value_or(call_name)), + client_data.GetMiddlewares(), + }; } } // namespace ugrpc::client::impl diff --git a/grpc/src/ugrpc/client/impl/client_data.cpp b/grpc/src/ugrpc/client/impl/client_data.cpp new file mode 100644 index 000000000000..90ffb1ce5d61 --- /dev/null +++ b/grpc/src/ugrpc/client/impl/client_data.cpp @@ -0,0 +1,34 @@ +#include + +#include +#include + +USERVER_NAMESPACE_BEGIN + +namespace ugrpc::client::impl { + +ugrpc::impl::MethodStatistics& ClientData::GetStatistics( + std::size_t method_id) const { + UASSERT(service_statistics_); + return service_statistics_->GetMethodStatistics(method_id); +} + +ugrpc::impl::MethodStatistics& ClientData::GetGenericStatistics( + std::string_view call_name) const { + return params_.statistics_storage.GetGenericStatistics(call_name, + params_.endpoint); +} + +const ugrpc::impl::StaticServiceMetadata& ClientData::GetMetadata() const { + UASSERT(metadata_); + return *metadata_; +} + +ugrpc::impl::ServiceStatistics& ClientData::GetServiceStatistics() { + return params_.statistics_storage.GetServiceStatistics(GetMetadata(), + params_.endpoint); +} + +} // namespace ugrpc::client::impl + +USERVER_NAMESPACE_END diff --git a/grpc/src/ugrpc/client/middlewares/base.cpp b/grpc/src/ugrpc/client/middlewares/base.cpp index f641abe222ed..4d0762296ce1 100644 --- a/grpc/src/ugrpc/client/middlewares/base.cpp +++ b/grpc/src/ugrpc/client/middlewares/base.cpp @@ -40,6 +40,20 @@ const ::google::protobuf::Message* MiddlewareCallContext::GetInitialRequest() { MiddlewareFactoryBase::~MiddlewareFactoryBase() = default; +namespace impl { + +Middlewares InstantiateMiddlewares(const MiddlewareFactories& factories, + const std::string& client_name) { + Middlewares mws; + mws.reserve(factories.size()); + for (const auto& mw_factory : factories) { + mws.push_back(mw_factory->GetMiddleware(client_name)); + } + return mws; +} + +} // namespace impl + } // namespace ugrpc::client USERVER_NAMESPACE_END diff --git a/grpc/src/ugrpc/client/middlewares/log/component.cpp b/grpc/src/ugrpc/client/middlewares/log/component.cpp index 4943cf140200..d8db078f3a3e 100644 --- a/grpc/src/ugrpc/client/middlewares/log/component.cpp +++ b/grpc/src/ugrpc/client/middlewares/log/component.cpp @@ -9,18 +9,27 @@ USERVER_NAMESPACE_BEGIN namespace ugrpc::client::middlewares::log { +Settings Parse(const yaml_config::YamlConfig& config, + formats::parse::To) { + Settings settings; + settings.max_msg_size = + config["msg-size-log-limit"].As(settings.max_msg_size); + settings.log_level = + config["log-level"].As(settings.log_level); + return settings; +} + Component::Component(const components::ComponentConfig& config, const components::ComponentContext& context) : MiddlewareComponentBase(config, context), - max_size_(config["msg-size-log-limit"].As(512)) { - log_level_ = config["log-level"].As(logging::Level::kDebug); -} + settings_(config.As()) {} std::shared_ptr Component::GetMiddlewareFactory() { - return std::make_shared( - Middleware::Settings{max_size_, log_level_}); + return std::make_shared(*settings_); } +Component::~Component() = default; + yaml_config::Schema Component::GetStaticConfigSchema() { return yaml_config::MergeSchemas(R"( type: object diff --git a/grpc/src/ugrpc/client/middlewares/log/middleware.cpp b/grpc/src/ugrpc/client/middlewares/log/middleware.cpp index b718d115b1f9..c0dbe3667e57 100644 --- a/grpc/src/ugrpc/client/middlewares/log/middleware.cpp +++ b/grpc/src/ugrpc/client/middlewares/log/middleware.cpp @@ -20,7 +20,7 @@ void Middleware::Handle(MiddlewareCallContext& context) const { context.Next(); } -MiddlewareFactory::MiddlewareFactory(const Middleware::Settings& settings) +MiddlewareFactory::MiddlewareFactory(const Settings& settings) : settings_(settings) {} std::shared_ptr MiddlewareFactory::GetMiddleware( diff --git a/grpc/src/ugrpc/client/middlewares/log/middleware.hpp b/grpc/src/ugrpc/client/middlewares/log/middleware.hpp index f14629a5c2dd..5b3b93a58d90 100644 --- a/grpc/src/ugrpc/client/middlewares/log/middleware.hpp +++ b/grpc/src/ugrpc/client/middlewares/log/middleware.hpp @@ -1,19 +1,24 @@ #pragma once +#include + #include USERVER_NAMESPACE_BEGIN namespace ugrpc::client::middlewares::log { +struct Settings { + /// Max gRPC message size, the rest will be truncated + std::size_t max_msg_size{512}; + + /// gRPC message logging level + logging::Level log_level{logging::Level::kDebug}; +}; + /// @brief middleware for RPC handler logging settings class Middleware final : public MiddlewareBase { public: - struct Settings { - size_t max_msg_size; ///< Max gRPC message size, the rest will be truncated - logging::Level log_level; ///< gRPC message logging level - }; - explicit Middleware(const Settings& settings); void Handle(MiddlewareCallContext& context) const override; @@ -25,13 +30,13 @@ class Middleware final : public MiddlewareBase { /// @cond class MiddlewareFactory final : public MiddlewareFactoryBase { public: - explicit MiddlewareFactory(const Middleware::Settings& settings); + explicit MiddlewareFactory(const Settings& settings); std::shared_ptr GetMiddleware( std::string_view client_name) const override; private: - Middleware::Settings settings_; + Settings settings_; }; /// @endcond diff --git a/grpc/src/ugrpc/impl/statistics.cpp b/grpc/src/ugrpc/impl/statistics.cpp index c9a81993c365..843e18921b70 100644 --- a/grpc/src/ugrpc/impl/statistics.cpp +++ b/grpc/src/ugrpc/impl/statistics.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -47,9 +48,15 @@ void DumpMetric(utils::statistics::Writer& writer, } // namespace -MethodStatistics::MethodStatistics(StatisticsDomain domain) : domain_(domain) {} +MethodStatistics::MethodStatistics( + StatisticsDomain domain, + utils::statistics::StripedRateCounter& global_started) + : domain_(domain), global_started_(global_started) {} -void MethodStatistics::AccountStarted() noexcept { ++started_; } +void MethodStatistics::AccountStarted() noexcept { + ++started_; + ++global_started_; +} void MethodStatistics::AccountStatus(grpc::StatusCode code) noexcept { if (static_cast(code) < kCodesCount) { @@ -158,10 +165,12 @@ std::uint64_t MethodStatistics::GetStarted() const noexcept { ServiceStatistics::~ServiceStatistics() = default; -ServiceStatistics::ServiceStatistics(const StaticServiceMetadata& metadata, - StatisticsDomain domain) +ServiceStatistics::ServiceStatistics( + const StaticServiceMetadata& metadata, StatisticsDomain domain, + utils::statistics::StripedRateCounter& global_started) : metadata_(metadata), - method_statistics_(metadata.method_full_names.size(), domain) {} + method_statistics_(metadata.method_full_names.size(), domain, + global_started) {} MethodStatistics& ServiceStatistics::GetMethodStatistics( std::size_t method_id) { diff --git a/grpc/src/ugrpc/impl/statistics_storage.cpp b/grpc/src/ugrpc/impl/statistics_storage.cpp index f46080f70bcb..897f6e1deea9 100644 --- a/grpc/src/ugrpc/impl/statistics_storage.cpp +++ b/grpc/src/ugrpc/impl/statistics_storage.cpp @@ -1,6 +1,9 @@ #include +#include + #include +#include #include #include @@ -30,17 +33,39 @@ StatisticsStorage::StatisticsStorage( StatisticsStorage::~StatisticsStorage() { statistics_holder_.Unregister(); } -ugrpc::impl::ServiceStatistics& StatisticsStorage::GetServiceStatistics( - const ugrpc::impl::StaticServiceMetadata& metadata, +ServiceStatistics& StatisticsStorage::GetServiceStatistics( + const StaticServiceMetadata& metadata, std::optional endpoint) { // We exploit the fact that 'service_full_name' always points to the same // static string for a given service. const ServiceId service_id = metadata.service_full_name.data(); - const auto service_key = ServiceKey{service_id, endpoint}; + ServiceKey service_key{service_id, std::move(endpoint)}; + + { + auto service_statistics = service_statistics_.SharedMutableLockUnsafe(); + if (auto* stats = utils::FindOrNullptr(*service_statistics, service_key)) { + return *stats; + } + } + + // All the other clients are blocked while we instantiate stats for a new + // service. This is OK, because it will only happen a finite number of times + // during startup. + auto service_statistics = service_statistics_.Lock(); + + const auto [iter, is_new] = service_statistics->try_emplace( + std::move(service_key), metadata, domain_, global_started_); + return iter->second; +} + +MethodStatistics& StatisticsStorage::GetGenericStatistics( + std::string_view call_name, std::optional endpoint) { + const GenericKeyView generic_key{call_name, endpoint}; { - const std::shared_lock lock(mutex_); - if (auto* stats = utils::FindOrNullptr(service_statistics_, service_key)) { + auto generic_statistics = generic_statistics_.SharedMutableLockUnsafe(); + if (auto* stats = utils::impl::FindTransparentOrNullptr(*generic_statistics, + generic_key)) { return *stats; } } @@ -48,18 +73,18 @@ ugrpc::impl::ServiceStatistics& StatisticsStorage::GetServiceStatistics( // All the other clients are blocked while we instantiate stats for a new // service. This is OK, because it will only happen a finite number of times // during startup. - const std::lock_guard lock(mutex_); + auto generic_statistics = generic_statistics_.Lock(); - const auto [iter, is_new] = service_statistics_.try_emplace( - std::move(service_key), metadata, domain_); + const auto [iter, is_new] = generic_statistics->try_emplace( + generic_key.Dereference(), domain_, global_started_); return iter->second; } void StatisticsStorage::ExtendStatistics(utils::statistics::Writer& writer) { - const std::shared_lock lock(mutex_); + auto by_destination = writer["by-destination"]; { - auto by_destination = writer["by-destination"]; - for (const auto& [key, service_stats] : service_statistics_) { + auto service_statistics = service_statistics_.SharedLock(); + for (const auto& [key, service_stats] : *service_statistics) { if (key.endpoint) { by_destination.ValueWithLabels(std::move(service_stats), {"endpoint", *key.endpoint}); @@ -68,17 +93,46 @@ void StatisticsStorage::ExtendStatistics(utils::statistics::Writer& writer) { } } } + { + auto generic_statistics = generic_statistics_.SharedLock(); + for (const auto& [key, service_stats] : *generic_statistics) { + const std::string_view call_name = key.call_name; + + const auto slash_pos = call_name.find('/'); + if (slash_pos == std::string_view::npos || slash_pos == 0) { + UASSERT(false); + continue; + } + + const auto service_name = call_name.substr(0, slash_pos); + const auto method_name = call_name.substr(slash_pos + 1); + + if (key.endpoint) { + by_destination.ValueWithLabels(std::move(service_stats), + {{"grpc_service", service_name}, + {"grpc_method", method_name}, + {"grpc_destination", key.call_name}, + {"endpoint", *key.endpoint}}); + } else { + by_destination.ValueWithLabels(std::move(service_stats), + {{"grpc_service", service_name}, + {"grpc_method", method_name}, + {"grpc_destination", key.call_name}}); + } + } + } } std::uint64_t StatisticsStorage::GetStartedRequests() const { - std::uint64_t result{0}; - // mutex_ is not locked as we might be inside a common thread w/o coroutine - // environment. service_statistics_ is not changed as all gRPC services (not - // clients!) are already registered. - for (const auto& [name, stats] : service_statistics_) { - result += stats.GetStartedRequests(); - } - return result; + return global_started_.Load().value; +} + +StatisticsStorage::GenericKey // +StatisticsStorage::GenericKeyView::Dereference() const { + return GenericKey{ + std::string{call_name}, + endpoint ? std::make_optional(std::string{*endpoint}) : std::nullopt, + }; } bool StatisticsStorage::ServiceKeyComparer::operator()(ServiceKey lhs, @@ -87,9 +141,33 @@ bool StatisticsStorage::ServiceKeyComparer::operator()(ServiceKey lhs, } std::size_t StatisticsStorage::ServiceKeyHasher::operator()( - const ServiceKey& key) const { - return std::hash{}(key.service_id) ^ - std::hash{}(key.endpoint); + const ServiceKey& key) const noexcept { + return boost::hash_value(std::tie(key.service_id, key.endpoint)); +} + +bool StatisticsStorage::GenericKeyComparer::operator()( + const GenericKey& lhs, const GenericKey& rhs) const { + return lhs.call_name == rhs.call_name && lhs.endpoint == rhs.endpoint; +} + +bool StatisticsStorage::GenericKeyComparer::operator()( + const GenericKeyView& lhs, const GenericKey& rhs) const { + return lhs.call_name == rhs.call_name && lhs.endpoint == rhs.endpoint; +} + +bool StatisticsStorage::GenericKeyComparer::operator()( + const GenericKey& lhs, const GenericKeyView& rhs) const { + return lhs.call_name == rhs.call_name && lhs.endpoint == rhs.endpoint; +} + +std::size_t StatisticsStorage::GenericKeyHasher::operator()( + const GenericKey& key) const noexcept { + return boost::hash_value(std::tie(key.call_name, key.endpoint)); +} + +std::size_t StatisticsStorage::GenericKeyHasher::operator()( + const GenericKeyView& key) const noexcept { + return boost::hash_value(std::tie(key.call_name, key.endpoint)); } } // namespace ugrpc::impl diff --git a/grpc/tests/src/base_test.cpp b/grpc/tests/src/base_test.cpp index fb11e32a05f8..96ec8a531eca 100644 --- a/grpc/tests/src/base_test.cpp +++ b/grpc/tests/src/base_test.cpp @@ -132,6 +132,7 @@ UTEST_F(GrpcClientTest, UnaryRPC) { out.set_name("userver"); auto call_for_move = client.SayHello(out, PrepareClientContext()); auto call = std::move(call_for_move); // test move operation + EXPECT_EQ(call.GetCallName(), "sample.ugrpc.UnitTestService/SayHello"); sample::ugrpc::GreetingResponse in; UEXPECT_NO_THROW(in = call.Finish()); diff --git a/grpc/tests/src/generic_client_test.cpp b/grpc/tests/src/generic_client_test.cpp new file mode 100644 index 000000000000..e58c939649c2 --- /dev/null +++ b/grpc/tests/src/generic_client_test.cpp @@ -0,0 +1,125 @@ +#include + +#include + +#include +#include +#include +#include +#include +#include + +#include + +USERVER_NAMESPACE_BEGIN + +namespace { + +const grpc::string kSayHelloCallName = "sample.ugrpc.UnitTestService/SayHello"; + +class UnitTestService final : public sample::ugrpc::UnitTestServiceBase { + public: + void SayHello(SayHelloCall& call, + sample::ugrpc::GreetingRequest&& request) override { + sample::ugrpc::GreetingResponse response; + response.set_name("Hello " + request.name()); + call.Finish(response); + } +}; + +using GenericClientTest = ugrpc::tests::ServiceFixture; + +sample::ugrpc::GreetingResponse PerformGenericUnaryCall( + ugrpc::tests::ServiceBase& test_service) { + auto& client_factory = test_service; + + /// [sample] + const auto client = client_factory.MakeClient(); + + const grpc::string call_name = "sample.ugrpc.UnitTestService/SayHello"; + + sample::ugrpc::GreetingRequest request; + request.set_name("generic"); + + auto rpc = client.UnaryCall(call_name, ugrpc::SerializeToByteBuffer(request)); + + auto response_bytes = rpc.Finish(); + sample::ugrpc::GreetingResponse response; + if (!ugrpc::ParseFromByteBuffer(std::move(response_bytes), response)) { + throw ugrpc::client::RpcError(rpc.GetCallName(), + "Failed to parse response"); + } + + return response; + /// [sample] +} + +} // namespace + +UTEST_F(GenericClientTest, UnaryCall) { + const auto response = PerformGenericUnaryCall(*this); + EXPECT_EQ(response.name(), "Hello generic"); +} + +UTEST_F(GenericClientTest, Metrics) { + PerformGenericUnaryCall(*this); + + const auto stats = GetStatistics( + "grpc.client.by-destination", + {{"grpc_method", "SayHello"}, + {"grpc_service", "sample.ugrpc.UnitTestService"}, + {"grpc_destination", "sample.ugrpc.UnitTestService/SayHello"}}); + UEXPECT_NO_THROW( + EXPECT_EQ(stats.SingleMetric("status", {{"grpc_code", "OK"}}), + utils::statistics::Rate{1}) + << testing::PrintToString(stats)) + << testing::PrintToString(stats); +} + +UTEST_F(GenericClientTest, MetricsCustomCallName) { + const auto client = MakeClient(); + + sample::ugrpc::GreetingRequest request; + request.set_name("generic"); + + ugrpc::client::GenericOptions options; + options.metrics_call_name = "GenericService/GenericMethod"; + + auto rpc = + client.UnaryCall(kSayHelloCallName, ugrpc::SerializeToByteBuffer(request), + std::make_unique(), options); + EXPECT_EQ(rpc.GetCallName(), kSayHelloCallName); + rpc.Finish(); + + const auto stats = + GetStatistics("grpc.client.by-destination", + {{"grpc_method", "GenericMethod"}, + {"grpc_service", "GenericService"}, + {"grpc_destination", "GenericService/GenericMethod"}}); + UEXPECT_NO_THROW( + EXPECT_EQ(stats.SingleMetric("status", {{"grpc_code", "OK"}}), + utils::statistics::Rate{1}) + << testing::PrintToString(stats)) + << testing::PrintToString(stats); +} + +namespace { + +using GenericClientLoggingTest = + utest::LogCaptureFixture>; + +} // namespace + +UTEST_F(GenericClientLoggingTest, Logs) { + PerformGenericUnaryCall(*this); + + const auto span_log = GetSingleLog( + GetLogCapture().Filter("", {{{std::string_view("stopwatch_name"), + std::string_view("external_grpc")}}})); + EXPECT_EQ(span_log.GetTagOptional("stopwatch_name"), + "external_grpc/sample.ugrpc.UnitTestService/SayHello") + << span_log; + EXPECT_EQ(span_log.GetTagOptional("grpc_code"), "OK") << span_log; +} + +USERVER_NAMESPACE_END diff --git a/grpc/tests/src/logging_test.cpp b/grpc/tests/src/logging_test.cpp index 4429a57252d2..974dbf3326e9 100644 --- a/grpc/tests/src/logging_test.cpp +++ b/grpc/tests/src/logging_test.cpp @@ -70,7 +70,7 @@ UTEST_F(GrpcAccessLog, Test) { R"(grpc_status=\d+\t)" R"(grpc_status_code=[A-Z_]+\n)"; - const auto logs = member.ExtractSingle(); + const auto logs = GetSingleLog(member.GetAll()); EXPECT_TRUE( utils::regex_match(logs.GetLogRaw(), utils::regex(kExpectedPattern))) << logs; diff --git a/grpc/utest/src/ugrpc/tests/service.cpp b/grpc/utest/src/ugrpc/tests/service.cpp index 29319e422f91..9e007b95b6f0 100644 --- a/grpc/utest/src/ugrpc/tests/service.cpp +++ b/grpc/utest/src/ugrpc/tests/service.cpp @@ -15,8 +15,7 @@ namespace ugrpc::tests { namespace { using ClientLogMiddlewareFactory = client::middlewares::log::MiddlewareFactory; -using ClientLogMiddlewareSettings = - client::middlewares::log::Middleware::Settings; +using ClientLogMiddlewareSettings = client::middlewares::log::Settings; using ClientDpMiddlewareFactory = client::middlewares::deadline_propagation::MiddlewareFactory; diff --git a/scripts/grpc/templates/client.usrv.cpp.jinja b/scripts/grpc/templates/client.usrv.cpp.jinja index 572c3383159c..77e05eaa9081 100644 --- a/scripts/grpc/templates/client.usrv.cpp.jinja +++ b/scripts/grpc/templates/client.usrv.cpp.jinja @@ -57,12 +57,12 @@ constexpr const auto& k{{service.name}}ClientQosConfig = std::unique_ptr<::grpc::ClientContext> context, const USERVER_NAMESPACE::ugrpc::client::Qos& qos ) const { + auto& stub = impl_.NextStub<{{utils.namespace_with_colons(proto.namespace)}}::{{service.name}}>(); return { USERVER_NAMESPACE::ugrpc::client::impl::CreateCallParams( impl_, {{method_id}}, std::move(context), k{{service.name}}ClientQosConfig, qos ), - impl_.NextStub<{{utils.namespace_with_colons(proto.namespace)}}::{{service.name}}>(), - &{{utils.namespace_with_colons(proto.namespace)}}::{{service.name}}::Stub::PrepareAsync{{method.name}}, + [&stub](auto&&... args) { return stub.PrepareAsync{{method.name}}(std::forward(args)...); }, {% if method.client_streaming %} }; {% else %} diff --git a/scripts/grpc/templates/client.usrv.hpp.jinja b/scripts/grpc/templates/client.usrv.hpp.jinja index 781bcc4dd322..e4d42a496ef1 100644 --- a/scripts/grpc/templates/client.usrv.hpp.jinja +++ b/scripts/grpc/templates/client.usrv.hpp.jinja @@ -52,11 +52,11 @@ class {{service.name}}Client final { static USERVER_NAMESPACE::ugrpc::impl::StaticServiceMetadata GetMetadata(); private: - {# All non-template related client data should be placed in ClientData #} template friend USERVER_NAMESPACE::ugrpc::client::impl::ClientData& USERVER_NAMESPACE::ugrpc::client::impl::GetClientData(Client& client); + {# All non-template related client data should be placed in ClientData #} USERVER_NAMESPACE::ugrpc::client::impl::ClientData impl_; }; {% endfor %} diff --git a/universal/include/userver/utils/impl/source_location.hpp b/universal/include/userver/utils/impl/source_location.hpp index 71234c7c8ec7..061933647688 100644 --- a/universal/include/userver/utils/impl/source_location.hpp +++ b/universal/include/userver/utils/impl/source_location.hpp @@ -95,6 +95,8 @@ class SourceLocation final { std::string_view function_name_; }; +std::string ToString(const SourceLocation& location); + } // namespace utils::impl USERVER_NAMESPACE_END diff --git a/universal/include/userver/utils/span.hpp b/universal/include/userver/utils/span.hpp index 9545a6d1b779..c075c1569ee1 100644 --- a/universal/include/userver/utils/span.hpp +++ b/universal/include/userver/utils/span.hpp @@ -17,6 +17,18 @@ USERVER_NAMESPACE_BEGIN namespace utils { +namespace impl { + +template +struct TypeIdentityImpl final { + using type = T; +}; + +template +using TypeIdentity = typename TypeIdentityImpl::type; + +} // namespace impl + /// A polyfill for std::span from C++20 template class span final { @@ -48,7 +60,11 @@ class span final { T*>>> // NOLINTNEXTLINE(cppcoreguidelines-missing-std-forward) constexpr /*implicit*/ span(Container&& cont) noexcept - : span(std::data(cont), std::data(cont) + std::size(cont)) {} + : span(std::data(cont), std::size(cont)) {} + + template + constexpr /*implicit*/ span(impl::TypeIdentity (&array)[Size]) noexcept + : span(std::data(array), std::size(array)) {} constexpr T* begin() const noexcept { return begin_; } constexpr T* end() const noexcept { return end_; } @@ -72,8 +88,8 @@ class span final { return span{begin_ + offset, end_}; } - constexpr span subspan(std::size_t offset, std::size_t count) const - noexcept { + constexpr span subspan(std::size_t offset, // + std::size_t count) const noexcept { UASSERT(offset + count <= size()); return span{begin_ + offset, begin_ + offset + count}; } diff --git a/universal/src/utils/impl/source_location.cpp b/universal/src/utils/impl/source_location.cpp new file mode 100644 index 000000000000..92d71cf097b5 --- /dev/null +++ b/universal/src/utils/impl/source_location.cpp @@ -0,0 +1,16 @@ +#include + +#include + +USERVER_NAMESPACE_BEGIN + +namespace utils::impl { + +std::string ToString(const SourceLocation& location) { + return StrCat(location.GetFunctionName(), " (", location.GetFileName(), ":", + location.GetLineString(), ")"); +} + +} // namespace utils::impl + +USERVER_NAMESPACE_END diff --git a/universal/utest/include/userver/utest/log_capture_fixture.hpp b/universal/utest/include/userver/utest/log_capture_fixture.hpp index 742395fa64b5..baf302ee2e0e 100644 --- a/universal/utest/include/userver/utest/log_capture_fixture.hpp +++ b/universal/utest/include/userver/utest/log_capture_fixture.hpp @@ -4,6 +4,7 @@ /// @brief @copybrief utest::LogCaptureFixture #include +#include #include #include #include @@ -17,6 +18,7 @@ #include #include #include +#include #include #include @@ -42,7 +44,10 @@ class LogRecord final { const std::string& GetTag(std::string_view key) const; /// @returns decoded value of the tag in the log record, or `std::nullopt` - const std::string* GetTagOptional(std::string_view key) const; + std::optional GetTagOptional(std::string_view key) const; + + /// @returns decoded value of the tag in the log record, or `nullptr` + const std::string* GetTagOrNullptr(std::string_view key) const; /// @returns serialized log record const std::string& GetLogRaw() const; @@ -66,6 +71,18 @@ std::ostream& operator<<(std::ostream&, const LogRecord& data); std::ostream& operator<<(std::ostream&, const std::vector& data); +/// Thrown by @ref GetSingleLog. +class NotSingleLogError final : public std::runtime_error { + public: + using std::runtime_error::runtime_error; +}; + +/// @returns the only log record from `log`. +/// @throws NotSingleLogError if there are zero or multiple log records. +LogRecord GetSingleLog(utils::span log, + const utils::impl::SourceLocation& source_location = + utils::impl::SourceLocation::Current()); + /// @brief A mocked logger that stores the log records in memory. /// @see @ref utest::LogCaptureFixture class LogCaptureLogger final { @@ -76,20 +93,19 @@ class LogCaptureLogger final { logging::LoggerPtr GetLogger() const; /// @returns all collected logs. + /// @see @ref GetSingleLog std::vector GetAll() const; - /// @returns the single collected log record, then clears logs. - /// @throws std::runtime_error if there are zero or multiple log records. - LogRecord ExtractSingle(); - /// @returns logs filtered by (optional) text substring and (optional) tags /// substrings. + /// @see @ref GetSingleLog std::vector Filter( std::string_view text_substring, - utils::span> + utils::span> tag_substrings = {}) const; /// @returns logs filtered by an arbitrary predicate. + /// @see @ref GetSingleLog std::vector Filter( utils::function_ref predicate) const; @@ -101,7 +117,9 @@ class LogCaptureLogger final { std::string ToStringViaLogging(const T& value) { Clear(); LOG_CRITICAL() << value; - return ExtractSingle().GetText(); + auto text = GetSingleLog(GetAll()).GetText(); + Clear(); + return text; } private: diff --git a/universal/utest/src/utest/log_capture_fixture.cpp b/universal/utest/src/utest/log_capture_fixture.cpp index 2d1d5036fad0..153cd053e4b5 100644 --- a/universal/utest/src/utest/log_capture_fixture.cpp +++ b/universal/utest/src/utest/log_capture_fixture.cpp @@ -15,6 +15,8 @@ USERVER_NAMESPACE_BEGIN namespace utest { +namespace {} // namespace + namespace impl { class ToStringLogger : public logging::impl::LoggerBase { @@ -64,7 +66,7 @@ class ToStringLogger : public logging::impl::LoggerBase { const std::string& LogRecord::GetText() const { return GetTag("text"); } const std::string& LogRecord::GetTag(std::string_view key) const { - auto tag_value = GetTagOptional(key); + auto tag_value = GetTagOrNullptr(key); if (!tag_value) { throw std::runtime_error( fmt::format("No '{}' tag in log record:\n{}", key, log_raw_)); @@ -72,7 +74,15 @@ const std::string& LogRecord::GetTag(std::string_view key) const { return *std::move(tag_value); } -const std::string* LogRecord::GetTagOptional(std::string_view key) const { +std::optional LogRecord::GetTagOptional( + std::string_view key) const { + if (const auto* const value = GetTagOrNullptr(key)) { + return *value; + } + return std::nullopt; +} + +const std::string* LogRecord::GetTagOrNullptr(std::string_view key) const { const auto iter = std::find_if(tags_.begin(), tags_.end(), [&](const std::pair& tag) { @@ -131,6 +141,21 @@ std::ostream& operator<<(std::ostream& os, const std::vector& data) { return os; } +LogRecord GetSingleLog(utils::span log, + const utils::impl::SourceLocation& source_location) { + if (log.size() != 1) { + std::string msg = + fmt::format("There are {} log records instead of 1 at {}:\n", + log.size(), ToString(source_location)); + for (const auto& record : log) { + msg += record.GetLogRaw(); + } + throw NotSingleLogError(msg); + } + auto single_record = std::move(log[0]); + return single_record; +} + LogCaptureLogger::LogCaptureLogger(logging::Format format) : logger_(utils::MakeSharedRef(format)) {} @@ -142,29 +167,14 @@ std::vector LogCaptureLogger::GetAll() const { return logger_->GetAll(); } -LogRecord LogCaptureLogger::ExtractSingle() { - auto log = GetAll(); - if (log.size() != 1) { - std::string msg = - fmt::format("There are {} log records instead of 1:\n", log.size()); - for (const auto& record : log) { - msg += record.GetLogRaw(); - } - throw std::runtime_error(msg); - } - auto single_record = std::move(log[0]); - Clear(); - return single_record; -} - std::vector LogCaptureLogger::Filter( std::string_view text_substring, - utils::span> tag_substrings) - const { + utils::span> + tag_substrings) const { return Filter([&](const LogRecord& record) { return record.GetText().find(text_substring) != std::string_view::npos && boost::algorithm::all_of(tag_substrings, [&](const auto& kv) { - const auto* tag_value = record.GetTagOptional(kv.first); + const auto* tag_value = record.GetTagOrNullptr(kv.first); return tag_value && tag_value->find(kv.second) != std::string_view::npos; }); diff --git a/ydb/tests/small_table.hpp b/ydb/tests/small_table.hpp index 968123d27ebc..6e1b09798773 100644 --- a/ydb/tests/small_table.hpp +++ b/ydb/tests/small_table.hpp @@ -56,11 +56,6 @@ inline const std::vector kPreFilledRows = { {"key3", "value3", 3}, }; -inline std::string ToString(const utils::impl::SourceLocation& location) { - return fmt::format("at {}, {}:{}", location.GetFunctionName(), - location.GetFileName(), location.GetLineString()); -} - template void AssertNullableColumn(ydb::Row& row, std::string_view column_name, const T& expected, @@ -89,7 +84,7 @@ inline auto AssertArePreFilledRows(ydb::Cursor cursor, const utils::impl::SourceLocation& location = utils::impl::SourceLocation::Current()) { ASSERT_THAT(cursor, testing::SizeIs(indexes.size())) - << "expected " << indexes.size() << " rows in cursor " + << "expected " << indexes.size() << " rows in cursor at " << ToString(location); for (auto [pos, row] : utils::enumerate(cursor)) { From f0808d018c2a5f7c6cd15a7d587950c49d72eec3 Mon Sep 17 00:00:00 2001 From: Antony Polukhin Date: Fri, 19 Jul 2024 15:35:40 +0300 Subject: [PATCH 055/629] feat build: update Kafka for Conan and disable building it by default in Conan MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tests: протестировано CI c4eb6d26af4da7b67f0ec7354ed2bad0e8c817d2 Pull Request resolved: https://github.com/userver-framework/userver/pull/650 --- conanfile.py | 2 +- kafka/CMakeLists.txt | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/conanfile.py b/conanfile.py index f794d4737f3d..e26fdd7e0131 100644 --- a/conanfile.py +++ b/conanfile.py @@ -57,7 +57,7 @@ class UserverConan(ConanFile): 'with_clickhouse': True, 'with_rabbitmq': True, 'with_utest': True, - 'with_kafka': True, + 'with_kafka': False, 'namespace': 'userver', 'namespace_begin': 'namespace userver {', 'namespace_end': '}', diff --git a/kafka/CMakeLists.txt b/kafka/CMakeLists.txt index 4822b209f2e6..5b20bb79fc15 100644 --- a/kafka/CMakeLists.txt +++ b/kafka/CMakeLists.txt @@ -7,7 +7,13 @@ file(GLOB_RECURSE SOURCES add_library(${PROJECT_NAME} STATIC ${SOURCES}) -include(SetupRdKafka) +if (USERVER_CONAN) + find_package(RdKafka REQUIRED) + set_target_properties(RdKafka::rdkafka++ PROPERTIES IMPORTED_GLOBAL TRUE) + add_library(rdkafka ALIAS RdKafka::rdkafka++) +else() + include(SetupRdKafka) +endif() target_compile_options(${PROJECT_NAME} PRIVATE "-Wno-ignored-qualifiers") target_link_libraries(${PROJECT_NAME} From 5ee5aa4c6db4e3da42a5983e73cb5134bb440944 Mon Sep 17 00:00:00 2001 From: segoon Date: Fri, 19 Jul 2024 18:16:22 +0300 Subject: [PATCH 056/629] feat chaotic: more golden tests d049171e77774ead49693917a9b8a2bc7bd0d850 --- .mapping.json | 30 +++- .../output/schemas/allof/allof.cpp | 154 ++++++++++++++++++ .../output/schemas/allof/allof.hpp | 97 +++++++++++ .../output/schemas/allof/allof_fwd.hpp | 7 + .../output/schemas/allof/allof_parsers.ipp | 83 ++++++++++ .../golden_tests/output/schemas/enum/enum.cpp | 106 ++++++++++++ .../golden_tests/output/schemas/enum/enum.hpp | 74 +++++++++ .../output/schemas/enum/enum_fwd.hpp | 7 + .../output/schemas/enum/enum_parsers.ipp | 51 ++++++ .../output/schemas/{ => int}/int.cpp | 0 .../output/schemas/{ => int}/int.hpp | 0 .../output/schemas/{ => int}/int_fwd.hpp | 0 .../output/schemas/{ => int}/int_parsers.ipp | 0 .../output/schemas/oneof/oneof.cpp | 58 +++++++ .../output/schemas/oneof/oneof.hpp | 43 +++++ .../output/schemas/oneof/oneof_fwd.hpp | 7 + .../output/schemas/oneof/oneof_parsers.ipp | 30 ++++ .../output/schemas/string/string.cpp | 55 +++++++ .../output/schemas/string/string.hpp | 38 +++++ .../output/schemas/string/string_fwd.hpp | 7 + .../output/schemas/string/string_parsers.ipp | 30 ++++ chaotic/golden_tests/schemas/allof/allof.yaml | 18 ++ chaotic/golden_tests/schemas/enum/enum.yaml | 12 ++ .../golden_tests/schemas/{ => int}/int.yaml | 0 chaotic/golden_tests/schemas/oneof/oneof.yaml | 11 ++ .../golden_tests/schemas/string/string.yaml | 8 + 26 files changed, 921 insertions(+), 5 deletions(-) create mode 100644 chaotic/golden_tests/output/schemas/allof/allof.cpp create mode 100644 chaotic/golden_tests/output/schemas/allof/allof.hpp create mode 100644 chaotic/golden_tests/output/schemas/allof/allof_fwd.hpp create mode 100644 chaotic/golden_tests/output/schemas/allof/allof_parsers.ipp create mode 100644 chaotic/golden_tests/output/schemas/enum/enum.cpp create mode 100644 chaotic/golden_tests/output/schemas/enum/enum.hpp create mode 100644 chaotic/golden_tests/output/schemas/enum/enum_fwd.hpp create mode 100644 chaotic/golden_tests/output/schemas/enum/enum_parsers.ipp rename chaotic/golden_tests/output/schemas/{ => int}/int.cpp (100%) rename chaotic/golden_tests/output/schemas/{ => int}/int.hpp (100%) rename chaotic/golden_tests/output/schemas/{ => int}/int_fwd.hpp (100%) rename chaotic/golden_tests/output/schemas/{ => int}/int_parsers.ipp (100%) create mode 100644 chaotic/golden_tests/output/schemas/oneof/oneof.cpp create mode 100644 chaotic/golden_tests/output/schemas/oneof/oneof.hpp create mode 100644 chaotic/golden_tests/output/schemas/oneof/oneof_fwd.hpp create mode 100644 chaotic/golden_tests/output/schemas/oneof/oneof_parsers.ipp create mode 100644 chaotic/golden_tests/output/schemas/string/string.cpp create mode 100644 chaotic/golden_tests/output/schemas/string/string.hpp create mode 100644 chaotic/golden_tests/output/schemas/string/string_fwd.hpp create mode 100644 chaotic/golden_tests/output/schemas/string/string_parsers.ipp create mode 100644 chaotic/golden_tests/schemas/allof/allof.yaml create mode 100644 chaotic/golden_tests/schemas/enum/enum.yaml rename chaotic/golden_tests/schemas/{ => int}/int.yaml (100%) create mode 100644 chaotic/golden_tests/schemas/oneof/oneof.yaml create mode 100644 chaotic/golden_tests/schemas/string/string.yaml diff --git a/.mapping.json b/.mapping.json index e056b493e9b1..0e836cb5c44f 100644 --- a/.mapping.json +++ b/.mapping.json @@ -51,11 +51,31 @@ "chaotic/chaotic/front/types.py":"taxi/uservices/userver/chaotic/chaotic/front/types.py", "chaotic/chaotic/main.py":"taxi/uservices/userver/chaotic/chaotic/main.py", "chaotic/golden_tests/CMakeLists.txt":"taxi/uservices/userver/chaotic/golden_tests/CMakeLists.txt", - "chaotic/golden_tests/output/schemas/int.cpp":"taxi/uservices/userver/chaotic/golden_tests/output/schemas/int.cpp", - "chaotic/golden_tests/output/schemas/int.hpp":"taxi/uservices/userver/chaotic/golden_tests/output/schemas/int.hpp", - "chaotic/golden_tests/output/schemas/int_fwd.hpp":"taxi/uservices/userver/chaotic/golden_tests/output/schemas/int_fwd.hpp", - "chaotic/golden_tests/output/schemas/int_parsers.ipp":"taxi/uservices/userver/chaotic/golden_tests/output/schemas/int_parsers.ipp", - "chaotic/golden_tests/schemas/int.yaml":"taxi/uservices/userver/chaotic/golden_tests/schemas/int.yaml", + "chaotic/golden_tests/output/schemas/allof/allof.cpp":"taxi/uservices/userver/chaotic/golden_tests/output/schemas/allof/allof.cpp", + "chaotic/golden_tests/output/schemas/allof/allof.hpp":"taxi/uservices/userver/chaotic/golden_tests/output/schemas/allof/allof.hpp", + "chaotic/golden_tests/output/schemas/allof/allof_fwd.hpp":"taxi/uservices/userver/chaotic/golden_tests/output/schemas/allof/allof_fwd.hpp", + "chaotic/golden_tests/output/schemas/allof/allof_parsers.ipp":"taxi/uservices/userver/chaotic/golden_tests/output/schemas/allof/allof_parsers.ipp", + "chaotic/golden_tests/output/schemas/enum/enum.cpp":"taxi/uservices/userver/chaotic/golden_tests/output/schemas/enum/enum.cpp", + "chaotic/golden_tests/output/schemas/enum/enum.hpp":"taxi/uservices/userver/chaotic/golden_tests/output/schemas/enum/enum.hpp", + "chaotic/golden_tests/output/schemas/enum/enum_fwd.hpp":"taxi/uservices/userver/chaotic/golden_tests/output/schemas/enum/enum_fwd.hpp", + "chaotic/golden_tests/output/schemas/enum/enum_parsers.ipp":"taxi/uservices/userver/chaotic/golden_tests/output/schemas/enum/enum_parsers.ipp", + "chaotic/golden_tests/output/schemas/int/int.cpp":"taxi/uservices/userver/chaotic/golden_tests/output/schemas/int/int.cpp", + "chaotic/golden_tests/output/schemas/int/int.hpp":"taxi/uservices/userver/chaotic/golden_tests/output/schemas/int/int.hpp", + "chaotic/golden_tests/output/schemas/int/int_fwd.hpp":"taxi/uservices/userver/chaotic/golden_tests/output/schemas/int/int_fwd.hpp", + "chaotic/golden_tests/output/schemas/int/int_parsers.ipp":"taxi/uservices/userver/chaotic/golden_tests/output/schemas/int/int_parsers.ipp", + "chaotic/golden_tests/output/schemas/oneof/oneof.cpp":"taxi/uservices/userver/chaotic/golden_tests/output/schemas/oneof/oneof.cpp", + "chaotic/golden_tests/output/schemas/oneof/oneof.hpp":"taxi/uservices/userver/chaotic/golden_tests/output/schemas/oneof/oneof.hpp", + "chaotic/golden_tests/output/schemas/oneof/oneof_fwd.hpp":"taxi/uservices/userver/chaotic/golden_tests/output/schemas/oneof/oneof_fwd.hpp", + "chaotic/golden_tests/output/schemas/oneof/oneof_parsers.ipp":"taxi/uservices/userver/chaotic/golden_tests/output/schemas/oneof/oneof_parsers.ipp", + "chaotic/golden_tests/output/schemas/string/string.cpp":"taxi/uservices/userver/chaotic/golden_tests/output/schemas/string/string.cpp", + "chaotic/golden_tests/output/schemas/string/string.hpp":"taxi/uservices/userver/chaotic/golden_tests/output/schemas/string/string.hpp", + "chaotic/golden_tests/output/schemas/string/string_fwd.hpp":"taxi/uservices/userver/chaotic/golden_tests/output/schemas/string/string_fwd.hpp", + "chaotic/golden_tests/output/schemas/string/string_parsers.ipp":"taxi/uservices/userver/chaotic/golden_tests/output/schemas/string/string_parsers.ipp", + "chaotic/golden_tests/schemas/allof/allof.yaml":"taxi/uservices/userver/chaotic/golden_tests/schemas/allof/allof.yaml", + "chaotic/golden_tests/schemas/enum/enum.yaml":"taxi/uservices/userver/chaotic/golden_tests/schemas/enum/enum.yaml", + "chaotic/golden_tests/schemas/int/int.yaml":"taxi/uservices/userver/chaotic/golden_tests/schemas/int/int.yaml", + "chaotic/golden_tests/schemas/oneof/oneof.yaml":"taxi/uservices/userver/chaotic/golden_tests/schemas/oneof/oneof.yaml", + "chaotic/golden_tests/schemas/string/string.yaml":"taxi/uservices/userver/chaotic/golden_tests/schemas/string/string.yaml", "chaotic/include/userver/chaotic/array.hpp":"taxi/uservices/userver/chaotic/include/userver/chaotic/array.hpp", "chaotic/include/userver/chaotic/convert.hpp":"taxi/uservices/userver/chaotic/include/userver/chaotic/convert.hpp", "chaotic/include/userver/chaotic/convert/to.hpp":"taxi/uservices/userver/chaotic/include/userver/chaotic/convert/to.hpp", diff --git a/chaotic/golden_tests/output/schemas/allof/allof.cpp b/chaotic/golden_tests/output/schemas/allof/allof.cpp new file mode 100644 index 000000000000..e32117ac6d74 --- /dev/null +++ b/chaotic/golden_tests/output/schemas/allof/allof.cpp @@ -0,0 +1,154 @@ +#include "allof.hpp" + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "allof_parsers.ipp" + +namespace ns { + +bool operator==(const ns::AllOf::Foo__P0& lhs, const ns::AllOf::Foo__P0& rhs) { + return lhs.foo == rhs.foo && lhs.extra == rhs.extra && + + true; +} + +bool operator==(const ns::AllOf::Foo__P1& lhs, const ns::AllOf::Foo__P1& rhs) { + return lhs.bar == rhs.bar && lhs.extra == rhs.extra && + + true; +} + +bool operator==(const ns::AllOf::Foo& lhs, const ns::AllOf::Foo& rhs) { + return static_cast(lhs) == + static_cast(rhs) && + static_cast(lhs) == + static_cast(rhs); +} + +bool operator==(const ns::AllOf& lhs, const ns::AllOf& rhs) { + return lhs.foo == rhs.foo && true; +} + +USERVER_NAMESPACE::logging::LogHelper& operator<<( + USERVER_NAMESPACE::logging::LogHelper& lh, + const ns::AllOf::Foo__P0& value) { + return lh << ToString(USERVER_NAMESPACE::formats::json::ValueBuilder(value) + .ExtractValue()); +} + +USERVER_NAMESPACE::logging::LogHelper& operator<<( + USERVER_NAMESPACE::logging::LogHelper& lh, + const ns::AllOf::Foo__P1& value) { + return lh << ToString(USERVER_NAMESPACE::formats::json::ValueBuilder(value) + .ExtractValue()); +} + +USERVER_NAMESPACE::logging::LogHelper& operator<<( + USERVER_NAMESPACE::logging::LogHelper& lh, const ns::AllOf::Foo& value) { + return lh << ToString(USERVER_NAMESPACE::formats::json::ValueBuilder(value) + .ExtractValue()); +} + +USERVER_NAMESPACE::logging::LogHelper& operator<<( + USERVER_NAMESPACE::logging::LogHelper& lh, const ns::AllOf& value) { + return lh << ToString(USERVER_NAMESPACE::formats::json::ValueBuilder(value) + .ExtractValue()); +} + +AllOf::Foo__P0 Parse( + USERVER_NAMESPACE::formats::json::Value json, + USERVER_NAMESPACE::formats::parse::To to) { + return Parse(json, to); +} + +AllOf::Foo__P1 Parse( + USERVER_NAMESPACE::formats::json::Value json, + USERVER_NAMESPACE::formats::parse::To to) { + return Parse(json, to); +} + +AllOf::Foo Parse(USERVER_NAMESPACE::formats::json::Value json, + USERVER_NAMESPACE::formats::parse::To to) { + return Parse(json, to); +} + +AllOf Parse(USERVER_NAMESPACE::formats::json::Value json, + USERVER_NAMESPACE::formats::parse::To to) { + return Parse(json, to); +} + +/* Parse(USERVER_NAMESPACE::formats::yaml::Value, To) was not + * generated: ns::AllOf@Foo__P0 has JSON-specific field "extra" */ + +/* Parse(USERVER_NAMESPACE::yaml_config::Value, To) was not + * generated: ns::AllOf@Foo__P0 has JSON-specific field "extra" */ + +USERVER_NAMESPACE::formats::json::Value Serialize( + [[maybe_unused]] const ns::AllOf::Foo__P0& value, + USERVER_NAMESPACE::formats::serialize::To< + USERVER_NAMESPACE::formats::json::Value>) { + USERVER_NAMESPACE::formats::json::ValueBuilder vb = value.extra; + + if (value.foo) { + vb["foo"] = + USERVER_NAMESPACE::chaotic::Primitive{value.foo.value()}; + } + + return vb.ExtractValue(); +} + +USERVER_NAMESPACE::formats::json::Value Serialize( + [[maybe_unused]] const ns::AllOf::Foo__P1& value, + USERVER_NAMESPACE::formats::serialize::To< + USERVER_NAMESPACE::formats::json::Value>) { + USERVER_NAMESPACE::formats::json::ValueBuilder vb = value.extra; + + if (value.bar) { + vb["bar"] = USERVER_NAMESPACE::chaotic::Primitive{value.bar.value()}; + } + + return vb.ExtractValue(); +} + +USERVER_NAMESPACE::formats::json::Value Serialize( + const ns::AllOf::Foo& value, USERVER_NAMESPACE::formats::serialize::To< + USERVER_NAMESPACE::formats::json::Value>) { + USERVER_NAMESPACE::formats::json::ValueBuilder vb = + USERVER_NAMESPACE::formats::common::Type::kObject; + USERVER_NAMESPACE::formats::common::Merge( + vb, + USERVER_NAMESPACE::formats::json::ValueBuilder{ + static_cast(value)} + .ExtractValue()); + USERVER_NAMESPACE::formats::common::Merge( + vb, + USERVER_NAMESPACE::formats::json::ValueBuilder{ + static_cast(value)} + .ExtractValue()); + return vb.ExtractValue(); +} + +USERVER_NAMESPACE::formats::json::Value Serialize( + [[maybe_unused]] const ns::AllOf& value, + USERVER_NAMESPACE::formats::serialize::To< + USERVER_NAMESPACE::formats::json::Value>) { + USERVER_NAMESPACE::formats::json::ValueBuilder vb = + USERVER_NAMESPACE::formats::common::Type::kObject; + + if (value.foo) { + vb["foo"] = USERVER_NAMESPACE::chaotic::Primitive{ + value.foo.value()}; + } + + return vb.ExtractValue(); +} + +} // namespace ns diff --git a/chaotic/golden_tests/output/schemas/allof/allof.hpp b/chaotic/golden_tests/output/schemas/allof/allof.hpp new file mode 100644 index 000000000000..b6aa69a93ef1 --- /dev/null +++ b/chaotic/golden_tests/output/schemas/allof/allof.hpp @@ -0,0 +1,97 @@ +#pragma once + +#include "allof_fwd.hpp" + +#include +#include +#include +#include + +#include + +namespace ns { +namespace impl {} +} // namespace ns +namespace ns { + +struct AllOf { + struct Foo__P0 { + std::optional foo{}; + + USERVER_NAMESPACE::formats::json::Value extra; + }; + + struct Foo__P1 { + std::optional bar{}; + + USERVER_NAMESPACE::formats::json::Value extra; + }; + + struct Foo : public ns::AllOf::Foo__P0, public ns::AllOf::Foo__P1 { + Foo() = default; + + Foo(ns::AllOf::Foo__P0&& a0, ns::AllOf::Foo__P1&& a1) + : ns::AllOf::Foo__P0(std::move(a0)), + ns::AllOf::Foo__P1(std::move(a1)) {} + }; + + std::optional foo{}; +}; + +bool operator==(const ns::AllOf::Foo__P0& lhs, const ns::AllOf::Foo__P0& rhs); + +bool operator==(const ns::AllOf::Foo__P1& lhs, const ns::AllOf::Foo__P1& rhs); + +bool operator==(const ns::AllOf::Foo& lhs, const ns::AllOf::Foo& rhs); + +bool operator==(const ns::AllOf& lhs, const ns::AllOf& rhs); + +USERVER_NAMESPACE::logging::LogHelper& operator<<( + USERVER_NAMESPACE::logging::LogHelper& lh, const ns::AllOf::Foo__P0& value); + +USERVER_NAMESPACE::logging::LogHelper& operator<<( + USERVER_NAMESPACE::logging::LogHelper& lh, const ns::AllOf::Foo__P1& value); + +USERVER_NAMESPACE::logging::LogHelper& operator<<( + USERVER_NAMESPACE::logging::LogHelper& lh, const ns::AllOf::Foo& value); + +USERVER_NAMESPACE::logging::LogHelper& operator<<( + USERVER_NAMESPACE::logging::LogHelper& lh, const ns::AllOf& value); + +AllOf::Foo__P0 Parse(USERVER_NAMESPACE::formats::json::Value json, + USERVER_NAMESPACE::formats::parse::To); + +AllOf::Foo__P1 Parse(USERVER_NAMESPACE::formats::json::Value json, + USERVER_NAMESPACE::formats::parse::To); + +AllOf::Foo Parse(USERVER_NAMESPACE::formats::json::Value json, + USERVER_NAMESPACE::formats::parse::To); + +AllOf Parse(USERVER_NAMESPACE::formats::json::Value json, + USERVER_NAMESPACE::formats::parse::To); + +/* Parse(USERVER_NAMESPACE::formats::yaml::Value, To) was not + * generated: ns::AllOf@Foo__P0 has JSON-specific field "extra" */ + +/* Parse(USERVER_NAMESPACE::yaml_config::Value, To) was not + * generated: ns::AllOf@Foo__P0 has JSON-specific field "extra" */ + +USERVER_NAMESPACE::formats::json::Value Serialize( + const ns::AllOf::Foo__P0& value, + USERVER_NAMESPACE::formats::serialize::To< + USERVER_NAMESPACE::formats::json::Value>); + +USERVER_NAMESPACE::formats::json::Value Serialize( + const ns::AllOf::Foo__P1& value, + USERVER_NAMESPACE::formats::serialize::To< + USERVER_NAMESPACE::formats::json::Value>); + +USERVER_NAMESPACE::formats::json::Value Serialize( + const ns::AllOf::Foo& value, USERVER_NAMESPACE::formats::serialize::To< + USERVER_NAMESPACE::formats::json::Value>); + +USERVER_NAMESPACE::formats::json::Value Serialize( + const ns::AllOf& value, USERVER_NAMESPACE::formats::serialize::To< + USERVER_NAMESPACE::formats::json::Value>); + +} // namespace ns diff --git a/chaotic/golden_tests/output/schemas/allof/allof_fwd.hpp b/chaotic/golden_tests/output/schemas/allof/allof_fwd.hpp new file mode 100644 index 000000000000..a3f5fd1726e6 --- /dev/null +++ b/chaotic/golden_tests/output/schemas/allof/allof_fwd.hpp @@ -0,0 +1,7 @@ +#pragma once + +namespace ns { + +struct AllOf; + +} diff --git a/chaotic/golden_tests/output/schemas/allof/allof_parsers.ipp b/chaotic/golden_tests/output/schemas/allof/allof_parsers.ipp new file mode 100644 index 000000000000..bad4fb318097 --- /dev/null +++ b/chaotic/golden_tests/output/schemas/allof/allof_parsers.ipp @@ -0,0 +1,83 @@ +#pragma once + +#include "allof.hpp" + +namespace ns { + +static constexpr USERVER_NAMESPACE::utils::TrivialSet + kns__AllOf__Foo__P0_PropertiesNames = [](auto selector) { + return selector().template Type().Case("foo"); + }; + +static constexpr USERVER_NAMESPACE::utils::TrivialSet + kns__AllOf__Foo__P1_PropertiesNames = [](auto selector) { + return selector().template Type().Case("bar"); + }; + +static constexpr USERVER_NAMESPACE::utils::TrivialSet + kns__AllOf_PropertiesNames = [](auto selector) { + return selector().template Type().Case("foo"); + }; + +template +ns::AllOf::Foo__P0 Parse( + Value value, USERVER_NAMESPACE::formats::parse::To) { + value.CheckNotMissing(); + value.CheckObjectOrNull(); + + ns::AllOf::Foo__P0 res; + + res.foo = value["foo"] + .template As>>(); + + res.extra = USERVER_NAMESPACE::chaotic::ExtractAdditionalPropertiesTrue( + value, kns__AllOf__Foo__P0_PropertiesNames); + + return res; +} + +template +ns::AllOf::Foo__P1 Parse( + Value value, USERVER_NAMESPACE::formats::parse::To) { + value.CheckNotMissing(); + value.CheckObjectOrNull(); + + ns::AllOf::Foo__P1 res; + + res.bar = + value["bar"] + .template As< + std::optional>>(); + + res.extra = USERVER_NAMESPACE::chaotic::ExtractAdditionalPropertiesTrue( + value, kns__AllOf__Foo__P1_PropertiesNames); + + return res; +} + +template +ns::AllOf::Foo Parse(Value value, + USERVER_NAMESPACE::formats::parse::To) { + return ns::AllOf::Foo(value.template As(), + value.template As()); +} + +template +ns::AllOf Parse(Value value, USERVER_NAMESPACE::formats::parse::To) { + value.CheckNotMissing(); + value.CheckObjectOrNull(); + + ns::AllOf res; + + res.foo = value["foo"] + .template As>>(); + + USERVER_NAMESPACE::chaotic::ValidateNoAdditionalProperties( + value, kns__AllOf_PropertiesNames); + + return res; +} + +} // namespace ns diff --git a/chaotic/golden_tests/output/schemas/enum/enum.cpp b/chaotic/golden_tests/output/schemas/enum/enum.cpp new file mode 100644 index 000000000000..b2e2e28d4b74 --- /dev/null +++ b/chaotic/golden_tests/output/schemas/enum/enum.cpp @@ -0,0 +1,106 @@ +#include "enum.hpp" + +#include + +#include +#include +#include +#include +#include +#include + +#include "enum_parsers.ipp" + +namespace ns { + +bool operator==(const ns::Enum& lhs, const ns::Enum& rhs) { + return lhs.foo == rhs.foo && true; +} + +USERVER_NAMESPACE::logging::LogHelper& operator<<( + USERVER_NAMESPACE::logging::LogHelper& lh, const ns::Enum::Foo& value) { + return lh << ToString(value); +} + +USERVER_NAMESPACE::logging::LogHelper& operator<<( + USERVER_NAMESPACE::logging::LogHelper& lh, const ns::Enum& value) { + return lh << ToString(USERVER_NAMESPACE::formats::json::ValueBuilder(value) + .ExtractValue()); +} + +Enum::Foo Parse(USERVER_NAMESPACE::formats::json::Value json, + USERVER_NAMESPACE::formats::parse::To to) { + return Parse(json, to); +} + +Enum Parse(USERVER_NAMESPACE::formats::json::Value json, + USERVER_NAMESPACE::formats::parse::To to) { + return Parse(json, to); +} + +Enum::Foo Parse(USERVER_NAMESPACE::formats::yaml::Value json, + USERVER_NAMESPACE::formats::parse::To to) { + return Parse(json, to); +} + +Enum Parse(USERVER_NAMESPACE::formats::yaml::Value json, + USERVER_NAMESPACE::formats::parse::To to) { + return Parse(json, to); +} + +Enum::Foo Parse(USERVER_NAMESPACE::yaml_config::Value json, + USERVER_NAMESPACE::formats::parse::To to) { + return Parse(json, to); +} + +Enum Parse(USERVER_NAMESPACE::yaml_config::Value json, + USERVER_NAMESPACE::formats::parse::To to) { + return Parse(json, to); +} + +ns::Enum::Foo FromString(std::string_view value, + USERVER_NAMESPACE::formats::parse::To) { + const auto result = kns__Enum__Foo_Mapping.TryFindBySecond(value); + if (result.has_value()) { + return result.value(); + } + throw std::runtime_error( + fmt::format("Invalid enum value ({}) for type ns::Enum::Foo", value)); +} + +ns::Enum::Foo Parse(std::string_view value, + USERVER_NAMESPACE::formats::parse::To to) { + return FromString(value, to); +} + +USERVER_NAMESPACE::formats::json::Value Serialize( + const ns::Enum::Foo& value, USERVER_NAMESPACE::formats::serialize::To< + USERVER_NAMESPACE::formats::json::Value>) { + return USERVER_NAMESPACE::formats::json::ValueBuilder(ToString(value)) + .ExtractValue(); +} + +USERVER_NAMESPACE::formats::json::Value Serialize( + [[maybe_unused]] const ns::Enum& value, + USERVER_NAMESPACE::formats::serialize::To< + USERVER_NAMESPACE::formats::json::Value>) { + USERVER_NAMESPACE::formats::json::ValueBuilder vb = + USERVER_NAMESPACE::formats::common::Type::kObject; + + if (value.foo) { + vb["foo"] = + USERVER_NAMESPACE::chaotic::Primitive{value.foo.value()}; + } + + return vb.ExtractValue(); +} + +std::string ToString(ns::Enum::Foo value) { + const auto result = kns__Enum__Foo_Mapping.TryFindByFirst(value); + if (result.has_value()) { + return std::string{result.value()}; + } + throw std::runtime_error("Bad enum value"); +} + +} // namespace ns diff --git a/chaotic/golden_tests/output/schemas/enum/enum.hpp b/chaotic/golden_tests/output/schemas/enum/enum.hpp new file mode 100644 index 000000000000..23bca7c50e5d --- /dev/null +++ b/chaotic/golden_tests/output/schemas/enum/enum.hpp @@ -0,0 +1,74 @@ +#pragma once + +#include "enum_fwd.hpp" + +#include +#include +#include + +#include + +namespace ns { +namespace impl {} +} // namespace ns +namespace ns { + +struct Enum { + enum class Foo { + kOne, + kTwo, + kThree, + }; + + static constexpr Foo kFooValues[] = { + Foo::kOne, + Foo::kTwo, + Foo::kThree, + }; + + std::optional foo{}; +}; + +bool operator==(const ns::Enum& lhs, const ns::Enum& rhs); + +USERVER_NAMESPACE::logging::LogHelper& operator<<( + USERVER_NAMESPACE::logging::LogHelper& lh, const ns::Enum::Foo& value); + +USERVER_NAMESPACE::logging::LogHelper& operator<<( + USERVER_NAMESPACE::logging::LogHelper& lh, const ns::Enum& value); + +Enum::Foo Parse(USERVER_NAMESPACE::formats::json::Value json, + USERVER_NAMESPACE::formats::parse::To); + +Enum Parse(USERVER_NAMESPACE::formats::json::Value json, + USERVER_NAMESPACE::formats::parse::To); + +Enum::Foo Parse(USERVER_NAMESPACE::formats::yaml::Value json, + USERVER_NAMESPACE::formats::parse::To); + +Enum Parse(USERVER_NAMESPACE::formats::yaml::Value json, + USERVER_NAMESPACE::formats::parse::To); + +Enum::Foo Parse(USERVER_NAMESPACE::yaml_config::Value json, + USERVER_NAMESPACE::formats::parse::To); + +Enum Parse(USERVER_NAMESPACE::yaml_config::Value json, + USERVER_NAMESPACE::formats::parse::To); + +Enum::Foo FromString(std::string_view value, + USERVER_NAMESPACE::formats::parse::To); + +Enum::Foo Parse(std::string_view value, + USERVER_NAMESPACE::formats::parse::To); + +USERVER_NAMESPACE::formats::json::Value Serialize( + const ns::Enum::Foo& value, USERVER_NAMESPACE::formats::serialize::To< + USERVER_NAMESPACE::formats::json::Value>); + +USERVER_NAMESPACE::formats::json::Value Serialize( + const ns::Enum& value, USERVER_NAMESPACE::formats::serialize::To< + USERVER_NAMESPACE::formats::json::Value>); + +std::string ToString(ns::Enum::Foo value); + +} // namespace ns diff --git a/chaotic/golden_tests/output/schemas/enum/enum_fwd.hpp b/chaotic/golden_tests/output/schemas/enum/enum_fwd.hpp new file mode 100644 index 000000000000..8bf7eb0dfa9c --- /dev/null +++ b/chaotic/golden_tests/output/schemas/enum/enum_fwd.hpp @@ -0,0 +1,7 @@ +#pragma once + +namespace ns { + +struct Enum; + +} diff --git a/chaotic/golden_tests/output/schemas/enum/enum_parsers.ipp b/chaotic/golden_tests/output/schemas/enum/enum_parsers.ipp new file mode 100644 index 000000000000..3bb782dddb07 --- /dev/null +++ b/chaotic/golden_tests/output/schemas/enum/enum_parsers.ipp @@ -0,0 +1,51 @@ +#pragma once + +#include "enum.hpp" + +namespace ns { + +static constexpr USERVER_NAMESPACE::utils::TrivialBiMap kns__Enum__Foo_Mapping = + [](auto selector) { + return selector() + .template Type() + .Case(ns::Enum::Foo::kOne, "one") + .Case(ns::Enum::Foo::kTwo, "two") + .Case(ns::Enum::Foo::kThree, "three"); + }; + +static constexpr USERVER_NAMESPACE::utils::TrivialSet + kns__Enum_PropertiesNames = [](auto selector) { + return selector().template Type().Case("foo"); + }; + +template +ns::Enum::Foo Parse(Value val, + USERVER_NAMESPACE::formats::parse::To) { + const auto value = val.template As(); + const auto result = kns__Enum__Foo_Mapping.TryFindBySecond(value); + if (result.has_value()) { + return result.value(); + } + USERVER_NAMESPACE::chaotic::ThrowForValue( + fmt::format("Invalid enum value ({}) for type ns::Enum::Foo", value), + val); +} + +template +ns::Enum Parse(Value value, USERVER_NAMESPACE::formats::parse::To) { + value.CheckNotMissing(); + value.CheckObjectOrNull(); + + ns::Enum res; + + res.foo = value["foo"] + .template As>>(); + + USERVER_NAMESPACE::chaotic::ValidateNoAdditionalProperties( + value, kns__Enum_PropertiesNames); + + return res; +} + +} // namespace ns diff --git a/chaotic/golden_tests/output/schemas/int.cpp b/chaotic/golden_tests/output/schemas/int/int.cpp similarity index 100% rename from chaotic/golden_tests/output/schemas/int.cpp rename to chaotic/golden_tests/output/schemas/int/int.cpp diff --git a/chaotic/golden_tests/output/schemas/int.hpp b/chaotic/golden_tests/output/schemas/int/int.hpp similarity index 100% rename from chaotic/golden_tests/output/schemas/int.hpp rename to chaotic/golden_tests/output/schemas/int/int.hpp diff --git a/chaotic/golden_tests/output/schemas/int_fwd.hpp b/chaotic/golden_tests/output/schemas/int/int_fwd.hpp similarity index 100% rename from chaotic/golden_tests/output/schemas/int_fwd.hpp rename to chaotic/golden_tests/output/schemas/int/int_fwd.hpp diff --git a/chaotic/golden_tests/output/schemas/int_parsers.ipp b/chaotic/golden_tests/output/schemas/int/int_parsers.ipp similarity index 100% rename from chaotic/golden_tests/output/schemas/int_parsers.ipp rename to chaotic/golden_tests/output/schemas/int/int_parsers.ipp diff --git a/chaotic/golden_tests/output/schemas/oneof/oneof.cpp b/chaotic/golden_tests/output/schemas/oneof/oneof.cpp new file mode 100644 index 000000000000..ce154609b7f8 --- /dev/null +++ b/chaotic/golden_tests/output/schemas/oneof/oneof.cpp @@ -0,0 +1,58 @@ +#include "oneof.hpp" + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "oneof_parsers.ipp" + +namespace ns { + +bool operator==(const ns::OneOf& lhs, const ns::OneOf& rhs) { + return lhs.foo == rhs.foo && true; +} + +USERVER_NAMESPACE::logging::LogHelper& operator<<( + USERVER_NAMESPACE::logging::LogHelper& lh, const ns::OneOf& value) { + return lh << ToString(USERVER_NAMESPACE::formats::json::ValueBuilder(value) + .ExtractValue()); +} + +OneOf Parse(USERVER_NAMESPACE::formats::json::Value json, + USERVER_NAMESPACE::formats::parse::To to) { + return Parse(json, to); +} + +OneOf Parse(USERVER_NAMESPACE::formats::yaml::Value json, + USERVER_NAMESPACE::formats::parse::To to) { + return Parse(json, to); +} + +OneOf Parse(USERVER_NAMESPACE::yaml_config::Value json, + USERVER_NAMESPACE::formats::parse::To to) { + return Parse(json, to); +} + +USERVER_NAMESPACE::formats::json::Value Serialize( + [[maybe_unused]] const ns::OneOf& value, + USERVER_NAMESPACE::formats::serialize::To< + USERVER_NAMESPACE::formats::json::Value>) { + USERVER_NAMESPACE::formats::json::ValueBuilder vb = + USERVER_NAMESPACE::formats::common::Type::kObject; + + if (value.foo) { + vb["foo"] = USERVER_NAMESPACE::chaotic::Variant< + USERVER_NAMESPACE::chaotic::Primitive, + USERVER_NAMESPACE::chaotic::Primitive>{value.foo.value()}; + } + + return vb.ExtractValue(); +} + +} // namespace ns diff --git a/chaotic/golden_tests/output/schemas/oneof/oneof.hpp b/chaotic/golden_tests/output/schemas/oneof/oneof.hpp new file mode 100644 index 000000000000..c959bdd774a3 --- /dev/null +++ b/chaotic/golden_tests/output/schemas/oneof/oneof.hpp @@ -0,0 +1,43 @@ +#pragma once + +#include "oneof_fwd.hpp" + +#include +#include +#include +#include +#include +#include + +#include + +namespace ns { +namespace impl {} +} // namespace ns +namespace ns { + +struct OneOf { + using Foo = std::variant; + + std::optional foo{}; +}; + +bool operator==(const ns::OneOf& lhs, const ns::OneOf& rhs); + +USERVER_NAMESPACE::logging::LogHelper& operator<<( + USERVER_NAMESPACE::logging::LogHelper& lh, const ns::OneOf& value); + +OneOf Parse(USERVER_NAMESPACE::formats::json::Value json, + USERVER_NAMESPACE::formats::parse::To); + +OneOf Parse(USERVER_NAMESPACE::formats::yaml::Value json, + USERVER_NAMESPACE::formats::parse::To); + +OneOf Parse(USERVER_NAMESPACE::yaml_config::Value json, + USERVER_NAMESPACE::formats::parse::To); + +USERVER_NAMESPACE::formats::json::Value Serialize( + const ns::OneOf& value, USERVER_NAMESPACE::formats::serialize::To< + USERVER_NAMESPACE::formats::json::Value>); + +} // namespace ns diff --git a/chaotic/golden_tests/output/schemas/oneof/oneof_fwd.hpp b/chaotic/golden_tests/output/schemas/oneof/oneof_fwd.hpp new file mode 100644 index 000000000000..ffb346fa80b2 --- /dev/null +++ b/chaotic/golden_tests/output/schemas/oneof/oneof_fwd.hpp @@ -0,0 +1,7 @@ +#pragma once + +namespace ns { + +struct OneOf; + +} diff --git a/chaotic/golden_tests/output/schemas/oneof/oneof_parsers.ipp b/chaotic/golden_tests/output/schemas/oneof/oneof_parsers.ipp new file mode 100644 index 000000000000..34445fbef0cb --- /dev/null +++ b/chaotic/golden_tests/output/schemas/oneof/oneof_parsers.ipp @@ -0,0 +1,30 @@ +#pragma once + +#include "oneof.hpp" + +namespace ns { + +static constexpr USERVER_NAMESPACE::utils::TrivialSet + kns__OneOf_PropertiesNames = [](auto selector) { + return selector().template Type().Case("foo"); + }; + +template +ns::OneOf Parse(Value value, USERVER_NAMESPACE::formats::parse::To) { + value.CheckNotMissing(); + value.CheckObjectOrNull(); + + ns::OneOf res; + + res.foo = value["foo"] + .template As, + USERVER_NAMESPACE::chaotic::Primitive>>>(); + + USERVER_NAMESPACE::chaotic::ValidateNoAdditionalProperties( + value, kns__OneOf_PropertiesNames); + + return res; +} + +} // namespace ns diff --git a/chaotic/golden_tests/output/schemas/string/string.cpp b/chaotic/golden_tests/output/schemas/string/string.cpp new file mode 100644 index 000000000000..f0b06cf452fa --- /dev/null +++ b/chaotic/golden_tests/output/schemas/string/string.cpp @@ -0,0 +1,55 @@ +#include "string.hpp" + +#include + +#include +#include +#include +#include +#include + +#include "string_parsers.ipp" + +namespace ns { + +bool operator==(const ns::String& lhs, const ns::String& rhs) { + return lhs.foo == rhs.foo && true; +} + +USERVER_NAMESPACE::logging::LogHelper& operator<<( + USERVER_NAMESPACE::logging::LogHelper& lh, const ns::String& value) { + return lh << ToString(USERVER_NAMESPACE::formats::json::ValueBuilder(value) + .ExtractValue()); +} + +String Parse(USERVER_NAMESPACE::formats::json::Value json, + USERVER_NAMESPACE::formats::parse::To to) { + return Parse(json, to); +} + +String Parse(USERVER_NAMESPACE::formats::yaml::Value json, + USERVER_NAMESPACE::formats::parse::To to) { + return Parse(json, to); +} + +String Parse(USERVER_NAMESPACE::yaml_config::Value json, + USERVER_NAMESPACE::formats::parse::To to) { + return Parse(json, to); +} + +USERVER_NAMESPACE::formats::json::Value Serialize( + [[maybe_unused]] const ns::String& value, + USERVER_NAMESPACE::formats::serialize::To< + USERVER_NAMESPACE::formats::json::Value>) { + USERVER_NAMESPACE::formats::json::ValueBuilder vb = + USERVER_NAMESPACE::formats::common::Type::kObject; + + if (value.foo) { + vb["foo"] = + USERVER_NAMESPACE::chaotic::Primitive{value.foo.value()}; + } + + return vb.ExtractValue(); +} + +} // namespace ns diff --git a/chaotic/golden_tests/output/schemas/string/string.hpp b/chaotic/golden_tests/output/schemas/string/string.hpp new file mode 100644 index 000000000000..ce28acfba972 --- /dev/null +++ b/chaotic/golden_tests/output/schemas/string/string.hpp @@ -0,0 +1,38 @@ +#pragma once + +#include "string_fwd.hpp" + +#include +#include +#include + +#include + +namespace ns { +namespace impl {} +} // namespace ns +namespace ns { + +struct String { + std::optional foo{}; +}; + +bool operator==(const ns::String& lhs, const ns::String& rhs); + +USERVER_NAMESPACE::logging::LogHelper& operator<<( + USERVER_NAMESPACE::logging::LogHelper& lh, const ns::String& value); + +String Parse(USERVER_NAMESPACE::formats::json::Value json, + USERVER_NAMESPACE::formats::parse::To); + +String Parse(USERVER_NAMESPACE::formats::yaml::Value json, + USERVER_NAMESPACE::formats::parse::To); + +String Parse(USERVER_NAMESPACE::yaml_config::Value json, + USERVER_NAMESPACE::formats::parse::To); + +USERVER_NAMESPACE::formats::json::Value Serialize( + const ns::String& value, USERVER_NAMESPACE::formats::serialize::To< + USERVER_NAMESPACE::formats::json::Value>); + +} // namespace ns diff --git a/chaotic/golden_tests/output/schemas/string/string_fwd.hpp b/chaotic/golden_tests/output/schemas/string/string_fwd.hpp new file mode 100644 index 000000000000..2c341bafb13d --- /dev/null +++ b/chaotic/golden_tests/output/schemas/string/string_fwd.hpp @@ -0,0 +1,7 @@ +#pragma once + +namespace ns { + +struct String; + +} diff --git a/chaotic/golden_tests/output/schemas/string/string_parsers.ipp b/chaotic/golden_tests/output/schemas/string/string_parsers.ipp new file mode 100644 index 000000000000..b1c1f1e6ac11 --- /dev/null +++ b/chaotic/golden_tests/output/schemas/string/string_parsers.ipp @@ -0,0 +1,30 @@ +#pragma once + +#include "string.hpp" + +namespace ns { + +static constexpr USERVER_NAMESPACE::utils::TrivialSet + kns__String_PropertiesNames = [](auto selector) { + return selector().template Type().Case("foo"); + }; + +template +ns::String Parse(Value value, + USERVER_NAMESPACE::formats::parse::To) { + value.CheckNotMissing(); + value.CheckObjectOrNull(); + + ns::String res; + + res.foo = value["foo"] + .template As>>(); + + USERVER_NAMESPACE::chaotic::ValidateNoAdditionalProperties( + value, kns__String_PropertiesNames); + + return res; +} + +} // namespace ns diff --git a/chaotic/golden_tests/schemas/allof/allof.yaml b/chaotic/golden_tests/schemas/allof/allof.yaml new file mode 100644 index 000000000000..b42d1b70a929 --- /dev/null +++ b/chaotic/golden_tests/schemas/allof/allof.yaml @@ -0,0 +1,18 @@ +components: + schemas: + AllOf: + type: object + additionalProperties: false + properties: + foo: + allOf: + - type: object + additionalProperties: true + properties: + foo: + type: string + - type: object + additionalProperties: true + properties: + bar: + type: integer diff --git a/chaotic/golden_tests/schemas/enum/enum.yaml b/chaotic/golden_tests/schemas/enum/enum.yaml new file mode 100644 index 000000000000..59319cfb86c9 --- /dev/null +++ b/chaotic/golden_tests/schemas/enum/enum.yaml @@ -0,0 +1,12 @@ +components: + schemas: + Enum: + type: object + additionalProperties: false + properties: + foo: + type: string + enum: + - one + - two + - three diff --git a/chaotic/golden_tests/schemas/int.yaml b/chaotic/golden_tests/schemas/int/int.yaml similarity index 100% rename from chaotic/golden_tests/schemas/int.yaml rename to chaotic/golden_tests/schemas/int/int.yaml diff --git a/chaotic/golden_tests/schemas/oneof/oneof.yaml b/chaotic/golden_tests/schemas/oneof/oneof.yaml new file mode 100644 index 000000000000..0de6ee82066f --- /dev/null +++ b/chaotic/golden_tests/schemas/oneof/oneof.yaml @@ -0,0 +1,11 @@ +components: + schemas: + OneOf: + type: object + additionalProperties: false + properties: + foo: + oneOf: + - type: integer + - type: string + diff --git a/chaotic/golden_tests/schemas/string/string.yaml b/chaotic/golden_tests/schemas/string/string.yaml new file mode 100644 index 000000000000..da7b68fa2adc --- /dev/null +++ b/chaotic/golden_tests/schemas/string/string.yaml @@ -0,0 +1,8 @@ +components: + schemas: + String: + type: object + additionalProperties: false + properties: + foo: + type: string From ae65c97dd4f9261df84632593b248e13ef32dee9 Mon Sep 17 00:00:00 2001 From: nepritula Date: Fri, 19 Jul 2024 18:47:41 +0300 Subject: [PATCH 057/629] feat docs: snippet in http_handler_base.hpp fixed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tests: протестировано локально и в CI ab0190f1bd0eddda9cc4df190130259fe183a12c --- core/include/userver/server/handlers/http_handler_base.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/include/userver/server/handlers/http_handler_base.hpp b/core/include/userver/server/handlers/http_handler_base.hpp index c5f026fd2d4a..c9b75e757d65 100644 --- a/core/include/userver/server/handlers/http_handler_base.hpp +++ b/core/include/userver/server/handlers/http_handler_base.hpp @@ -41,7 +41,7 @@ class HttpHandlerMethodStatistics; class HttpHandlerStatisticsScope; // clang-format off - +/// /// @ingroup userver_components userver_http_handlers userver_base_classes /// /// @brief Base class for all the @@ -59,8 +59,8 @@ class HttpHandlerStatisticsScope; /// /// ## Example usage: /// -/// @snippet samples/hello_service/hello_service.cpp Hello service sample - component - +/// @snippet samples/hello_service/src/hello_service.cpp Hello service sample - component +/// // clang-format on class HttpHandlerBase : public HandlerBase { From b40d0ac47b756ebf6e58885f4aa23c958cb131fd Mon Sep 17 00:00:00 2001 From: Antony Polukhin Date: Sat, 20 Jul 2024 09:57:58 +0300 Subject: [PATCH 058/629] bug build: fix kafka build with Conan MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tests: протестировано CI b31e23ba0f84ecb93567b5fc6a016b1e8a77063d Pull Request resolved: https://github.com/userver-framework/userver/pull/653 --- conan/test_package/test_kafka.cpp | 15 +++++++-------- conanfile.py | 6 ++++-- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/conan/test_package/test_kafka.cpp b/conan/test_package/test_kafka.cpp index 4c275db786f3..8d1cdd08e5b8 100644 --- a/conan/test_package/test_kafka.cpp +++ b/conan/test_package/test_kafka.cpp @@ -11,14 +11,13 @@ #include "hello.hpp" int main(int argc, char* argv[]) { - auto component_list = - userver::components::MinimalServerComponentList() - .Append() - .Append() - .Append() - .Append() - .Append("") - .Append(""); + auto component_list = userver::components::MinimalServerComponentList() + .Append() + .Append() + .Append() + .Append() + .Append() + .Append(); service_template::AppendHello(component_list); diff --git a/conanfile.py b/conanfile.py index e26fdd7e0131..743bc90b0184 100644 --- a/conanfile.py +++ b/conanfile.py @@ -57,7 +57,7 @@ class UserverConan(ConanFile): 'with_clickhouse': True, 'with_rabbitmq': True, 'with_utest': True, - 'with_kafka': False, + 'with_kafka': True, 'namespace': 'userver', 'namespace_begin': 'namespace userver {', 'namespace_end': '}', @@ -484,7 +484,9 @@ def clickhouse(): ) def librdkafka(): - return ['RdKafka::rdkafka++'] if self.options.with_kafka else [] + return ( + ['librdkafka::librdkafka'] if self.options.with_kafka else [] + ) userver_components = [ { From 8e77af623ad3ef76184e7682e77044d6de3b5d17 Mon Sep 17 00:00:00 2001 From: volcolac Date: Sun, 21 Jul 2024 14:53:38 +0300 Subject: [PATCH 059/629] feat httpclient: add query_params_propagator plugin 9578713698db6d1bbe52d8f6e16eb1be888484ec --- core/include/userver/clients/http/plugin.hpp | 2 ++ .../server/request/task_inherited_request.hpp | 10 ++++++++++ core/src/clients/http/plugin.cpp | 10 ++++++++++ .../server/request/task_inherited_request.cpp | 17 +++++++++++++++++ 4 files changed, 39 insertions(+) diff --git a/core/include/userver/clients/http/plugin.hpp b/core/include/userver/clients/http/plugin.hpp index 3574743160cb..d44e59d98db6 100644 --- a/core/include/userver/clients/http/plugin.hpp +++ b/core/include/userver/clients/http/plugin.hpp @@ -26,6 +26,8 @@ class PluginRequest final { void SetHeader(std::string_view name, std::string value); + void AddQueryParams(std::string_view params); + void SetTimeout(std::chrono::milliseconds ms); private: diff --git a/core/include/userver/server/request/task_inherited_request.hpp b/core/include/userver/server/request/task_inherited_request.hpp index ff2cd0cf4224..547ad751f5a6 100644 --- a/core/include/userver/server/request/task_inherited_request.hpp +++ b/core/include/userver/server/request/task_inherited_request.hpp @@ -33,6 +33,16 @@ bool HasTaskInheritedHeader(std::string_view header_name); bool HasTaskInheritedHeader( const USERVER_NAMESPACE::http::headers::PredefinedHeader& header_name); +/// @brief Get a query parameter from server::http::HttpRequest that is handled +/// by the current task hierarchy. +/// @return Parameter value or an empty string, if none such +const std::string& GetTaskInheritedQueryParameter(std::string_view name); + +/// @brief Checks whether specified query parameter exists in +/// server::http::HttpRequest that is handled by the current task hierarchy. +/// @return `true` if the parameter exists, `false` otherwise +bool HasTaskInheritedQueryParameter(std::string_view name); + } // namespace server::request USERVER_NAMESPACE_END diff --git a/core/src/clients/http/plugin.cpp b/core/src/clients/http/plugin.cpp index 53296a0799c6..08cfed056b53 100644 --- a/core/src/clients/http/plugin.cpp +++ b/core/src/clients/http/plugin.cpp @@ -2,6 +2,7 @@ #include #include +#include USERVER_NAMESPACE_BEGIN @@ -13,6 +14,15 @@ void PluginRequest::SetHeader(std::string_view name, std::string value) { state_.easy().add_header(name, value); } +void PluginRequest::AddQueryParams(std::string_view params) { + const auto& url = state_.easy().get_original_url(); + if (url.find('?') != std::string::npos) { + state_.easy().set_url(utils::StrCat(url, "&", params)); + } else { + state_.easy().set_url(utils::StrCat(url, "?", params)); + } +} + void PluginRequest::SetTimeout(std::chrono::milliseconds ms) { state_.set_timeout(ms.count()); state_.SetEasyTimeout(ms); diff --git a/core/src/server/request/task_inherited_request.cpp b/core/src/server/request/task_inherited_request.cpp index 14e701f33173..19b2b37fa67a 100644 --- a/core/src/server/request/task_inherited_request.cpp +++ b/core/src/server/request/task_inherited_request.cpp @@ -9,6 +9,7 @@ namespace server::request { namespace { const std::string kEmptyHeader{}; +const std::string kEmptyParameter{}; template const std::string& DoGetTaskInheritedHeader(const Header& header_name) { @@ -48,6 +49,22 @@ bool HasTaskInheritedHeader( return DoHasTaskInheritedHeader(header_name); } +const std::string& GetTaskInheritedQueryParameter(std::string_view name) { + const auto* request = kTaskInheritedRequest.GetOptional(); + if (request == nullptr) { + return kEmptyParameter; + } + return (*request)->GetArg(name); +} + +bool HasTaskInheritedQueryParameter(std::string_view name) { + const auto* request = kTaskInheritedRequest.GetOptional(); + if (request == nullptr) { + return false; + } + return (*request)->HasArg(name); +} + } // namespace server::request USERVER_NAMESPACE_END From ce74bee833e05e1a25a6131b6da7a23f2b202777 Mon Sep 17 00:00:00 2001 From: vasilievss Date: Mon, 22 Jul 2024 11:09:49 +0300 Subject: [PATCH 060/629] feat universal: Remove tmp file on file operations error 8809c0b4c606a18cd299903bc283cae2dc7191bd --- universal/src/fs/blocking/write.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/universal/src/fs/blocking/write.cpp b/universal/src/fs/blocking/write.cpp index 34cfec2b195c..1b548da1be65 100644 --- a/universal/src/fs/blocking/write.cpp +++ b/universal/src/fs/blocking/write.cpp @@ -92,7 +92,7 @@ void RewriteFileContentsAtomically(const std::string& path, boost::filesystem::perms perms) { const auto tmp_path = fmt::format("{}{}.tmp", path, utils::generators::GenerateBoostUuid()); - const boost::filesystem::path boost_tmp_path{path}; + const boost::filesystem::path boost_tmp_path{tmp_path}; const auto directory = boost_tmp_path.parent_path().string(); utils::FastScopeGuard guard{[&boost_tmp_path]() noexcept { From a9c816c57d0b6c501e1cb00d8efaf1bc492fbf38 Mon Sep 17 00:00:00 2001 From: nepritula Date: Mon, 22 Jul 2024 12:14:12 +0300 Subject: [PATCH 061/629] feat docs: gRPC client and service tutorial snippet fix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tests: протестировано локально и в CI 64c17330b9789dd18eb0290920f5f7b011ecb463 --- samples/grpc_service/src/greeter_service.cpp | 4 ++-- .../docs/en/userver/tutorial/grpc_service.md | 22 +++++++++---------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/samples/grpc_service/src/greeter_service.cpp b/samples/grpc_service/src/greeter_service.cpp index b6bd040af65d..f1b5c4a6af1b 100644 --- a/samples/grpc_service/src/greeter_service.cpp +++ b/samples/grpc_service/src/greeter_service.cpp @@ -26,7 +26,7 @@ void GreeterService::SayHello(api::GreeterServiceBase::SayHelloCall& call, } /// [server RPC handling] -/// [service] +/// [component] GreeterServiceComponent::GreeterServiceComponent( const components::ComponentConfig& config, const components::ComponentContext& context) @@ -34,7 +34,7 @@ GreeterServiceComponent::GreeterServiceComponent( service_(config["greeting-prefix"].As()) { RegisterService(service_); } -/// [service] +/// [component] yaml_config::Schema GreeterServiceComponent::GetStaticConfigSchema() { return yaml_config::MergeSchemas(R"( diff --git a/scripts/docs/en/userver/tutorial/grpc_service.md b/scripts/docs/en/userver/tutorial/grpc_service.md index 203b0d95fb3b..337c24c01fd8 100644 --- a/scripts/docs/en/userver/tutorial/grpc_service.md +++ b/scripts/docs/en/userver/tutorial/grpc_service.md @@ -59,17 +59,17 @@ that inherits from both However, for in this example we will also test our service using gtest, so we need to split the logic from the component. -@snippet samples/grpc_service/src/grpc_service.hpp includes +@snippet samples/grpc_service/src/greeter_service.hpp includes -@snippet samples/grpc_service/src/grpc_service.hpp service +@snippet samples/grpc_service/src/greeter_service.hpp service -@snippet samples/grpc_service/src/grpc_service.hpp component +@snippet samples/grpc_service/src/greeter_service.hpp component -@snippet samples/grpc_service/src/grpc_service.cpp component +@snippet samples/grpc_service/src/greeter_service.cpp component A single request-response RPC handling is simple: fill in the `response` and send it. -@snippet samples/grpc_service/src/grpc_service.cpp service +@snippet samples/grpc_service/src/greeter_service.cpp server RPC handling Fill in the static config entries for the server side: @@ -114,7 +114,7 @@ service some preparational steps should be done. First of all, import the required modules and add the required pytest_userver.plugins.grpc pytest plugin: -@snippet samples/grpc_service/tests/conftest.py Prepare modules +@snippet samples/grpc_service/testsuite/conftest.py Prepare modules #### gRPC server mock @@ -122,26 +122,26 @@ pytest_userver.plugins.grpc pytest plugin: To mock the gRPC server provide a hook for the static config to change the endpoint: -@snippet samples/grpc_service/tests/conftest.py Prepare configs +@snippet samples/grpc_service/testsuite/conftest.py Prepare configs Write the mocking fixtures using @ref pytest_userver.plugins.grpc_mockserver.grpc_mockserver "grpc_mockserver": -@snippet samples/grpc_service/tests/conftest.py Prepare server mock +@snippet samples/grpc_service/testsuite/conftest.py Prepare server mock After that everything is ready to check the service client requests: -@snippet samples/grpc_service/tests/test_grpc.py grpc client test +@snippet samples/grpc_service/testsuite/test_grpc.py grpc client test #### gRPC client To do the gRPC requests write a client fixture using @ref pytest_userver.plugins.grpc_client.grpc_channel "grpc_channel": -@snippet samples/grpc_service/tests/conftest.py grpc client +@snippet samples/grpc_service/testsuite/conftest.py grpc client Use it to do gRPC requests to the service: -@snippet samples/grpc_service/tests/test_grpc.py grpc server test +@snippet samples/grpc_service/testsuite/test_grpc.py grpc server test ### Unit testing for the sample gRPC service and client (gtest) From bedde8b69715441aa6cf4b0be41097bcbfa4a5d9 Mon Sep 17 00:00:00 2001 From: segoon Date: Mon, 22 Jul 2024 14:47:28 +0300 Subject: [PATCH 062/629] bug otlp: fix timezone ac16e11e316f24b02aff98900e3e412be70f75a1 --- .mapping.json | 1 + .../userver/ugrpc/server/middlewares/fwd.hpp | 5 +- grpc/utest/src/ugrpc/tests/service.cpp | 4 +- otlp/CMakeLists.txt | 13 ++ otlp/src/otlp/logs/logger.cpp | 6 +- otlp/tests/service_test.cpp | 127 ++++++++++++++++++ 6 files changed, 145 insertions(+), 11 deletions(-) create mode 100644 otlp/tests/service_test.cpp diff --git a/.mapping.json b/.mapping.json index 0e836cb5c44f..18cce9a9c33e 100644 --- a/.mapping.json +++ b/.mapping.json @@ -2282,6 +2282,7 @@ "otlp/src/otlp/logs/component.cpp":"taxi/uservices/userver/otlp/src/otlp/logs/component.cpp", "otlp/src/otlp/logs/logger.cpp":"taxi/uservices/userver/otlp/src/otlp/logs/logger.cpp", "otlp/src/otlp/logs/logger.hpp":"taxi/uservices/userver/otlp/src/otlp/logs/logger.hpp", + "otlp/tests/service_test.cpp":"taxi/uservices/userver/otlp/tests/service_test.cpp", "postgresql/CMakeLists.txt":"taxi/uservices/userver/postgresql/CMakeLists.txt", "postgresql/README.md":"taxi/uservices/userver/postgresql/README.md", "postgresql/functional_tests/CMakeLists.txt":"taxi/uservices/userver/postgresql/functional_tests/CMakeLists.txt", diff --git a/grpc/include/userver/ugrpc/server/middlewares/fwd.hpp b/grpc/include/userver/ugrpc/server/middlewares/fwd.hpp index 97f6324041d8..52855d8d18bd 100644 --- a/grpc/include/userver/ugrpc/server/middlewares/fwd.hpp +++ b/grpc/include/userver/ugrpc/server/middlewares/fwd.hpp @@ -6,8 +6,6 @@ #include #include -#include - USERVER_NAMESPACE_BEGIN namespace ugrpc::server { @@ -17,8 +15,7 @@ class MiddlewareBase; class MiddlewareCallContext; /// @brief A chain of middlewares -using Middlewares = - boost::container::small_vector, 5>; +using Middlewares = std::vector>; } // namespace ugrpc::server diff --git a/grpc/utest/src/ugrpc/tests/service.cpp b/grpc/utest/src/ugrpc/tests/service.cpp index 9e007b95b6f0..47681b593320 100644 --- a/grpc/utest/src/ugrpc/tests/service.cpp +++ b/grpc/utest/src/ugrpc/tests/service.cpp @@ -35,9 +35,7 @@ ServiceBase::ServiceBase(server::ServerConfig&& server_config) : config_storage_(dynamic_config::MakeDefaultStorage({})), server_(std::move(server_config), statistics_storage_, config_storage_.GetSource()), - server_middlewares_( - {std::make_shared(ServerLogMiddlewareSettings{}), - std::make_shared()}), + server_middlewares_({std::make_shared()}), middleware_factories_({std::make_shared( ClientLogMiddlewareSettings{}), std::make_shared()}), diff --git a/otlp/CMakeLists.txt b/otlp/CMakeLists.txt index 5e84f64b6640..47e6c71fcb67 100644 --- a/otlp/CMakeLists.txt +++ b/otlp/CMakeLists.txt @@ -48,3 +48,16 @@ _userver_directory_install(COMPONENT otlp FILES "${USERVER_ROOT_DIR}/cmake/install/userver-otlp-config.cmake" DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/userver ) + +if (USERVER_IS_THE_ROOT_PROJECT) + add_executable(${PROJECT_NAME}-unittest ${UNIT_TEST_SOURCES}) + target_include_directories(${PROJECT_NAME}-unittest PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/src + ) + target_link_libraries(${PROJECT_NAME}-unittest + userver-utest + userver-grpc-utest + ${PROJECT_NAME} + ) + add_google_tests(${PROJECT_NAME}-unittest) +endif() diff --git a/otlp/src/otlp/logs/logger.cpp b/otlp/src/otlp/logs/logger.cpp index d9a56019dd03..4f34ce36f7c4 100644 --- a/otlp/src/otlp/logs/logger.cpp +++ b/otlp/src/otlp/logs/logger.cpp @@ -34,7 +34,6 @@ Logger::Logger( queue_(Queue::Create(config_.max_queue_size)), queue_producer_(queue_->GetMultiProducer()) { SetLevel(config_.log_level); - Log(logging::Level::kInfo, "OTLP logger has started"); std::cerr << "OTLP logger has started\n"; sender_task_ = engine::CriticalAsyncNoSpan( @@ -88,9 +87,8 @@ void Logger::Log(logging::Level level, std::string_view msg) { return true; } if (key == "timestamp") { - timestamp = utils::datetime::Stringtime( - std::string{value}, utils::datetime::kDefaultTimezone, - kTimestampFormat); + timestamp = utils::datetime::LocalTimezoneStringtime( + std::string{value}, kTimestampFormat); return true; } if (key == "level") { diff --git a/otlp/tests/service_test.cpp b/otlp/tests/service_test.cpp new file mode 100644 index 000000000000..4cc4c82ba000 --- /dev/null +++ b/otlp/tests/service_test.cpp @@ -0,0 +1,127 @@ +#include + +#include + +#include +#include + +#include +#include + +#include +#include +#include + +USERVER_NAMESPACE_BEGIN + +namespace { + +template +class Service : public ugrpc::tests::ServiceBase { + public: + explicit Service(ugrpc::server::ServerConfig&& server_config) + : ServiceBase(std::move(server_config)), service1_(), service2_() { + RegisterService(service1_); + RegisterService(service2_); + StartServer(); + } + + ~Service() override { StopServer(); } + + /// @returns the stored service. + GrpcService1& GetService1() { return service1_; } + GrpcService2& GetService2() { return service2_; } + + private: + GrpcService1 service1_{}; + GrpcService2 service2_{}; +}; + +class LogService final + : public opentelemetry::proto::collector::logs::v1::LogsServiceBase { + public: + void Export( + ExportCall& call, + ::opentelemetry::proto::collector::logs::v1::ExportLogsServiceRequest&& + request) override { + // Don't emit new traces to avoid recursive traces/logs + tracing::Span::CurrentSpan().SetLogLevel(logging::Level::kNone); + + for (const auto& rl : request.resource_logs()) { + for (const auto& sl : rl.scope_logs()) { + for (const auto& lr : sl.log_records()) { + logs.push_back(lr); + } + } + } + + call.Finish({}); + } + + // no sync as there is only a single grpc client + std::vector<::opentelemetry::proto::logs::v1::LogRecord> logs; +}; + +class TraceService final + : public opentelemetry::proto::collector::trace::v1::TraceServiceBase { + public: + void Export( + ExportCall& call, + ::opentelemetry::proto::collector::trace::v1::ExportTraceServiceRequest&&) + override { + // Don't emit new traces to avoid recursive traces/logs + tracing::Span::CurrentSpan().SetLogLevel(logging::Level::kNone); + + call.Finish({}); + } +}; + +// NOLINTNEXTLINE(fuchsia-multiple-inheritance) +class LogServiceTest : public Service, + public utest::DefaultLoggerFixture<::testing::Test> { + public: + LogServiceTest() : Service({}) { + logger_ = std::make_shared( + MakeClient< + opentelemetry::proto::collector::logs::v1::LogsServiceClient>(), + MakeClient< + opentelemetry::proto::collector::trace::v1::TraceServiceClient>(), + otlp::LoggerConfig{}); + SetDefaultLogger(logger_); + } + + ~LogServiceTest() override { logger_->Stop(); } + + private: + std::shared_ptr logger_; +}; + +} // namespace + +UTEST_F(LogServiceTest, DISABLED_NoInfiniteLogsInTrace) { + LOG_INFO() << "dummy log"; + + engine::SleepFor(std::chrono::seconds(1)); + EXPECT_EQ(GetService1().logs.size(), 1); +} + +UTEST_F(LogServiceTest, DISABLED_Smoke) { + auto timestamp = std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()); + LOG_INFO() << "log"; + + while (GetService1().logs.size() < 1) { + engine::SleepFor(std::chrono::milliseconds(10)); + } + + auto& log = GetService1().logs[0]; + + EXPECT_EQ(log.body().string_value(), "log"); + EXPECT_EQ(log.severity_text(), "INFO"); + + EXPECT_LE(timestamp.count(), log.time_unix_nano()); + EXPECT_LE(log.time_unix_nano(), timestamp.count() + 1'000'000'000) + << log.time_unix_nano() - timestamp.count() - 1'000'000'000; +} + +USERVER_NAMESPACE_END From fa146587c32c596a51159fc33bf96d3f37c4eacd Mon Sep 17 00:00:00 2001 From: svshevtsov Date: Mon, 22 Jul 2024 16:37:39 +0300 Subject: [PATCH 063/629] Revert commit rXXXXXX, feat userver: remove pg_last_xlog_replay_location() for AWS Aurora df75ffc011af64dfa826b8299b8f609ebcb799ee --- .../postgres/detail/topology/hot_standby.cpp | 42 ++++--------------- 1 file changed, 7 insertions(+), 35 deletions(-) diff --git a/postgresql/src/storages/postgres/detail/topology/hot_standby.cpp b/postgresql/src/storages/postgres/detail/topology/hot_standby.cpp index 23aa27634059..badeb47cde74 100644 --- a/postgresql/src/storages/postgres/detail/topology/hot_standby.cpp +++ b/postgresql/src/storages/postgres/detail/topology/hot_standby.cpp @@ -39,58 +39,31 @@ using ReplicationLag = std::chrono::milliseconds; constexpr const char* kDiscoveryTaskName = "pg_topology"; const std::string kShowSyncStandbyNames = "SHOW synchronous_standby_names"; -const std::string kCheckAuroraQuery = - "SELECT 'aws_commons' = ANY(string_to_array(setting, ', ')) " - "FROM pg_settings " - "WHERE name = 'rds.extensions';"; - -bool IsAuroraConnection(std::unique_ptr& connection) { - const auto res = connection->Execute(kCheckAuroraQuery); - - if (res.IsEmpty()) { - return false; - } - bool is_aurora = false; - res.Front().To(is_aurora); - - return is_aurora; -} struct WalInfoStatements { int min_version; - bool is_aurora; std::string master; std::string slave; }; -const WalInfoStatements& GetWalInfoStatementsForVersion(int version, - bool is_aurora) { +const WalInfoStatements& GetWalInfoStatementsForVersion(int version) { // must be in descending min_version order static const WalInfoStatements kKnownStatements[]{ // Master doesn't provide last xact timestamp without // `track_commit_timestamp` enabled, use current server time // as an approximation. // Also note that all times here are sourced from the master. - {100000, /* is_aurora */ false, "SELECT pg_current_wal_lsn(), now()", + {100000, "SELECT pg_current_wal_lsn(), now()", "SELECT pg_last_wal_replay_lsn(), pg_last_xact_replay_timestamp()"}, // Versions for 9.4-9.x servers. // (Functions were actually available since 8.2 but returned TEXT.) - {90400, /* is_aurora */ false, "SELECT pg_current_xlog_location(), now()", + {90400, "SELECT pg_current_xlog_location(), now()", "SELECT pg_last_xlog_replay_location(), " - "pg_last_xact_replay_timestamp()"}, - - // AWS Aurora doesn't support pg functions above, so use aurora's - // alternatives instead - {90400, /* is_aurora */ true, - "SELECT highest_lsn_rcvd, now() " - "FROM aurora_replica_status() WHERE session_id != 'MASTER_SESSION_ID'", - "SELECT current_read_lsn, " - "now() - (replica_lag_in_msec || ' milliseconds')::interval " - "FROM aurora_replica_status() WHERE session_id != 'MASTER_SESSION_ID'"}}; + "pg_last_xact_replay_timestamp()"}}; for (const auto& cand : kKnownStatements) { - if (is_aurora == cand.is_aurora && version >= cand.min_version) return cand; + if (version >= cand.min_version) return cand; } throw PoolError{fmt::format("Unsupported database version: {}", version)}; } @@ -338,9 +311,8 @@ void HotStandby::RunCheck(DsnIndex idx) { state.roundtrip_time = std::chrono::duration_cast( std::chrono::steady_clock::now() - start); - const bool is_aurora = IsAuroraConnection(state.connection); - const auto& wal_info_stmts = GetWalInfoStatementsForVersion( - state.connection->GetServerVersion(), is_aurora); + const auto& wal_info_stmts = + GetWalInfoStatementsForVersion(state.connection->GetServerVersion()); std::optional current_xact_timestamp; const auto wal_info = state.connection->Execute( From 9e0858f5cb1dfe99a6390cc201965cb12da12f6e Mon Sep 17 00:00:00 2001 From: nepritula Date: Mon, 22 Jul 2024 16:40:41 +0300 Subject: [PATCH 064/629] feat docs: pathes to snippets fixed in the tutorial: mongo_service.md, redis_service.md, websocket_service.md, multipart_service.md and json_to_yaml.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tests: протестировано локально и в CI af0f9908e1f6118d3fa873d30f13f2ace7ce57ee --- samples/multipart_service/tests/conftest.py | 2 ++ samples/websocket_service/tests/conftest.py | 2 ++ scripts/docs/en/userver/tutorial/json_to_yaml.md | 12 ++++++------ scripts/docs/en/userver/tutorial/mongo_service.md | 4 ++-- scripts/docs/en/userver/tutorial/redis_service.md | 6 +++--- 5 files changed, 15 insertions(+), 11 deletions(-) diff --git a/samples/multipart_service/tests/conftest.py b/samples/multipart_service/tests/conftest.py index f22f982eed19..9bc78958279a 100644 --- a/samples/multipart_service/tests/conftest.py +++ b/samples/multipart_service/tests/conftest.py @@ -1,3 +1,5 @@ # /// [registration] + pytest_plugins = ['pytest_userver.plugins.core'] + # /// [registration] diff --git a/samples/websocket_service/tests/conftest.py b/samples/websocket_service/tests/conftest.py index f22f982eed19..9bc78958279a 100644 --- a/samples/websocket_service/tests/conftest.py +++ b/samples/websocket_service/tests/conftest.py @@ -1,3 +1,5 @@ # /// [registration] + pytest_plugins = ['pytest_userver.plugins.core'] + # /// [registration] diff --git a/scripts/docs/en/userver/tutorial/json_to_yaml.md b/scripts/docs/en/userver/tutorial/json_to_yaml.md index e3bcf09cb292..e26bbbf999c6 100644 --- a/scripts/docs/en/userver/tutorial/json_to_yaml.md +++ b/scripts/docs/en/userver/tutorial/json_to_yaml.md @@ -23,12 +23,12 @@ Let's write a simple JSON to YAML converter with the help of `userver-universal` The implementation is quite straightforward. Include the necessary C++ Standard library and userver headers: -@snippet samples/json2yaml/json2yaml.hpp json2yaml - includes +@snippet samples/json2yaml/src/json2yaml.hpp json2yaml - includes Write the logic for converting each of the JSON types to YAML type: -@snippet samples/json2yaml/json2yaml.hpp json2yaml - convert hpp -@snippet samples/json2yaml/json2yaml.hpp json2yaml - convert cpp +@snippet samples/json2yaml/src/json2yaml.hpp json2yaml - convert hpp +@snippet samples/json2yaml/src/json2yaml.cpp json2yaml - convert cpp Finally, read data from `std::cin`, parse it as JSON, convert to YAML and output it as text: @@ -71,7 +71,7 @@ The code could be tested using any of the unit-testing frameworks, like Boost.Test, GTest, and so forth. Here's how to do it with GTest: * Write a test: - @snippet samples/json2yaml/json2yaml_test.cpp json2yaml - unittest + @snippet samples/json2yaml/unittests/json2yaml_test.cpp json2yaml - unittest * Add its run to CMakeLists.txt: @snippet samples/json2yaml/CMakeLists.txt add_unit_test * Run the test via `ctest -V` @@ -85,10 +85,10 @@ Python with `pytest`: @snippet samples/json2yaml/CMakeLists.txt add_test * Add a fixture to `conftest.py` to get the path to the CMake built binary: - @snippet samples/json2yaml/tests/conftest.py pytest + @snippet samples/json2yaml/testsuite/conftest.py pytest * Write the test: - @snippet samples/json2yaml/tests/test_basic.py pytest + @snippet samples/json2yaml/testsuite/test_basic.py pytest ## Full sources diff --git a/scripts/docs/en/userver/tutorial/mongo_service.md b/scripts/docs/en/userver/tutorial/mongo_service.md index 0046129a46e5..d347756998ef 100644 --- a/scripts/docs/en/userver/tutorial/mongo_service.md +++ b/scripts/docs/en/userver/tutorial/mongo_service.md @@ -126,13 +126,13 @@ implemented using the testsuite. To do that you have to: * Turn on the pytest_userver.plugins.mongo plugin and provide Mongo settings info for the testsuite: - @snippet samples/mongo_service/tests/conftest.py mongodb settings + @snippet samples/mongo_service/testsuite/conftest.py mongodb settings The pytest_userver.plugins.service_client.auto_client_deps() fixture already known about the mongodb fixture, so there's no need to override the extra_client_deps() fixture. * Write the test: - @snippet samples/mongo_service/tests/test_mongo.py Functional test + @snippet samples/mongo_service/testsuite/test_mongo.py Functional test ## Full sources diff --git a/scripts/docs/en/userver/tutorial/redis_service.md b/scripts/docs/en/userver/tutorial/redis_service.md index 7a65cfa67306..f08bd6c5a98e 100644 --- a/scripts/docs/en/userver/tutorial/redis_service.md +++ b/scripts/docs/en/userver/tutorial/redis_service.md @@ -156,10 +156,10 @@ implemented with one of UTEST macros in the following way: implemented using the testsuite. To do that you have to: * Prepare the pytest by importing the pytest_userver.plugins.redis plugin: - @snippet samples/redis_service/tests/conftest.py redis setup + @snippet samples/redis_service/testsuite/conftest.py redis setup * Add the Redis Secdist settings info to the service environment variable: - @snippet samples/redis_service/tests/conftest.py service_env + @snippet samples/redis_service/testsuite/conftest.py service_env The @ref pytest_userver.plugins.service_client.auto_client_deps "auto_client_deps" fixture already knows about the redis_store fixture, so there's no need to override the @ref pytest_userver.plugins.service_client.extra_client_deps "extra_client_deps" @@ -168,7 +168,7 @@ implemented using the testsuite. To do that you have to: For details on Redis Secdist format, see @ref components::Redis. * Write the test: - @snippet samples/redis_service/tests/test_redis.py Functional test + @snippet samples/redis_service/testsuite/test_redis.py Functional test ## Full sources From 1af2edab01625a0961f10b2c4665079de20a5a0b Mon Sep 17 00:00:00 2001 From: segoon Date: Mon, 22 Jul 2024 17:43:17 +0300 Subject: [PATCH 065/629] docs otlp: add to doxygen.conf 7277f151603b0cb50a5d4e8ea2c2aeb31b04b4c3 --- scripts/docs/doxygen.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/docs/doxygen.conf b/scripts/docs/doxygen.conf index cd35fa1db6af..9a2733def415 100644 --- a/scripts/docs/doxygen.conf +++ b/scripts/docs/doxygen.conf @@ -983,6 +983,7 @@ INPUT = CONTRIBUTING.md \ rabbitmq/include \ mysql/include \ ydb/include \ + otlp/include \ testsuite/pytest_plugins/ \ docker-compose.yml \ scripts/docker/ From 60ca797df849f48dfc125c23a794ba8c5bd3737f Mon Sep 17 00:00:00 2001 From: nepritula Date: Mon, 22 Jul 2024 22:57:10 +0300 Subject: [PATCH 066/629] feat docs: pathes to snippets fixed in json_to_yaml.md and in hello_service.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tests: протестировано локально и в CI e8f1fffd72782fdf578450845fd4d75febb987f9 --- scripts/docs/en/userver/tutorial/hello_service.md | 2 +- scripts/docs/en/userver/tutorial/json_to_yaml.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/docs/en/userver/tutorial/hello_service.md b/scripts/docs/en/userver/tutorial/hello_service.md index b317748a35c0..3a1028395b37 100644 --- a/scripts/docs/en/userver/tutorial/hello_service.md +++ b/scripts/docs/en/userver/tutorial/hello_service.md @@ -60,7 +60,7 @@ HTTP response code will be set to `500`. Note that the component in an anonymous namespace and the only way to add it to the component lists in via calling `AppendHello` from "hello_service.hpp": -@snippet samples/hello_service/src/hello_service.hpp Hello service sample - component interface +@snippet samples/hello_service/main.cpp Hello service sample - main ### Static config diff --git a/scripts/docs/en/userver/tutorial/json_to_yaml.md b/scripts/docs/en/userver/tutorial/json_to_yaml.md index e26bbbf999c6..680d7f4d45f5 100644 --- a/scripts/docs/en/userver/tutorial/json_to_yaml.md +++ b/scripts/docs/en/userver/tutorial/json_to_yaml.md @@ -33,7 +33,7 @@ Write the logic for converting each of the JSON types to YAML type: Finally, read data from `std::cin`, parse it as JSON, convert to YAML and output it as text: -@snippet samples/json2yaml/json2yaml.cpp json2yaml - main +@snippet samples/json2yaml/main.cpp json2yaml - main ### Build and Run @@ -49,7 +49,7 @@ cmake -DCMAKE_BUILD_TYPE=Release .. make userver-samples-json2yaml ``` -After that a tool is compiled an it could be used: +After that a tool is compiled and it could be used: ``` bash $ samples/json2yaml/userver-samples-json2yaml From 6d729dc9139ebd8ae97aad5605ee9f91870ff76b Mon Sep 17 00:00:00 2001 From: nepritula Date: Tue, 23 Jul 2024 15:22:56 +0300 Subject: [PATCH 067/629] feat docs: path to image fixed in framework_comparison.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tests: протестировано локально и в CI d40b7d279cbfbac621c9704bc79a85a6f8a60b71 --- scripts/docs/en/userver/framework_comparison.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/docs/en/userver/framework_comparison.md b/scripts/docs/en/userver/framework_comparison.md index f2895decbe2c..95af7785286f 100644 --- a/scripts/docs/en/userver/framework_comparison.md +++ b/scripts/docs/en/userver/framework_comparison.md @@ -82,7 +82,7 @@ use ❌ and ❓ respectively. The table above shows the well-developed functionality of userver. Additionally userver is highly effective which is proven by the results of the Techempower benchmark run. Userver comes ahead of its competitors from the table. @ref bnchrs "[3]", see [[↗]][techempower-run] -![top15](scripts/docs/img/top15.png) +![top15](/top15.png) [techempower-run]: https://www.techempower.com/benchmarks/#section=test&runid=3c2e9871-9c2a-4ff3-bc31-620f65da4e74&hw=ph&test=composite From 120cb60d699cad1f7a0f29040b5a6f6805ec5403 Mon Sep 17 00:00:00 2001 From: antonyzhilin Date: Tue, 23 Jul 2024 16:51:42 +0300 Subject: [PATCH 068/629] fix grpc: always have 1 type=request and 1 type=response log in server 1620cd0ad8fba04bd760bfd129254dcd10027d1e --- cmake/UserverTestsuite.cmake | 2 +- .../userver/ugrpc/server/middlewares/base.hpp | 6 ---- grpc/src/ugrpc/server/middlewares/base.cpp | 8 ----- .../deadline_propagation/middleware.cpp | 3 +- .../server/middlewares/log/middleware.cpp | 34 ++++++++++++++----- 5 files changed, 29 insertions(+), 24 deletions(-) diff --git a/cmake/UserverTestsuite.cmake b/cmake/UserverTestsuite.cmake index 0cbe65197198..ca62db7be97e 100644 --- a/cmake/UserverTestsuite.cmake +++ b/cmake/UserverTestsuite.cmake @@ -31,7 +31,7 @@ function(_userver_prepare_testsuite) if(NOT PYTHONCONFIG_FOUND) message(FATAL_ERROR "Python dev is not found") endif() - set(USERVER_PYTHON_DEV_CHECKED TRUE CACHE INTERNAL "") + set(USERVER_PYTHON_DEV_CHECKED TRUE CACHE INTERNAL "" FORCE) endif() if(NOT USERVER_TESTSUITE_DIR) diff --git a/grpc/include/userver/ugrpc/server/middlewares/base.hpp b/grpc/include/userver/ugrpc/server/middlewares/base.hpp index adefa6488c1d..b113eae434a9 100644 --- a/grpc/include/userver/ugrpc/server/middlewares/base.hpp +++ b/grpc/include/userver/ugrpc/server/middlewares/base.hpp @@ -32,12 +32,6 @@ class MiddlewareCallContext final { /// @brief Get original gRPC Call CallAnyBase& GetCall() const; - /// @brief Get name of gRPC service - std::string_view GetServiceName() const; - - /// @brief Get name of called gRPC method - std::string_view GetMethodName() const; - /// @brief Get values extracted from dynamic_config. Snapshot will be /// deleted when the last middleware completes const dynamic_config::Snapshot& GetInitialDynamicConfig() const; diff --git a/grpc/src/ugrpc/server/middlewares/base.cpp b/grpc/src/ugrpc/server/middlewares/base.cpp index f5907d047096..0d0701c4c89d 100644 --- a/grpc/src/ugrpc/server/middlewares/base.cpp +++ b/grpc/src/ugrpc/server/middlewares/base.cpp @@ -44,14 +44,6 @@ void MiddlewareCallContext::ClearMiddlewaresResources() { CallAnyBase& MiddlewareCallContext::GetCall() const { return call_; } -std::string_view MiddlewareCallContext::GetServiceName() const { - return call_.GetServiceName(); -} - -std::string_view MiddlewareCallContext::GetMethodName() const { - return call_.GetMethodName(); -} - const dynamic_config::Snapshot& MiddlewareCallContext::GetInitialDynamicConfig() const { UASSERT(config_); diff --git a/grpc/src/ugrpc/server/middlewares/deadline_propagation/middleware.cpp b/grpc/src/ugrpc/server/middlewares/deadline_propagation/middleware.cpp index a7b53bede2e5..86a707f72986 100644 --- a/grpc/src/ugrpc/server/middlewares/deadline_propagation/middleware.cpp +++ b/grpc/src/ugrpc/server/middlewares/deadline_propagation/middleware.cpp @@ -59,7 +59,8 @@ void Middleware::Handle(MiddlewareCallContext& context) const { auto& call = context.GetCall(); if (!CheckAndSetupDeadline(call.GetSpan(), call.GetContext(), - context.GetServiceName(), context.GetMethodName(), + context.GetCall().GetServiceName(), + context.GetCall().GetMethodName(), call.Statistics(ugrpc::impl::InternalTag()), context.GetInitialDynamicConfig())) { call.FinishWithError(grpc::Status{ diff --git a/grpc/src/ugrpc/server/middlewares/log/middleware.cpp b/grpc/src/ugrpc/server/middlewares/log/middleware.cpp index d0dcbe3bbf61..6989a574cd92 100644 --- a/grpc/src/ugrpc/server/middlewares/log/middleware.cpp +++ b/grpc/src/ugrpc/server/middlewares/log/middleware.cpp @@ -52,7 +52,12 @@ void Middleware::CallRequestHook(const MiddlewareCallContext& context, if (storage.Get(kIsFirstRequest)) { storage.Set(kIsFirstRequest, false); - log_extra.Extend("type", "request"); + + const auto call_kind = context.GetCall().GetCallKind(); + if (call_kind == CallKind::kUnaryCall || + call_kind == CallKind::kResponseStream) { + log_extra.Extend("type", "request"); + } } LOG(span.GetLogLevel()) << "gRPC request message" << std::move(log_extra); } @@ -65,7 +70,6 @@ void Middleware::CallResponseHook(const MiddlewareCallContext& context, if (call_kind == CallKind::kUnaryCall || call_kind == CallKind::kRequestStream) { span.AddTag("grpc_type", "response"); - span.AddNonInheritableTag("type", "response"); span.AddNonInheritableTag("body", GetMessageForLogging(response, settings_)); } else { @@ -83,16 +87,30 @@ void Middleware::Handle(MiddlewareCallContext& context) const { auto& span = context.GetCall().GetSpan(); if (settings_.local_log_level) { - span.SetLocalLogLevel(*settings_.local_log_level); + span.SetLocalLogLevel(settings_.local_log_level); } - const auto meta_type = - fmt::format("{}/{}", context.GetServiceName(), context.GetMethodName()); - span.AddTag("meta_type", meta_type); + span.AddTag("meta_type", std::string{context.GetCall().GetCallName()}); + span.AddNonInheritableTag("type", "response"); if (call_kind == CallKind::kResponseStream || call_kind == CallKind::kBidirectionalStream) { - span.AddNonInheritableTag("type", "response"); - span.AddNonInheritableTag("body", "stream finished"); + // Just like in HTTP, there must be a single trailing Span log + // with type=response and some body. We don't have a real single response + // (responses are written separately, 1 log per response), so we fake + // the required response log. + span.AddNonInheritableTag("body", "response stream finished"); + } + + if (call_kind == CallKind::kRequestStream || + call_kind == CallKind::kBidirectionalStream) { + // Just like in HTTP, there must be a single initial log + // with type=request and some body. We don't have a real single request + // (requests are written separately, 1 log per request), so we fake + // the required request log. + LOG(span.GetLogLevel()) + << "gRPC request stream" + << logging::LogExtra{{"body", "request stream started"}, + {"type", "request"}}; } context.Next(); From 9f9c19d8a00178a20910c0867074f9c4a53cfbf1 Mon Sep 17 00:00:00 2001 From: antonyzhilin Date: Tue, 23 Jul 2024 17:11:43 +0300 Subject: [PATCH 069/629] feat grpc: add GenericServiceBase Implements [grpc::AsyncGenericService](https://grpc.github.io/grpc/cpp/classgrpc_1_1_async_generic_service.html) for ugrpc. See documentation and sample usage on `ugrpc::server::GenericServiceBase`. bcd265729dbe92020859711e39521e925de6d11b --- .mapping.json | 49 ++++---- .../userver/utils/statistics/testing.hpp | 3 + core/utest/src/utils/statistics/testing.cpp | 7 ++ grpc/CMakeLists.txt | 4 +- grpc/include/userver/ugrpc/client/generic.hpp | 2 +- grpc/include/userver/ugrpc/client/rpc.hpp | 2 +- .../ugrpc/server/generic_service_base.hpp | 74 ++++++++++++ .../ugrpc/server/impl/async_service.hpp | 4 +- .../userver/ugrpc/server/impl/call_traits.hpp | 5 + .../ugrpc/server/impl/service_worker_impl.hpp | 101 +++++++++------- grpc/include/userver/ugrpc/server/rpc.hpp | 39 ++++-- grpc/include/userver/ugrpc/server/server.hpp | 5 + .../src/ugrpc/server/generic_service_base.cpp | 11 ++ .../server/impl/generic_service_worker.cpp | 97 +++++++++++++++ .../server/impl/generic_service_worker.hpp | 35 ++++++ .../ugrpc/server/impl/service_worker_impl.cpp | 15 +++ grpc/src/ugrpc/server/server.cpp | 43 ++++++- grpc/tests/{src => }/async_test.cpp | 0 grpc/tests/{src => }/baggage_test.cpp | 0 grpc/tests/{src => }/base_test.cpp | 0 grpc/tests/{src => }/cancel_test.cpp | 0 grpc/tests/{src => }/channels_test.cpp | 0 grpc/tests/{src => }/client_cancel_test.cpp | 0 grpc/tests/{src => }/client_factory_test.cpp | 0 .../tests/{src => }/deadline_metrics_test.cpp | 0 grpc/tests/{src => }/deadline_test.cpp | 0 grpc/tests/{src => }/error_test.cpp | 0 grpc/tests/{src => }/generic_client_test.cpp | 10 +- grpc/tests/generic_server_test.cpp | 114 ++++++++++++++++++ grpc/tests/{src => }/logging_test.cpp | 0 grpc/tests/{src => }/middlewares_test.cpp | 0 grpc/tests/{src => }/serialization_test.cpp | 0 grpc/tests/{src => }/service_config_test.cpp | 0 grpc/tests/{src => }/statistics_test.cpp | 0 grpc/tests/{src => }/stream_test.cpp | 0 .../{src => }/tests/deadline_helpers.hpp | 0 .../{src => }/tests/service_multichannel.hpp | 0 grpc/tests/{src => }/tracing_test.cpp | 0 grpc/tests/{src => }/unimplemented_test.cpp | 0 grpc/tests/{src => }/wait_any_test.cpp | 0 .../include/userver/ugrpc/tests/service.hpp | 9 ++ .../userver/ugrpc/tests/service_fixtures.hpp | 12 +- grpc/utest/src/ugrpc/tests/service.cpp | 17 ++- 43 files changed, 563 insertions(+), 95 deletions(-) create mode 100644 grpc/include/userver/ugrpc/server/generic_service_base.hpp create mode 100644 grpc/src/ugrpc/server/generic_service_base.cpp create mode 100644 grpc/src/ugrpc/server/impl/generic_service_worker.cpp create mode 100644 grpc/src/ugrpc/server/impl/generic_service_worker.hpp rename grpc/tests/{src => }/async_test.cpp (100%) rename grpc/tests/{src => }/baggage_test.cpp (100%) rename grpc/tests/{src => }/base_test.cpp (100%) rename grpc/tests/{src => }/cancel_test.cpp (100%) rename grpc/tests/{src => }/channels_test.cpp (100%) rename grpc/tests/{src => }/client_cancel_test.cpp (100%) rename grpc/tests/{src => }/client_factory_test.cpp (100%) rename grpc/tests/{src => }/deadline_metrics_test.cpp (100%) rename grpc/tests/{src => }/deadline_test.cpp (100%) rename grpc/tests/{src => }/error_test.cpp (100%) rename grpc/tests/{src => }/generic_client_test.cpp (95%) create mode 100644 grpc/tests/generic_server_test.cpp rename grpc/tests/{src => }/logging_test.cpp (100%) rename grpc/tests/{src => }/middlewares_test.cpp (100%) rename grpc/tests/{src => }/serialization_test.cpp (100%) rename grpc/tests/{src => }/service_config_test.cpp (100%) rename grpc/tests/{src => }/statistics_test.cpp (100%) rename grpc/tests/{src => }/stream_test.cpp (100%) rename grpc/tests/{src => }/tests/deadline_helpers.hpp (100%) rename grpc/tests/{src => }/tests/service_multichannel.hpp (100%) rename grpc/tests/{src => }/tracing_test.cpp (100%) rename grpc/tests/{src => }/unimplemented_test.cpp (100%) rename grpc/tests/{src => }/wait_any_test.cpp (100%) diff --git a/.mapping.json b/.mapping.json index 18cce9a9c33e..6b5726cc5fd0 100644 --- a/.mapping.json +++ b/.mapping.json @@ -1828,6 +1828,7 @@ "grpc/include/userver/ugrpc/impl/statistics_storage.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/impl/statistics_storage.hpp", "grpc/include/userver/ugrpc/proto_json.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/proto_json.hpp", "grpc/include/userver/ugrpc/server/exceptions.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/server/exceptions.hpp", + "grpc/include/userver/ugrpc/server/generic_service_base.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/server/generic_service_base.hpp", "grpc/include/userver/ugrpc/server/impl/async_method_invocation.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/server/impl/async_method_invocation.hpp", "grpc/include/userver/ugrpc/server/impl/async_methods.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/server/impl/async_methods.hpp", "grpc/include/userver/ugrpc/server/impl/async_service.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/server/impl/async_service.hpp", @@ -1906,9 +1907,12 @@ "grpc/src/ugrpc/impl/to_string.hpp":"taxi/uservices/userver/grpc/src/ugrpc/impl/to_string.hpp", "grpc/src/ugrpc/proto_json.cpp":"taxi/uservices/userver/grpc/src/ugrpc/proto_json.cpp", "grpc/src/ugrpc/server/exceptions.cpp":"taxi/uservices/userver/grpc/src/ugrpc/server/exceptions.cpp", + "grpc/src/ugrpc/server/generic_service_base.cpp":"taxi/uservices/userver/grpc/src/ugrpc/server/generic_service_base.cpp", "grpc/src/ugrpc/server/impl/async_method_invocation.cpp":"taxi/uservices/userver/grpc/src/ugrpc/server/impl/async_method_invocation.cpp", "grpc/src/ugrpc/server/impl/async_methods.cpp":"taxi/uservices/userver/grpc/src/ugrpc/server/impl/async_methods.cpp", "grpc/src/ugrpc/server/impl/error_code.cpp":"taxi/uservices/userver/grpc/src/ugrpc/server/impl/error_code.cpp", + "grpc/src/ugrpc/server/impl/generic_service_worker.cpp":"taxi/uservices/userver/grpc/src/ugrpc/server/impl/generic_service_worker.cpp", + "grpc/src/ugrpc/server/impl/generic_service_worker.hpp":"taxi/uservices/userver/grpc/src/ugrpc/server/impl/generic_service_worker.hpp", "grpc/src/ugrpc/server/impl/parse_config.cpp":"taxi/uservices/userver/grpc/src/ugrpc/server/impl/parse_config.cpp", "grpc/src/ugrpc/server/impl/parse_config.hpp":"taxi/uservices/userver/grpc/src/ugrpc/server/impl/parse_config.hpp", "grpc/src/ugrpc/server/impl/queue_holder.cpp":"taxi/uservices/userver/grpc/src/ugrpc/server/impl/queue_holder.cpp", @@ -1936,28 +1940,29 @@ "grpc/src/ugrpc/server/service_base.cpp":"taxi/uservices/userver/grpc/src/ugrpc/server/service_base.cpp", "grpc/src/ugrpc/server/service_component_base.cpp":"taxi/uservices/userver/grpc/src/ugrpc/server/service_component_base.cpp", "grpc/src/ugrpc/status_codes.cpp":"taxi/uservices/userver/grpc/src/ugrpc/status_codes.cpp", - "grpc/tests/src/async_test.cpp":"taxi/uservices/userver/grpc/tests/src/async_test.cpp", - "grpc/tests/src/baggage_test.cpp":"taxi/uservices/userver/grpc/tests/src/baggage_test.cpp", - "grpc/tests/src/base_test.cpp":"taxi/uservices/userver/grpc/tests/src/base_test.cpp", - "grpc/tests/src/cancel_test.cpp":"taxi/uservices/userver/grpc/tests/src/cancel_test.cpp", - "grpc/tests/src/channels_test.cpp":"taxi/uservices/userver/grpc/tests/src/channels_test.cpp", - "grpc/tests/src/client_cancel_test.cpp":"taxi/uservices/userver/grpc/tests/src/client_cancel_test.cpp", - "grpc/tests/src/client_factory_test.cpp":"taxi/uservices/userver/grpc/tests/src/client_factory_test.cpp", - "grpc/tests/src/deadline_metrics_test.cpp":"taxi/uservices/userver/grpc/tests/src/deadline_metrics_test.cpp", - "grpc/tests/src/deadline_test.cpp":"taxi/uservices/userver/grpc/tests/src/deadline_test.cpp", - "grpc/tests/src/error_test.cpp":"taxi/uservices/userver/grpc/tests/src/error_test.cpp", - "grpc/tests/src/generic_client_test.cpp":"taxi/uservices/userver/grpc/tests/src/generic_client_test.cpp", - "grpc/tests/src/logging_test.cpp":"taxi/uservices/userver/grpc/tests/src/logging_test.cpp", - "grpc/tests/src/middlewares_test.cpp":"taxi/uservices/userver/grpc/tests/src/middlewares_test.cpp", - "grpc/tests/src/serialization_test.cpp":"taxi/uservices/userver/grpc/tests/src/serialization_test.cpp", - "grpc/tests/src/service_config_test.cpp":"taxi/uservices/userver/grpc/tests/src/service_config_test.cpp", - "grpc/tests/src/statistics_test.cpp":"taxi/uservices/userver/grpc/tests/src/statistics_test.cpp", - "grpc/tests/src/stream_test.cpp":"taxi/uservices/userver/grpc/tests/src/stream_test.cpp", - "grpc/tests/src/tests/deadline_helpers.hpp":"taxi/uservices/userver/grpc/tests/src/tests/deadline_helpers.hpp", - "grpc/tests/src/tests/service_multichannel.hpp":"taxi/uservices/userver/grpc/tests/src/tests/service_multichannel.hpp", - "grpc/tests/src/tracing_test.cpp":"taxi/uservices/userver/grpc/tests/src/tracing_test.cpp", - "grpc/tests/src/unimplemented_test.cpp":"taxi/uservices/userver/grpc/tests/src/unimplemented_test.cpp", - "grpc/tests/src/wait_any_test.cpp":"taxi/uservices/userver/grpc/tests/src/wait_any_test.cpp", + "grpc/tests/async_test.cpp":"taxi/uservices/userver/grpc/tests/async_test.cpp", + "grpc/tests/baggage_test.cpp":"taxi/uservices/userver/grpc/tests/baggage_test.cpp", + "grpc/tests/base_test.cpp":"taxi/uservices/userver/grpc/tests/base_test.cpp", + "grpc/tests/cancel_test.cpp":"taxi/uservices/userver/grpc/tests/cancel_test.cpp", + "grpc/tests/channels_test.cpp":"taxi/uservices/userver/grpc/tests/channels_test.cpp", + "grpc/tests/client_cancel_test.cpp":"taxi/uservices/userver/grpc/tests/client_cancel_test.cpp", + "grpc/tests/client_factory_test.cpp":"taxi/uservices/userver/grpc/tests/client_factory_test.cpp", + "grpc/tests/deadline_metrics_test.cpp":"taxi/uservices/userver/grpc/tests/deadline_metrics_test.cpp", + "grpc/tests/deadline_test.cpp":"taxi/uservices/userver/grpc/tests/deadline_test.cpp", + "grpc/tests/error_test.cpp":"taxi/uservices/userver/grpc/tests/error_test.cpp", + "grpc/tests/generic_client_test.cpp":"taxi/uservices/userver/grpc/tests/generic_client_test.cpp", + "grpc/tests/generic_server_test.cpp":"taxi/uservices/userver/grpc/tests/generic_server_test.cpp", + "grpc/tests/logging_test.cpp":"taxi/uservices/userver/grpc/tests/logging_test.cpp", + "grpc/tests/middlewares_test.cpp":"taxi/uservices/userver/grpc/tests/middlewares_test.cpp", + "grpc/tests/serialization_test.cpp":"taxi/uservices/userver/grpc/tests/serialization_test.cpp", + "grpc/tests/service_config_test.cpp":"taxi/uservices/userver/grpc/tests/service_config_test.cpp", + "grpc/tests/statistics_test.cpp":"taxi/uservices/userver/grpc/tests/statistics_test.cpp", + "grpc/tests/stream_test.cpp":"taxi/uservices/userver/grpc/tests/stream_test.cpp", + "grpc/tests/tests/deadline_helpers.hpp":"taxi/uservices/userver/grpc/tests/tests/deadline_helpers.hpp", + "grpc/tests/tests/service_multichannel.hpp":"taxi/uservices/userver/grpc/tests/tests/service_multichannel.hpp", + "grpc/tests/tracing_test.cpp":"taxi/uservices/userver/grpc/tests/tracing_test.cpp", + "grpc/tests/unimplemented_test.cpp":"taxi/uservices/userver/grpc/tests/unimplemented_test.cpp", + "grpc/tests/wait_any_test.cpp":"taxi/uservices/userver/grpc/tests/wait_any_test.cpp", "grpc/utest/include/userver/ugrpc/tests/service.hpp":"taxi/uservices/userver/grpc/utest/include/userver/ugrpc/tests/service.hpp", "grpc/utest/include/userver/ugrpc/tests/service_fixtures.hpp":"taxi/uservices/userver/grpc/utest/include/userver/ugrpc/tests/service_fixtures.hpp", "grpc/utest/src/ugrpc/tests/service.cpp":"taxi/uservices/userver/grpc/utest/src/ugrpc/tests/service.cpp", diff --git a/core/utest/include/userver/utils/statistics/testing.hpp b/core/utest/include/userver/utils/statistics/testing.hpp index 1757acdfccf7..9ab341fc30ea 100644 --- a/core/utest/include/userver/utils/statistics/testing.hpp +++ b/core/utest/include/userver/utils/statistics/testing.hpp @@ -74,6 +74,9 @@ class Snapshot final { /// @endcode void PrintTo(const Snapshot& data, std::ostream*); +/// @brief Support for gtest diagnostics for utils::statistics::MetricValue. +void PrintTo(MetricValue value, std::ostream*); + } // namespace utils::statistics USERVER_NAMESPACE_END diff --git a/core/utest/src/utils/statistics/testing.cpp b/core/utest/src/utils/statistics/testing.cpp index d01020de67d2..c83e39986331 100644 --- a/core/utest/src/utils/statistics/testing.cpp +++ b/core/utest/src/utils/statistics/testing.cpp @@ -11,6 +11,7 @@ #include #include +#include #include #include @@ -130,6 +131,7 @@ MetricValue Snapshot::SingleMetric(std::string path, } void PrintTo(const Snapshot& data, std::ostream* out) { + UASSERT(out); *out << "{"; for (const auto& [path, entry] : data.data_->metrics) { *out << fmt::format("{};{} {}", path, fmt::join(entry.labels, ";"), @@ -139,6 +141,11 @@ void PrintTo(const Snapshot& data, std::ostream* out) { *out << "}"; } +void PrintTo(MetricValue value, std::ostream* out) { + UASSERT(out); + *out << fmt::to_string(value); +} + } // namespace utils::statistics USERVER_NAMESPACE_END diff --git a/grpc/CMakeLists.txt b/grpc/CMakeLists.txt index 2527ab3a8d94..9425b32cf1c0 100644 --- a/grpc/CMakeLists.txt +++ b/grpc/CMakeLists.txt @@ -24,7 +24,7 @@ file(GLOB_RECURSE SOURCES file(GLOB_RECURSE UNIT_TEST_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/src/*_test.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/tests/src/*pp + ${CMAKE_CURRENT_SOURCE_DIR}/tests/*pp ${CMAKE_CURRENT_SOURCE_DIR}/utest/src/*_test.cpp ) list(REMOVE_ITEM SOURCES ${UNIT_TEST_SOURCES}) @@ -149,7 +149,7 @@ if (USERVER_IS_THE_ROOT_PROJECT) add_executable(${PROJECT_NAME}-unittest ${UNIT_TEST_SOURCES}) target_include_directories(${PROJECT_NAME}-unittest PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src - ${CMAKE_CURRENT_SOURCE_DIR}/tests/src + ${CMAKE_CURRENT_SOURCE_DIR}/tests ) target_link_libraries(${PROJECT_NAME}-unittest PUBLIC diff --git a/grpc/include/userver/ugrpc/client/generic.hpp b/grpc/include/userver/ugrpc/client/generic.hpp index 7a1222f236f6..bfd5d819af37 100644 --- a/grpc/include/userver/ugrpc/client/generic.hpp +++ b/grpc/include/userver/ugrpc/client/generic.hpp @@ -58,7 +58,7 @@ struct GenericOptions { /// /// ## Example GenericClient usage with known message types /// -/// @snippet grpc/tests/src/generic_client_test.cpp sample +/// @snippet grpc/tests/generic_client_test.cpp sample class GenericClient final { public: GenericClient(GenericClient&&) noexcept = default; diff --git a/grpc/include/userver/ugrpc/client/rpc.hpp b/grpc/include/userver/ugrpc/client/rpc.hpp index a9262d62f703..864128928b85 100644 --- a/grpc/include/userver/ugrpc/client/rpc.hpp +++ b/grpc/include/userver/ugrpc/client/rpc.hpp @@ -336,7 +336,7 @@ class [[nodiscard]] OutputStream final : public CallAnyBase { /// will throw an exception. /// ## Usage example: /// -/// @snippet grpc/tests/src/stream_test.cpp concurrent bidirectional stream +/// @snippet grpc/tests/stream_test.cpp concurrent bidirectional stream /// template class [[nodiscard]] BidirectionalStream final : public CallAnyBase { diff --git a/grpc/include/userver/ugrpc/server/generic_service_base.hpp b/grpc/include/userver/ugrpc/server/generic_service_base.hpp new file mode 100644 index 000000000000..e0b3b172e506 --- /dev/null +++ b/grpc/include/userver/ugrpc/server/generic_service_base.hpp @@ -0,0 +1,74 @@ +#pragma once + +/// @file userver/ugrpc/server/generic_service_base.hpp +/// @brief @copybrief ugrpc::server::GenericServiceBase + +#include + +#include +#include + +USERVER_NAMESPACE_BEGIN + +namespace ugrpc::server { + +/// @brief Allows to handle RPCs with dynamic method names. +/// +/// To use: +/// +/// 1. inherit from this class in e.g. `YourGenericService`; +/// 2. put `YourGenericService` in a descendent of @ref ServiceComponentBase, +/// e.g. `YourGenericServiceComponent` +/// +/// Alternatively, just inherit from @ref GenericServiceBase::Component, with +/// the disadvantage that the service will not be unit-testable. +/// +/// The API is mainly intended for proxies, where the request-response body is +/// passed unchanged, with settings taken solely from the RPC metadata. +/// In cases where the code needs to operate on the actual messages, +/// serialization of requests and responses is left as an excercise to the user. +/// +/// Middlewares are customizable and are applied as usual, except that no +/// message hooks are called, meaning that there won't be any logs of messages +/// from the default middleware. +/// +/// Metrics are written per-method by default. If malicious clients initiate +/// RPCs with a lot of different garbage names, it may cause OOM due to +/// per-method metrics piling up. +/// TODO(TAXICOMMON-9108) allow limiting the amount of metrics. +/// +/// Statically-typed services, if registered, take priority over generic +/// services. It only makes sense to register at most 1 generic service. +/// +/// ## Example GenericServiceBase usage with known message types +/// +/// @snippet grpc/tests/generic_server_test.cpp sample +class GenericServiceBase { + public: + /// Inherits from both @ref GenericServiceBase and @ref ServiceComponentBase. + /// Allows to implement the service directly in a component. + /// The disadvantage is that such services are not unit-testable. + using Component = impl::ServiceComponentBase; + + using Call = BidirectionalStream; + + GenericServiceBase(GenericServiceBase&&) = delete; + GenericServiceBase& operator=(GenericServiceBase&&) = delete; + virtual ~GenericServiceBase(); + + /// @brief Override this method in the derived class to handle all RPCs. + /// RPC name can be obtained through @ref CallAnyBase::GetCallName. + /// @note RPCs of all kinds (including unary RPCs) are represented using + /// BidirectionalStream API. + /// @note The implementation of the method should call `Finish` or + /// `FinishWithError`, otherwise the server will respond with an "internal + /// server error" status. + virtual void Handle(Call& call) = 0; + + protected: + GenericServiceBase() = default; +}; + +} // namespace ugrpc::server + +USERVER_NAMESPACE_END diff --git a/grpc/include/userver/ugrpc/server/impl/async_service.hpp b/grpc/include/userver/ugrpc/server/impl/async_service.hpp index 0c0899525b60..b36aa04344a3 100644 --- a/grpc/include/userver/ugrpc/server/impl/async_service.hpp +++ b/grpc/include/userver/ugrpc/server/impl/async_service.hpp @@ -38,9 +38,11 @@ class AsyncService final : public Service::Service { this->RequestAsyncServerStreaming(method_id, &context, &initial_request, &stream, &call_cq, ¬ification_cq, tag); - } else { + } else if constexpr (kCallCategory == CallCategory::kBidirectionalStream) { this->RequestAsyncBidiStreaming(method_id, &context, &stream, &call_cq, ¬ification_cq, tag); + } else { + static_assert(!sizeof(CallTraits), "Invalid kCallCategory"); } } }; diff --git a/grpc/include/userver/ugrpc/server/impl/call_traits.hpp b/grpc/include/userver/ugrpc/server/impl/call_traits.hpp index bf09f55b4835..fde07140f2e4 100644 --- a/grpc/include/userver/ugrpc/server/impl/call_traits.hpp +++ b/grpc/include/userver/ugrpc/server/impl/call_traits.hpp @@ -13,6 +13,7 @@ enum class CallCategory { kInputStream, kOutputStream, kBidirectionalStream, + kGeneric, }; template @@ -28,6 +29,7 @@ struct CallTraits&, using RawCall = impl::RawResponseWriter; using InitialRequest = Request; using Call = UnaryCall; + using ContextType = ::grpc::ServerContext; using ServiceMethod = void (ServiceBase::*)(Call&, Request&&); static constexpr auto kCallCategory = CallCategory::kUnary; }; @@ -42,6 +44,7 @@ struct CallTraits; using InitialRequest = NoInitialRequest; using Call = InputStream; + using ContextType = ::grpc::ServerContext; using ServiceMethod = void (ServiceBase::*)(Call&); static constexpr auto kCallCategory = CallCategory::kInputStream; }; @@ -56,6 +59,7 @@ struct CallTraits&, using RawCall = impl::RawWriter; using InitialRequest = Request; using Call = OutputStream; + using ContextType = ::grpc::ServerContext; using ServiceMethod = void (ServiceBase::*)(Call&, Request&&); static constexpr auto kCallCategory = CallCategory::kOutputStream; }; @@ -70,6 +74,7 @@ struct CallTraits; using InitialRequest = NoInitialRequest; using Call = BidirectionalStream; + using ContextType = ::grpc::ServerContext; using ServiceMethod = void (ServiceBase::*)(Call&); static constexpr auto kCallCategory = CallCategory::kBidirectionalStream; }; diff --git a/grpc/include/userver/ugrpc/server/impl/service_worker_impl.hpp b/grpc/include/userver/ugrpc/server/impl/service_worker_impl.hpp index 01a54338f185..64e2608f78b0 100644 --- a/grpc/include/userver/ugrpc/server/impl/service_worker_impl.hpp +++ b/grpc/include/userver/ugrpc/server/impl/service_worker_impl.hpp @@ -58,30 +58,35 @@ void ReportCustomError( void SetupSpan(std::optional& span_holder, grpc::ServerContext& context, std::string_view call_name); +void ParseGenericCallName(std::string_view generic_call_name, + std::string_view& call_name, + std::string_view& service_name, + std::string_view& method_name); + /// Per-gRPC-service data template struct ServiceData final { ServiceData(const ServiceSettings& settings, - const ugrpc::impl::StaticServiceMetadata& metadata) + const ugrpc::impl::StaticServiceMetadata& metadata, + ugrpc::impl::ServiceStatistics* service_statistics) : settings(settings), metadata(metadata), - statistics(settings.statistics_storage.GetServiceStatistics( - metadata, std::nullopt)) {} + service_statistics(service_statistics) {} - ~ServiceData() = default; + ~ServiceData() { wait_tokens.WaitForAllTokens(); } const ServiceSettings settings; const ugrpc::impl::StaticServiceMetadata metadata; AsyncService async_service{metadata.method_full_names.size()}; utils::impl::WaitTokenStorage wait_tokens; - ugrpc::impl::ServiceStatistics& statistics; + ugrpc::impl::ServiceStatistics* const service_statistics; }; /// Per-gRPC-method data template struct MethodData final { ServiceData& service_data; - int queue_num{0}; + const std::size_t queue_id{}; const std::size_t method_id{}; typename CallTraits::ServiceBase& service; const typename CallTraits::ServiceMethod service_method; @@ -91,8 +96,10 @@ struct MethodData final { // Remove name of the service and slash std::string_view method_name{ call_name.substr(service_data.metadata.service_full_name.size() + 1)}; - ugrpc::impl::MethodStatistics& statistics{ - service_data.statistics.GetMethodStatistics(method_id)}; + ugrpc::impl::MethodStatistics* const statistics{ + service_data.service_statistics + ? &service_data.service_statistics->GetMethodStatistics(method_id) + : nullptr}; }; template @@ -116,10 +123,10 @@ class CallData final { context_.AsyncNotifyWhenDone(notify_when_done.GetTag()); - // the request for an incoming RPC must be performed synchronously auto& queue = method_data_.service_data.settings.queue.GetQueue( - method_data_.queue_num); + method_data_.queue_id); + // the request for an incoming RPC must be performed synchronously method_data_.service_data.async_service.template Prepare( method_data_.method_id, context_, initial_request_, raw_responder_, queue, queue, prepare_.GetTag()); @@ -137,16 +144,18 @@ class CallData final { return; } + utils::FastScopeGuard await_notify_when_done([&]() noexcept { + // Even if we finished before receiving notification that call is done, we + // should wait on this async operation. CompletionQueue has a pointer to + // stack-allocated object, that object is going to be freed upon exit. To + // prevent segfaults, wait until queue is done with this object. + notify_when_done.Wait(); + }); + // start a concurrent listener immediately, as advised by gRPC docs ListenAsync(method_data_); HandleRpc(); - - // Even if we finished before receiving notification that call is done, we - // should wait on this async operation. CompletionQueue has a pointer to - // stack-allocated object, that object is going to be freed upon exit. To - // prevent segfaults, wait until queue is done with this object. - notify_when_done.Wait(); } static void ListenAsync(const MethodData& data) { @@ -162,20 +171,24 @@ class CallData final { using Call = typename CallTraits::Call; void HandleRpc() { - const auto call_name = method_data_.call_name; - auto& service = method_data_.service; - const auto service_method = method_data_.service_method; - - const auto& service_name = - method_data_.service_data.metadata.service_full_name; - const auto& method_name = method_data_.method_name; + auto call_name = method_data_.call_name; + auto service_name = method_data_.service_data.metadata.service_full_name; + auto method_name = method_data_.method_name; + if constexpr (CallTraits::kCallCategory == CallCategory::kGeneric) { + ParseGenericCallName(context_.method(), call_name, service_name, + method_name); + } const auto& middlewares = method_data_.service_data.settings.middlewares; SetupSpan(span_, context_, call_name); utils::FastScopeGuard destroy_span([&]() noexcept { span_.reset(); }); - ugrpc::impl::RpcStatisticsScope statistics_scope(method_data_.statistics); + ugrpc::impl::RpcStatisticsScope statistics_scope{ + method_data_.statistics + ? *method_data_.statistics + : method_data_.service_data.settings.statistics_storage + .GetGenericStatistics(call_name, std::nullopt)}; auto& access_tskv_logger = method_data_.service_data.settings.access_tskv_logger; @@ -186,9 +199,10 @@ class CallData final { raw_responder_); auto do_call = [&] { if constexpr (std::is_same_v) { - (service.*service_method)(responder); + (method_data_.service.*(method_data_.service_method))(responder); } else { - (service.*service_method)(responder, std::move(initial_request_)); + (method_data_.service.*(method_data_.service_method))( + responder, std::move(initial_request_)); } }; @@ -219,13 +233,25 @@ class CallData final { MethodData method_data_; - grpc::ServerContext context_{}; + typename CallTraits::ContextType context_{}; InitialRequest initial_request_{}; RawCall raw_responder_{&context_}; ugrpc::impl::AsyncMethodInvocation prepare_; std::optional span_{}; }; +template +void StartServing(ServiceData& service_data, Service& service, + ServiceMethods... service_methods) { + for (std::size_t queue_id = 0; + queue_id < service_data.settings.queue.GetSize(); ++queue_id) { + std::size_t method_id = 0; + (CallData>::ListenAsync( + {service_data, queue_id, method_id++, service, service_methods}), + ...); + } +} + template class ServiceWorkerImpl final : public ServiceWorker { public: @@ -233,20 +259,12 @@ class ServiceWorkerImpl final : public ServiceWorker { ServiceWorkerImpl(ServiceSettings&& settings, ugrpc::impl::StaticServiceMetadata&& metadata, Service& service, ServiceMethods... service_methods) - : service_data_(settings, metadata), - start_{[this, &service, service_methods...] { - for (size_t i = 0; i < service_data_.settings.queue.GetSize(); i++) { - std::size_t method_id = 0; - (CallData>::ListenAsync( - {service_data_, static_cast(i), method_id++, service, - service_methods}), - ...); - } - }} {} - - ~ServiceWorkerImpl() override { - service_data_.wait_tokens.WaitForAllTokens(); - } + : statistics_(settings.statistics_storage.GetServiceStatistics( + metadata, std::nullopt)), + service_data_(std::move(settings), std::move(metadata), &statistics_), + start_([this, &service, service_methods...] { + impl::StartServing(service_data_, service, service_methods...); + }) {} grpc::Service& GetService() override { return service_data_.async_service; } @@ -257,6 +275,7 @@ class ServiceWorkerImpl final : public ServiceWorker { void Start() override { start_(); } private: + ugrpc::impl::ServiceStatistics& statistics_; ServiceData service_data_; std::function start_; }; diff --git a/grpc/include/userver/ugrpc/server/rpc.hpp b/grpc/include/userver/ugrpc/server/rpc.hpp index 1a822f8fdc95..1c7e8799fd24 100644 --- a/grpc/include/userver/ugrpc/server/rpc.hpp +++ b/grpc/include/userver/ugrpc/server/rpc.hpp @@ -3,6 +3,7 @@ /// @file userver/ugrpc/server/rpc.hpp /// @brief Classes representing an incoming RPC +#include #include #include #include @@ -468,13 +469,15 @@ void InputStream::Finish(Response& response) { UINVARIANT(state_ != State::kFinished, "'Finish' called on a finished stream"); state_ = State::kFinished; - LogFinish(grpc::Status::OK); + + const auto& status = grpc::Status::OK; + LogFinish(status); ApplyResponseHook(&response); - impl::Finish(stream_, response, grpc::Status::OK, GetCallName()); - Statistics().OnExplicitFinish(grpc::StatusCode::OK); - ugrpc::impl::UpdateSpanWithStatus(GetSpan(), grpc::Status::OK); + impl::Finish(stream_, response, status, GetCallName()); + Statistics().OnExplicitFinish(status.error_code()); + ugrpc::impl::UpdateSpanWithStatus(GetSpan(), status); } template @@ -536,7 +539,8 @@ void OutputStream::Finish() { UINVARIANT(state_ != State::kFinished, "'Finish' called on a finished stream"); state_ = State::kFinished; - const auto status = grpc::Status::OK; + + const auto& status = grpc::Status::OK; LogFinish(status); impl::Finish(stream_, status, GetCallName()); Statistics().OnExplicitFinish(grpc::StatusCode::OK); @@ -570,12 +574,14 @@ void OutputStream::WriteAndFinish(Response& response) { // may never actually be delivered grpc::WriteOptions write_options{}; - const auto status = grpc::Status::OK; + const auto& status = grpc::Status::OK; LogFinish(status); ApplyResponseHook(&response); impl::WriteAndFinish(stream_, response, write_options, status, GetCallName()); + Statistics().OnExplicitFinish(grpc::StatusCode::OK); + ugrpc::impl::UpdateSpanWithStatus(GetSpan(), status); } template @@ -603,7 +609,9 @@ bool BidirectionalStream::Read(Request& request) { UINVARIANT(!are_reads_done_, "'Read' called while the stream is half-closed for reads"); if (impl::Read(stream_, request)) { - ApplyRequestHook(&request); + if constexpr (std::is_base_of_v) { + ApplyRequestHook(&request); + } return true; } else { are_reads_done_ = true; @@ -623,7 +631,9 @@ void BidirectionalStream::Write(Response& response) { // Don't buffer writes, optimize for ping-pong-style interaction grpc::WriteOptions write_options{}; - ApplyResponseHook(&response); + if constexpr (std::is_base_of_v) { + ApplyResponseHook(&response); + } try { impl::Write(stream_, response, write_options, GetCallName()); @@ -637,7 +647,8 @@ template void BidirectionalStream::Finish() { UINVARIANT(!is_finished_, "'Finish' called on a finished stream"); is_finished_ = true; - const auto status = grpc::Status::OK; + + const auto& status = grpc::Status::OK; LogFinish(status); impl::Finish(stream_, status, GetCallName()); Statistics().OnExplicitFinish(grpc::StatusCode::OK); @@ -671,12 +682,16 @@ void BidirectionalStream::WriteAndFinish( // Don't buffer writes, optimize for ping-pong-style interaction grpc::WriteOptions write_options{}; - const auto status = grpc::Status::OK; + const auto& status = grpc::Status::OK; + LogFinish(status); - ApplyResponseHook(&response); + if constexpr (std::is_base_of_v) { + ApplyResponseHook(&response); + } - LogFinish(status); impl::WriteAndFinish(stream_, response, write_options, status, GetCallName()); + Statistics().OnExplicitFinish(status.error_code()); + ugrpc::impl::UpdateSpanWithStatus(GetSpan(), status); } template diff --git a/grpc/include/userver/ugrpc/server/server.hpp b/grpc/include/userver/ugrpc/server/server.hpp index 769d6a941ce1..90d2985319a8 100644 --- a/grpc/include/userver/ugrpc/server/server.hpp +++ b/grpc/include/userver/ugrpc/server/server.hpp @@ -27,6 +27,8 @@ USERVER_NAMESPACE_BEGIN namespace ugrpc::server { +class GenericServiceBase; + /// Settings relating to the whole gRPC server struct ServerConfig final { /// The port to listen to. If `0`, a free port will be picked automatically. @@ -80,6 +82,9 @@ class Server final /// least until `Stop` is called. void AddService(ServiceBase& service, ServiceConfig&& config); + /// @overload + void AddService(GenericServiceBase& service, ServiceConfig&& config); + /// @brief Get names of all registered services std::vector GetServiceNames() const; diff --git a/grpc/src/ugrpc/server/generic_service_base.cpp b/grpc/src/ugrpc/server/generic_service_base.cpp new file mode 100644 index 000000000000..4cfc288431c4 --- /dev/null +++ b/grpc/src/ugrpc/server/generic_service_base.cpp @@ -0,0 +1,11 @@ +#include + +USERVER_NAMESPACE_BEGIN + +namespace ugrpc::server { + +GenericServiceBase::~GenericServiceBase() = default; + +} // namespace ugrpc::server + +USERVER_NAMESPACE_END diff --git a/grpc/src/ugrpc/server/impl/generic_service_worker.cpp b/grpc/src/ugrpc/server/impl/generic_service_worker.cpp new file mode 100644 index 000000000000..0ac561bef1f6 --- /dev/null +++ b/grpc/src/ugrpc/server/impl/generic_service_worker.cpp @@ -0,0 +1,97 @@ +#include + +#include + +#include +#include +#include + +USERVER_NAMESPACE_BEGIN + +namespace ugrpc::server::impl { + +namespace { + +constexpr std::string_view kGenericMethodFullNamesFake[] = { + "Generic/Generic", +}; +constexpr std::string_view kGenericServiceNameFake = "Generic"; + +} // namespace + +template <> +struct CallTraits + final { + using ServiceBase = GenericServiceBase; + using Request = grpc::ByteBuffer; + using Response = grpc::ByteBuffer; + using RawCall = impl::RawReaderWriter; + using InitialRequest = NoInitialRequest; + using Call = BidirectionalStream; + using ContextType = grpc::GenericServerContext; + using ServiceMethod = void (ServiceBase::*)(Call&); + static constexpr auto kCallCategory = CallCategory::kGeneric; +}; + +struct GenericServiceTag final {}; + +template <> +class AsyncService final { + public: + explicit AsyncService(std::size_t method_count) { + UASSERT(method_count == 1); + } + + template + void Prepare(int method_id, grpc::GenericServerContext& context, + typename CallTraits::InitialRequest& /*initial_request*/, + typename CallTraits::RawCall& stream, + grpc::CompletionQueue& call_cq, + grpc::ServerCompletionQueue& notification_cq, void* tag) { + static_assert(CallTraits::kCallCategory == CallCategory::kGeneric); + UASSERT(method_id == 0); + service_.RequestCall(&context, &stream, &call_cq, ¬ification_cq, tag); + } + + grpc::AsyncGenericService& GetService() { return service_; } + + private: + grpc::AsyncGenericService service_; +}; + +struct GenericServiceWorker::Impl { + Impl(GenericServiceBase& service, ServiceSettings&& settings) + : service(service), + service_data(std::move(settings), + ugrpc::impl::StaticServiceMetadata{ + kGenericServiceNameFake, kGenericMethodFullNamesFake}, + /*service_statistics*/ nullptr) {} + + GenericServiceBase& service; + ServiceData service_data; +}; + +GenericServiceWorker::GenericServiceWorker(GenericServiceBase& service, + ServiceSettings&& settings) + : impl_(service, std::move(settings)) {} + +GenericServiceWorker::GenericServiceWorker(GenericServiceWorker&&) noexcept = + default; + +GenericServiceWorker& GenericServiceWorker::operator=( + GenericServiceWorker&&) noexcept = default; + +GenericServiceWorker::~GenericServiceWorker() = default; + +grpc::AsyncGenericService& GenericServiceWorker::GetService() { + return impl_->service_data.async_service.GetService(); +} + +void GenericServiceWorker::Start() { + impl::StartServing(impl_->service_data, impl_->service, + &GenericServiceBase::Handle); +} + +} // namespace ugrpc::server::impl + +USERVER_NAMESPACE_END diff --git a/grpc/src/ugrpc/server/impl/generic_service_worker.hpp b/grpc/src/ugrpc/server/impl/generic_service_worker.hpp new file mode 100644 index 000000000000..3c9f2c56fcaa --- /dev/null +++ b/grpc/src/ugrpc/server/impl/generic_service_worker.hpp @@ -0,0 +1,35 @@ +#pragma once + +#include + +#include +#include + +USERVER_NAMESPACE_BEGIN + +namespace ugrpc::server { +class GenericServiceBase; +} // namespace ugrpc::server + +namespace ugrpc::server::impl { + +class GenericServiceWorker final { + public: + GenericServiceWorker(GenericServiceBase& service, ServiceSettings&& settings); + + GenericServiceWorker(GenericServiceWorker&&) noexcept; + GenericServiceWorker& operator=(GenericServiceWorker&&) noexcept; + ~GenericServiceWorker(); + + grpc::AsyncGenericService& GetService(); + + void Start(); + + private: + struct Impl; + utils::Box impl_; +}; + +} // namespace ugrpc::server::impl + +USERVER_NAMESPACE_END diff --git a/grpc/src/ugrpc/server/impl/service_worker_impl.cpp b/grpc/src/ugrpc/server/impl/service_worker_impl.cpp index 53dc19b05a12..d57aaf9ddd97 100644 --- a/grpc/src/ugrpc/server/impl/service_worker_impl.cpp +++ b/grpc/src/ugrpc/server/impl/service_worker_impl.cpp @@ -100,6 +100,21 @@ void SetupSpan(std::optional& span_holder, ugrpc::impl::ToGrpcString(span.GetLink())); } +void ParseGenericCallName(std::string_view generic_call_name, + std::string_view& call_name, + std::string_view& service_name, + std::string_view& method_name) { + UINVARIANT(!generic_call_name.empty() && generic_call_name[0] == '/', + "Generic service call name must start with a '/'"); + generic_call_name.remove_prefix(1); + const auto slash_pos = generic_call_name.find('/'); + UINVARIANT(slash_pos != std::string_view::npos, + "Generic service call name must contain a '/'"); + call_name = generic_call_name; + service_name = generic_call_name.substr(0, slash_pos); + method_name = generic_call_name.substr(slash_pos + 1); +} + } // namespace ugrpc::server::impl USERVER_NAMESPACE_END diff --git a/grpc/src/ugrpc/server/server.cpp b/grpc/src/ugrpc/server/server.cpp index b8d1b3898e9a..b8531b9c822f 100644 --- a/grpc/src/ugrpc/server/server.cpp +++ b/grpc/src/ugrpc/server/server.cpp @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -80,6 +81,8 @@ class Server::Impl final { void AddService(ServiceBase& service, ServiceConfig&& config); + void AddService(GenericServiceBase& service, ServiceConfig&& config); + std::vector GetServiceNames() const; void WithServerBuilder(SetupHook setup); @@ -104,6 +107,8 @@ class Server::Impl final { kStopped, }; + impl::ServiceSettings MakeServiceSettings(ServiceConfig&& config); + void AddListeningPort(int port); void AddListeningUnixSocket(std::string_view path); @@ -114,6 +119,7 @@ class Server::Impl final { std::optional server_builder_; std::optional port_; std::vector> service_workers_; + std::vector generic_service_workers_; std::optional queue_; std::unique_ptr server_; mutable engine::Mutex configuration_mutex_; @@ -189,18 +195,31 @@ void Server::Impl::AddListeningUnixSocket(std::string_view path) { grpc::InsecureServerCredentials()); } -void Server::Impl::AddService(ServiceBase& service, ServiceConfig&& config) { - const std::lock_guard lock(configuration_mutex_); - UASSERT(state_ == State::kConfiguration); - - service_workers_.push_back(service.MakeWorker(impl::ServiceSettings{ +impl::ServiceSettings Server::Impl::MakeServiceSettings( + ServiceConfig&& config) { + return impl::ServiceSettings{ *queue_, config.task_processor, statistics_storage_, std::move(config.middlewares), access_tskv_logger_, config_source_, - })); + }; +} + +void Server::Impl::AddService(ServiceBase& service, ServiceConfig&& config) { + const std::lock_guard lock(configuration_mutex_); + UASSERT(state_ == State::kConfiguration); + service_workers_.push_back( + service.MakeWorker(MakeServiceSettings(std::move(config)))); +} + +void Server::Impl::AddService(GenericServiceBase& service, + ServiceConfig&& config) { + const std::lock_guard lock(configuration_mutex_); + UASSERT(state_ == State::kConfiguration); + generic_service_workers_.emplace_back(service, + MakeServiceSettings(std::move(config))); } std::vector Server::Impl::GetServiceNames() const { @@ -262,6 +281,7 @@ void Server::Impl::Stop() noexcept { server_->Shutdown(engine::Deadline::FromDuration(kShutdownGracePeriod)); } service_workers_.clear(); + generic_service_workers_.clear(); queue_.reset(); server_.reset(); @@ -275,6 +295,7 @@ void Server::Impl::StopServing() noexcept { server_->Shutdown(engine::Deadline::FromDuration(kShutdownGracePeriod)); } service_workers_.clear(); + generic_service_workers_.clear(); state_ = State::kServingStopped; } @@ -292,6 +313,9 @@ void Server::Impl::DoStart() { for (auto& worker : service_workers_) { server_builder_->RegisterService(&worker->GetService()); } + for (auto& worker : generic_service_workers_) { + server_builder_->RegisterAsyncGenericService(&worker.GetService()); + } server_ = server_builder_->BuildAndStart(); UINVARIANT(server_, "See grpcpp logs for details"); @@ -300,6 +324,9 @@ void Server::Impl::DoStart() { for (auto& worker : service_workers_) { worker->Start(); } + for (auto& worker : generic_service_workers_) { + worker.Start(); + } if (port_) { LOG_INFO() << "gRPC server started on port " << *port_; @@ -320,6 +347,10 @@ void Server::AddService(ServiceBase& service, ServiceConfig&& config) { impl_->AddService(service, std::move(config)); } +void Server::AddService(GenericServiceBase& service, ServiceConfig&& config) { + impl_->AddService(service, std::move(config)); +} + std::vector Server::GetServiceNames() const { return impl_->GetServiceNames(); } diff --git a/grpc/tests/src/async_test.cpp b/grpc/tests/async_test.cpp similarity index 100% rename from grpc/tests/src/async_test.cpp rename to grpc/tests/async_test.cpp diff --git a/grpc/tests/src/baggage_test.cpp b/grpc/tests/baggage_test.cpp similarity index 100% rename from grpc/tests/src/baggage_test.cpp rename to grpc/tests/baggage_test.cpp diff --git a/grpc/tests/src/base_test.cpp b/grpc/tests/base_test.cpp similarity index 100% rename from grpc/tests/src/base_test.cpp rename to grpc/tests/base_test.cpp diff --git a/grpc/tests/src/cancel_test.cpp b/grpc/tests/cancel_test.cpp similarity index 100% rename from grpc/tests/src/cancel_test.cpp rename to grpc/tests/cancel_test.cpp diff --git a/grpc/tests/src/channels_test.cpp b/grpc/tests/channels_test.cpp similarity index 100% rename from grpc/tests/src/channels_test.cpp rename to grpc/tests/channels_test.cpp diff --git a/grpc/tests/src/client_cancel_test.cpp b/grpc/tests/client_cancel_test.cpp similarity index 100% rename from grpc/tests/src/client_cancel_test.cpp rename to grpc/tests/client_cancel_test.cpp diff --git a/grpc/tests/src/client_factory_test.cpp b/grpc/tests/client_factory_test.cpp similarity index 100% rename from grpc/tests/src/client_factory_test.cpp rename to grpc/tests/client_factory_test.cpp diff --git a/grpc/tests/src/deadline_metrics_test.cpp b/grpc/tests/deadline_metrics_test.cpp similarity index 100% rename from grpc/tests/src/deadline_metrics_test.cpp rename to grpc/tests/deadline_metrics_test.cpp diff --git a/grpc/tests/src/deadline_test.cpp b/grpc/tests/deadline_test.cpp similarity index 100% rename from grpc/tests/src/deadline_test.cpp rename to grpc/tests/deadline_test.cpp diff --git a/grpc/tests/src/error_test.cpp b/grpc/tests/error_test.cpp similarity index 100% rename from grpc/tests/src/error_test.cpp rename to grpc/tests/error_test.cpp diff --git a/grpc/tests/src/generic_client_test.cpp b/grpc/tests/generic_client_test.cpp similarity index 95% rename from grpc/tests/src/generic_client_test.cpp rename to grpc/tests/generic_client_test.cpp index e58c939649c2..02444f9b139f 100644 --- a/grpc/tests/src/generic_client_test.cpp +++ b/grpc/tests/generic_client_test.cpp @@ -1,13 +1,13 @@ -#include +#include #include #include #include #include -#include #include #include +#include #include @@ -15,7 +15,8 @@ USERVER_NAMESPACE_BEGIN namespace { -const grpc::string kSayHelloCallName = "sample.ugrpc.UnitTestService/SayHello"; +constexpr std::string_view kSayHelloCallName = + "sample.ugrpc.UnitTestService/SayHello"; class UnitTestService final : public sample::ugrpc::UnitTestServiceBase { public: @@ -36,7 +37,8 @@ sample::ugrpc::GreetingResponse PerformGenericUnaryCall( /// [sample] const auto client = client_factory.MakeClient(); - const grpc::string call_name = "sample.ugrpc.UnitTestService/SayHello"; + constexpr std::string_view call_name = + "sample.ugrpc.UnitTestService/SayHello"; sample::ugrpc::GreetingRequest request; request.set_name("generic"); diff --git a/grpc/tests/generic_server_test.cpp b/grpc/tests/generic_server_test.cpp new file mode 100644 index 000000000000..e12bd4fc2fdf --- /dev/null +++ b/grpc/tests/generic_server_test.cpp @@ -0,0 +1,114 @@ +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +USERVER_NAMESPACE_BEGIN + +namespace { + +/// [sample] +constexpr std::string_view kSayHelloCallName = + "sample.ugrpc.UnitTestService/SayHello"; + +class SampleGenericService final : public ugrpc::server::GenericServiceBase { + public: + void Handle(Call& call) override { + EXPECT_EQ(call.GetCallName(), kSayHelloCallName); + + grpc::ByteBuffer request_bytes; + ASSERT_TRUE(call.Read(request_bytes)); + + sample::ugrpc::GreetingRequest request; + if (!ugrpc::ParseFromByteBuffer(std::move(request_bytes), request)) { + call.FinishWithError(grpc::Status{grpc::StatusCode::INVALID_ARGUMENT, + "Failed to parse request"}); + return; + } + + sample::ugrpc::GreetingResponse response; + response.set_name("Hello " + request.name()); + + call.WriteAndFinish(ugrpc::SerializeToByteBuffer(response)); + } +}; +/// [sample] + +class GenericServiceTest : public ugrpc::tests::ServiceFixtureBase { + protected: + GenericServiceTest() { + RegisterService(service_); + StartServer(); + } + + ~GenericServiceTest() override { StopServer(); } + + private: + SampleGenericService service_{}; +}; + +void PerformGenericUnaryCall( + const sample::ugrpc::UnitTestServiceClient& client) { + sample::ugrpc::GreetingRequest out; + out.set_name("generic"); + + auto call = client.SayHello(out); + + sample::ugrpc::GreetingResponse in; + UASSERT_NO_THROW(in = call.Finish()); + EXPECT_EQ(in.name(), "Hello generic"); +} + +} // namespace + +UTEST_F(GenericServiceTest, UnaryCall) { + PerformGenericUnaryCall(MakeClient()); +} + +UTEST_F(GenericServiceTest, Metrics) { + PerformGenericUnaryCall(MakeClient()); + + // Server writes metrics after Finish, after the client might have returned + // from Finish. + GetServer().StopServing(); + + const auto stats = GetStatistics( + "grpc.server.by-destination", + {{"grpc_method", "SayHello"}, + {"grpc_service", "sample.ugrpc.UnitTestService"}, + {"grpc_destination", "sample.ugrpc.UnitTestService/SayHello"}}); + UEXPECT_NO_THROW( + EXPECT_EQ(stats.SingleMetric("status", {{"grpc_code", "OK"}}), + utils::statistics::Rate{1}) + << testing::PrintToString(stats)) + << testing::PrintToString(stats); +} + +using GenericServerLoggingTest = utest::LogCaptureFixture; + +UTEST_F(GenericServerLoggingTest, Logs) { + PerformGenericUnaryCall(MakeClient()); + + // Server writes metrics after Finish, after the client might have returned + // from Finish. + GetServer().StopServing(); + + const auto span_log = + GetSingleLog(GetLogCapture().Filter([](const utest::LogRecord& record) { + const auto span_name = record.GetTagOptional("stopwatch_name"); + return span_name && utils::text::StartsWith(*span_name, "grpc/"); + })); + EXPECT_EQ(span_log.GetTagOptional("stopwatch_name"), + "grpc/sample.ugrpc.UnitTestService/SayHello") + << span_log; + EXPECT_EQ(span_log.GetTagOptional("grpc_status"), "OK") << span_log; +} + +USERVER_NAMESPACE_END diff --git a/grpc/tests/src/logging_test.cpp b/grpc/tests/logging_test.cpp similarity index 100% rename from grpc/tests/src/logging_test.cpp rename to grpc/tests/logging_test.cpp diff --git a/grpc/tests/src/middlewares_test.cpp b/grpc/tests/middlewares_test.cpp similarity index 100% rename from grpc/tests/src/middlewares_test.cpp rename to grpc/tests/middlewares_test.cpp diff --git a/grpc/tests/src/serialization_test.cpp b/grpc/tests/serialization_test.cpp similarity index 100% rename from grpc/tests/src/serialization_test.cpp rename to grpc/tests/serialization_test.cpp diff --git a/grpc/tests/src/service_config_test.cpp b/grpc/tests/service_config_test.cpp similarity index 100% rename from grpc/tests/src/service_config_test.cpp rename to grpc/tests/service_config_test.cpp diff --git a/grpc/tests/src/statistics_test.cpp b/grpc/tests/statistics_test.cpp similarity index 100% rename from grpc/tests/src/statistics_test.cpp rename to grpc/tests/statistics_test.cpp diff --git a/grpc/tests/src/stream_test.cpp b/grpc/tests/stream_test.cpp similarity index 100% rename from grpc/tests/src/stream_test.cpp rename to grpc/tests/stream_test.cpp diff --git a/grpc/tests/src/tests/deadline_helpers.hpp b/grpc/tests/tests/deadline_helpers.hpp similarity index 100% rename from grpc/tests/src/tests/deadline_helpers.hpp rename to grpc/tests/tests/deadline_helpers.hpp diff --git a/grpc/tests/src/tests/service_multichannel.hpp b/grpc/tests/tests/service_multichannel.hpp similarity index 100% rename from grpc/tests/src/tests/service_multichannel.hpp rename to grpc/tests/tests/service_multichannel.hpp diff --git a/grpc/tests/src/tracing_test.cpp b/grpc/tests/tracing_test.cpp similarity index 100% rename from grpc/tests/src/tracing_test.cpp rename to grpc/tests/tracing_test.cpp diff --git a/grpc/tests/src/unimplemented_test.cpp b/grpc/tests/unimplemented_test.cpp similarity index 100% rename from grpc/tests/src/unimplemented_test.cpp rename to grpc/tests/unimplemented_test.cpp diff --git a/grpc/tests/src/wait_any_test.cpp b/grpc/tests/wait_any_test.cpp similarity index 100% rename from grpc/tests/src/wait_any_test.cpp rename to grpc/tests/wait_any_test.cpp diff --git a/grpc/utest/include/userver/ugrpc/tests/service.hpp b/grpc/utest/include/userver/ugrpc/tests/service.hpp index 1c4a39928724..a7f8ee4412bb 100644 --- a/grpc/utest/include/userver/ugrpc/tests/service.hpp +++ b/grpc/utest/include/userver/ugrpc/tests/service.hpp @@ -18,6 +18,10 @@ USERVER_NAMESPACE_BEGIN +namespace ugrpc::server { +class GenericServiceBase; +} // namespace ugrpc::server + /// userver gRPC testing facilities namespace ugrpc::tests { @@ -36,6 +40,9 @@ class ServiceBase { /// should ensure that the services live at least until StopServer is called. void RegisterService(server::ServiceBase& service); + /// @overload + void RegisterService(server::GenericServiceBase& service); + /// Starts the server and connects a grpc channel to it. /// Should be called after the services are registered. void StartServer(client::ClientFactorySettings&& settings = {}); @@ -72,6 +79,8 @@ class ServiceBase { utils::statistics::Storage& GetStatisticsStorage(); private: + server::ServiceConfig MakeServiceConfig(); + utils::statistics::Storage statistics_storage_; dynamic_config::StorageMock config_storage_; server::Server server_; diff --git a/grpc/utest/include/userver/ugrpc/tests/service_fixtures.hpp b/grpc/utest/include/userver/ugrpc/tests/service_fixtures.hpp index ea47b0112b3f..7d87c8c361e9 100644 --- a/grpc/utest/include/userver/ugrpc/tests/service_fixtures.hpp +++ b/grpc/utest/include/userver/ugrpc/tests/service_fixtures.hpp @@ -16,7 +16,17 @@ namespace ugrpc::tests { /// @see @ref ugrpc::tests::ServiceBase // NOLINTNEXTLINE(fuchsia-multiple-inheritance) -class ServiceFixtureBase : protected ServiceBase, public ::testing::Test {}; +class ServiceFixtureBase : protected ServiceBase, public ::testing::Test { + protected: + /// @returns the statistics of the server and clients. + utils::statistics::Snapshot GetStatistics( + std::string prefix, + std::vector require_labels = {}) { + return utils::statistics::Snapshot{this->GetStatisticsStorage(), + std::move(prefix), + std::move(require_labels)}; + } +}; /// @see @ref ugrpc::tests::Service template diff --git a/grpc/utest/src/ugrpc/tests/service.cpp b/grpc/utest/src/ugrpc/tests/service.cpp index 47681b593320..3080e63af134 100644 --- a/grpc/utest/src/ugrpc/tests/service.cpp +++ b/grpc/utest/src/ugrpc/tests/service.cpp @@ -45,10 +45,19 @@ ServiceBase::~ServiceBase() = default; void ServiceBase::RegisterService(server::ServiceBase& service) { adding_middlewares_allowed_ = false; - server_.AddService(service, server::ServiceConfig{ - engine::current_task::GetTaskProcessor(), - server_middlewares_, - }); + server_.AddService(service, MakeServiceConfig()); +} + +void ServiceBase::RegisterService(server::GenericServiceBase& service) { + adding_middlewares_allowed_ = false; + server_.AddService(service, MakeServiceConfig()); +} + +server::ServiceConfig ServiceBase::MakeServiceConfig() { + return server::ServiceConfig{ + engine::current_task::GetTaskProcessor(), + server_middlewares_, + }; } void ServiceBase::StartServer( From 02ec868b514bae8fc900d202f00568906e3e87ea Mon Sep 17 00:00:00 2001 From: segoon Date: Tue, 23 Jul 2024 18:06:08 +0300 Subject: [PATCH 070/629] docs otlp: add more docs b9283ddd5067b1c56fef244f2ce03020a78bb7ed --- scripts/docs/en/userver/logging.md | 1 + scripts/docs/en/userver/tutorial/build.md | 2 ++ 2 files changed, 3 insertions(+) diff --git a/scripts/docs/en/userver/logging.md b/scripts/docs/en/userver/logging.md index 70d138cc04d7..367c03a63fda 100644 --- a/scripts/docs/en/userver/logging.md +++ b/scripts/docs/en/userver/logging.md @@ -229,6 +229,7 @@ For example, this is how you can disable logging of all Span for MongoDB (that i ``` +@anchor opentelemetry ## OpenTelemetry protocol It is possible to use OpenTelemetry protocol for logs and tracing exporting. diff --git a/scripts/docs/en/userver/tutorial/build.md b/scripts/docs/en/userver/tutorial/build.md index d068e1aa6fc9..e3921e13dffd 100644 --- a/scripts/docs/en/userver/tutorial/build.md +++ b/scripts/docs/en/userver/tutorial/build.md @@ -79,6 +79,7 @@ The following CMake options are used by userver: | USERVER_FEATURE_MYSQL | Provide asynchronous driver for MySQL/MariaDB | ${USERVER_IS_THE_ROOT_PROJECT} | | USERVER_FEATURE_ROCKS | Provide asynchronous driver for RocksDB | ${USERVER_IS_THE_ROOT_PROJECT} | | USERVER_FEATURE_YDB | Provide asynchronous driver for YDB | ${USERVER_IS_THE_ROOT_PROJECT} AND C++ standard >= 20 | +| USERVER_FEATURE_OTLP | Provide Logger for OpenTelemetry protocol | ${USERVER_IS_THE_ROOT_PROJECT} | | USERVER_FEATURE_UTEST | Provide 'utest' and 'ubench' for unit testing and benchmarking coroutines | ${USERVER_FEATURE_CORE} | | USERVER_FEATURE_CRYPTOPP_BLAKE2 | Provide wrappers for blake2 algorithms of crypto++ | ON | | USERVER_FEATURE_PATCH_LIBPQ | Apply patches to the libpq (add portals support), requires libpq.a | ON | @@ -165,6 +166,7 @@ userver is split into multiple CMake libraries. | `userver::mysql` | `USERVER_FEATURE_MYSQL` | `mysql` | @ref scripts/docs/en/userver/mysql/mysql_driver.md | | `userver::rocks` | `USERVER_FEATURE_ROCKS` | `rocks` | TODO | | `userver::ydb` | `USERVER_FEATURE_YDB` | `ydb` | TODO | +| `userver::otlp` | `USERVER_FEATURE_OTLP` | `otlp` | @ref opentelemetry "OpenTelemetry Protocol" | Make sure to: From 2e2bf6667bdf1c142ec033c65c3c86a6309c8cd6 Mon Sep 17 00:00:00 2001 From: Matrixon Date: Tue, 23 Jul 2024 18:13:26 +0300 Subject: [PATCH 071/629] feat core: allow #end, #file and #fallback in config_vars MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes: https://github.com/userver-framework/userver/issues/618 Tests: протестировано CI Co-authored-by: Antony Polukhin 0eedddab3d15d0e95dc422f94f2f9d528dee27ae Pull Request resolved: https://github.com/userver-framework/userver/pull/641 --- .../userver/yaml_config/yaml_config.hpp | 3 + universal/src/yaml_config/yaml_config.cpp | 105 ++++++++++++------ .../src/yaml_config/yaml_config_test.cpp | 52 +++++++++ 3 files changed, 123 insertions(+), 37 deletions(-) diff --git a/universal/include/userver/yaml_config/yaml_config.hpp b/universal/include/userver/yaml_config/yaml_config.hpp index 45078a3ce91c..636e150e442e 100644 --- a/universal/include/userver/yaml_config/yaml_config.hpp +++ b/universal/include/userver/yaml_config/yaml_config.hpp @@ -61,6 +61,9 @@ using ParseException = formats::yaml::ParseException; /// `yaml["some_element"]["value"].As()` is the value of `ENV_NAME` /// environment variable if it exists; otherwise it is `5`. /// +/// @note `#env`, `#file` and `#fallback` also work for keys inside +/// `config_vars`. +/// /// @warning YamlConfig::Mode::kEnvAllowed or /// YamlConfig::Mode::kEnvAndFileAllowed should be used only on configs that /// come from trusted environments. Otherwise, an attacker could create a diff --git a/universal/src/yaml_config/yaml_config.cpp b/universal/src/yaml_config/yaml_config.cpp index 9eb0c5daf2a3..281ddb7c3f5b 100644 --- a/universal/src/yaml_config/yaml_config.cpp +++ b/universal/src/yaml_config/yaml_config.cpp @@ -93,62 +93,87 @@ std::optional GetFromFileImpl( return formats::yaml::blocking::FromFile(str_filename); } -} // namespace - -YamlConfig::YamlConfig(formats::yaml::Value yaml, - formats::yaml::Value config_vars, Mode mode) - : yaml_(std::move(yaml)), - config_vars_(std::move(config_vars)), - mode_(mode) {} +std::optional GetSharpCommandValue(const formats::yaml::Value& yaml, + YamlConfig::Mode mode, + std::string_view key, + bool met_substitution) { + const auto env_name = yaml[GetEnvName(key)]; + auto env_value = GetFromEnvImpl(env_name, mode); + if (env_value) { + // Strip substitutions off to disallow nested substitutions + return YamlConfig{std::move(*env_value), {}, YamlConfig::Mode::kSecure}; + } -const formats::yaml::Value& YamlConfig::Yaml() const { return yaml_; } + const auto file_name = yaml[GetFileName(key)]; + auto file_value = GetFromFileImpl(file_name, mode); + if (file_value) { + // Strip substitutions off to disallow nested substitutions + return YamlConfig{std::move(*file_value), {}, YamlConfig::Mode::kSecure}; + } -YamlConfig YamlConfig::operator[](std::string_view key) const { - if (utils::text::EndsWith(key, "#env") || - utils::text::EndsWith(key, "#file") || - utils::text::EndsWith(key, "#fallback")) { - UASSERT_MSG(false, "Do not use names ending on #env, #file and #fallback"); - return MakeMissingConfig(*this, key); + if (met_substitution || !env_name.IsMissing() || !file_name.IsMissing()) { + const auto fallback_name = GetFallbackName(key); + if (yaml.HasMember(fallback_name)) { + LOG_INFO() << "using fallback value for '" << key << '\''; + // Strip substitutions off to disallow nested substitutions + return YamlConfig{yaml[fallback_name], {}, YamlConfig::Mode::kSecure}; + } } - auto value = yaml_[key]; + return {}; +} + +std::optional GetYamlConfig(const formats::yaml::Value& yaml, + const formats::yaml::Value& config_vars, + YamlConfig::Mode mode, + std::string_view key) { + auto value = yaml[key]; const bool is_substitution = IsSubstitution(value); if (is_substitution) { const auto var_name = GetSubstitutionVarName(value); - - auto var_data = config_vars_[var_name]; + auto var_data = config_vars[var_name]; if (!var_data.IsMissing()) { // Strip substitutions off to disallow nested substitutions - return YamlConfig{std::move(var_data), {}, Mode::kSecure}; + return YamlConfig{std::move(var_data), {}, YamlConfig::Mode::kSecure}; + } + + auto res = GetSharpCommandValue(config_vars, mode, var_name, + /*met_substitution*/ false); + if (res) { + return std::move(*res); } } if (!value.IsMissing() && !is_substitution) { - return YamlConfig{std::move(value), config_vars_, mode_}; + return YamlConfig{std::move(value), config_vars, mode}; } - const auto env_name = yaml_[GetEnvName(key)]; - auto env_value = GetFromEnvImpl(env_name, mode_); - if (env_value) { - // Strip substitutions off to disallow nested substitutions - return YamlConfig{std::move(*env_value), {}, Mode::kSecure}; - } + return GetSharpCommandValue(yaml, mode, key, + /*met_substitution*/ is_substitution); +} - const auto file_name = yaml_[GetFileName(key)]; - auto file_value = GetFromFileImpl(file_name, mode_); - if (file_value) { - // Strip substitutions off to disallow nested substitutions - return YamlConfig{std::move(*file_value), {}, Mode::kSecure}; +} // namespace + +YamlConfig::YamlConfig(formats::yaml::Value yaml, + formats::yaml::Value config_vars, Mode mode) + : yaml_(std::move(yaml)), + config_vars_(std::move(config_vars)), + mode_(mode) {} + +const formats::yaml::Value& YamlConfig::Yaml() const { return yaml_; } + +YamlConfig YamlConfig::operator[](std::string_view key) const { + if (utils::text::EndsWith(key, "#env") || + utils::text::EndsWith(key, "#file") || + utils::text::EndsWith(key, "#fallback")) { + UASSERT_MSG(false, "Do not use names ending on #env, #file and #fallback"); + return MakeMissingConfig(*this, key); } - if (is_substitution || !env_name.IsMissing() || !file_name.IsMissing()) { - const auto fallback_name = GetFallbackName(key); - if (yaml_.HasMember(fallback_name)) { - LOG_INFO() << "using fallback value for '" << key << '\''; - // Strip substitutions off to disallow nested substitutions - return YamlConfig{yaml_[fallback_name], {}, Mode::kSecure}; - } + auto yaml_config = GetYamlConfig(yaml_, config_vars_, mode_, key); + if (yaml_config) { + return std::move(*yaml_config); } return MakeMissingConfig(*this, key); @@ -166,6 +191,12 @@ YamlConfig YamlConfig::operator[](size_t index) const { return YamlConfig{std::move(var_data), {}, Mode::kSecure}; } + auto res = GetSharpCommandValue(config_vars_, mode_, var_name, + /*met_substitution*/ false); + if (res) { + return std::move(*res); + } + // Avoid parsing $substitution as a string return MakeMissingConfig(*this, index); } diff --git a/universal/src/yaml_config/yaml_config_test.cpp b/universal/src/yaml_config/yaml_config_test.cpp index 88c63f1177f3..77615780b6f5 100644 --- a/universal/src/yaml_config/yaml_config_test.cpp +++ b/universal/src/yaml_config/yaml_config_test.cpp @@ -768,4 +768,56 @@ TEST(YamlConfig, ExplicitStringType) { << ToString(json); } +TEST(YamlConfig, SampleMultipleWithVarsEnv) { + const auto node = formats::yaml::FromString(R"( +# /// [sample multiple] +# yaml +some_element: + some: $variable + some#file: /some/path/to/the/file.yaml + some#env: SOME_ENV_VARIABLE + some#fallback: 100500 +# /// [sample multiple] + )"); + const auto vars = formats::yaml::FromString("variable#env: VARIABLE_ENV"); + + // NOLINTNEXTLINE(concurrency-mt-unsafe) + ::setenv("SOME_ENV_VARIABLE", "100", 1); + + // NOLINTNEXTLINE(concurrency-mt-unsafe) + ::setenv("VARIABLE_ENV", "42", 1); + + yaml_config::YamlConfig yaml(node, vars); + UEXPECT_THROW(yaml["some_element"]["some"].As(), std::exception); + + yaml = yaml_config::YamlConfig(node, vars, + yaml_config::YamlConfig::Mode::kEnvAllowed); + EXPECT_EQ(yaml["some_element"]["some"].As(), 42); + + yaml = yaml_config::YamlConfig( + node, vars, yaml_config::YamlConfig::Mode::kEnvAndFileAllowed); + EXPECT_EQ(yaml["some_element"]["some"].As(), 42); + + // NOLINTNEXTLINE(concurrency-mt-unsafe) + ::unsetenv("VARIABLE_ENV"); + + yaml = yaml_config::YamlConfig(node, {}, + yaml_config::YamlConfig::Mode::kEnvAllowed); + EXPECT_EQ(yaml["some_element"]["some"].As(), 100); + + yaml = yaml_config::YamlConfig(node, {}); + UEXPECT_THROW(yaml["some_element"]["some"].As(), std::exception); + + // NOLINTNEXTLINE(concurrency-mt-unsafe) + ::unsetenv("SOME_ENV_VARIABLE"); + + yaml = yaml_config::YamlConfig(node, {}, + yaml_config::YamlConfig::Mode::kEnvAllowed); + UEXPECT_THROW(yaml["some_element"]["some"].As(), std::exception); + + yaml = yaml_config::YamlConfig( + node, {}, yaml_config::YamlConfig::Mode::kEnvAndFileAllowed); + EXPECT_EQ(yaml["some_element"]["some"].As(), 100500); +} + USERVER_NAMESPACE_END From 151408313095697580acd07cbf68bc1e13516050 Mon Sep 17 00:00:00 2001 From: segoon Date: Tue, 23 Jul 2024 19:07:22 +0300 Subject: [PATCH 072/629] feat gh: do not build chaotic 978c86d0e3a2cc7ed259d0a76243a44f5daa8091 --- .github/workflows/docker.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index 9892557d79fd..84c338201198 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -15,6 +15,7 @@ jobs: matrix: include: - cmake-flags: >- + -DUSERVER_FEATURE_CHAOTIC=0 -DUSERVER_FEATURE_MYSQL=0 -DUSERVER_FEATURE_MONGODB=0 -DUSERVER_FEATURE_POSTGRESQL=0 From 0776ab0bce72c4578f9291a875c6c62c7932ceae Mon Sep 17 00:00:00 2001 From: nepritula Date: Wed, 24 Jul 2024 11:49:35 +0300 Subject: [PATCH 073/629] feat docs: snippets fixed in Generic Development, Testing and benchmarking and Runtime service features MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tests: протестировано локально и в CI f6d57a63f704a3bd83a0e230a0d2ee4d95e7ea6c --- core/src/logging/component.cpp | 2 ++ core/src/tracing/span_test.cpp | 2 ++ samples/mongo-support/CMakeLists.txt | 2 ++ scripts/docs/en/userver/functional_testing.md | 4 ++-- scripts/docs/en/userver/logging.md | 2 +- scripts/docs/en/userver/testing.md | 2 +- 6 files changed, 10 insertions(+), 4 deletions(-) diff --git a/core/src/logging/component.cpp b/core/src/logging/component.cpp index 9080223f388f..8b333b513eaa 100644 --- a/core/src/logging/component.cpp +++ b/core/src/logging/component.cpp @@ -133,7 +133,9 @@ Logging::~Logging() { Stop(); } void Logging::Stop() noexcept { /// [Signals sample - destr] + signal_subscriber_.Unsubscribe(); + /// [Signals sample - destr] flush_task_.Stop(); diff --git a/core/src/tracing/span_test.cpp b/core/src/tracing/span_test.cpp index 9d6b60b84c7d..20e281b9b89a 100644 --- a/core/src/tracing/span_test.cpp +++ b/core/src/tracing/span_test.cpp @@ -518,7 +518,9 @@ UTEST_F(Span, DocsData) { } { /// [Example get current span] + tracing::Span::CurrentSpan().AddTag("key", "value"); + /// [Example get current span] } } diff --git a/samples/mongo-support/CMakeLists.txt b/samples/mongo-support/CMakeLists.txt index 852680c2182c..3bfe11b04e24 100644 --- a/samples/mongo-support/CMakeLists.txt +++ b/samples/mongo-support/CMakeLists.txt @@ -8,5 +8,7 @@ add_executable(${PROJECT_NAME} ${SOURCES}) target_link_libraries(${PROJECT_NAME} userver::mongo) # /// [testsuite - cmake] + userver_testsuite_add_simple() + # /// [testsuite - cmake] diff --git a/scripts/docs/en/userver/functional_testing.md b/scripts/docs/en/userver/functional_testing.md index ed1db1c84c5e..8899b5fc4af6 100644 --- a/scripts/docs/en/userver/functional_testing.md +++ b/scripts/docs/en/userver/functional_testing.md @@ -235,7 +235,7 @@ collected functions and fixtures are applied. Example usage: -@snippet samples/grpc_service/tests/conftest.py Prepare configs +@snippet samples/grpc_service/testsuite/conftest.py Prepare configs #### Service client @@ -256,7 +256,7 @@ caches, mocked time, etc. Use @ref pytest_userver.plugins.service.service_env "service_env" fixture to provide extra environment variables for your service: -@snippet samples/redis_service/tests/conftest.py service_env +@snippet samples/redis_service/testsuite/conftest.py service_env #### Extra client dependencies diff --git a/scripts/docs/en/userver/logging.md b/scripts/docs/en/userver/logging.md index 367c03a63fda..6e4c52642a4c 100644 --- a/scripts/docs/en/userver/logging.md +++ b/scripts/docs/en/userver/logging.md @@ -182,7 +182,7 @@ All logs in the current task are implicitly linked to the current `Span` for the If you want to get the current `Span` (for example, you want to write something to `LogExtra`, but do not want to create an additional `Span`), then you can use the following approach: -@snippet tracing/span_test.cpp Example get current span +@snippet core/src/tracing/span_test.cpp Example get current span ### Creating a Span diff --git a/scripts/docs/en/userver/testing.md b/scripts/docs/en/userver/testing.md index f74bf87721ff..05ba74d76bad 100644 --- a/scripts/docs/en/userver/testing.md +++ b/scripts/docs/en/userver/testing.md @@ -88,7 +88,7 @@ equivalents with proper diagnostics are available in ` Date: Wed, 24 Jul 2024 12:42:08 +0300 Subject: [PATCH 074/629] feat chaotic: optimize string functions b60f6d601c2d34ac2df8f05e34d5a07c3cf1138a --- chaotic/chaotic/back/cpp/templates/type.cpp.jinja | 8 ++++---- .../chaotic/back/cpp/templates/type_parsers.ipp.jinja | 8 ++++---- chaotic/golden_tests/output/schemas/allof/allof.cpp | 9 ++++----- .../golden_tests/output/schemas/allof/allof_parsers.ipp | 6 +++--- chaotic/golden_tests/output/schemas/enum/enum.cpp | 6 +++--- .../golden_tests/output/schemas/enum/enum_parsers.ipp | 6 +++--- chaotic/golden_tests/output/schemas/int/int.cpp | 2 +- chaotic/golden_tests/output/schemas/int/int_parsers.ipp | 2 +- chaotic/golden_tests/output/schemas/oneof/oneof.cpp | 2 +- .../golden_tests/output/schemas/oneof/oneof_parsers.ipp | 2 +- chaotic/golden_tests/output/schemas/string/string.cpp | 3 +-- .../output/schemas/string/string_parsers.ipp | 2 +- 12 files changed, 27 insertions(+), 29 deletions(-) diff --git a/chaotic/chaotic/back/cpp/templates/type.cpp.jinja b/chaotic/chaotic/back/cpp/templates/type.cpp.jinja index dd06d7610a0a..754733f1b070 100644 --- a/chaotic/chaotic/back/cpp/templates/type.cpp.jinja +++ b/chaotic/chaotic/back/cpp/templates/type.cpp.jinja @@ -55,7 +55,7 @@ { const auto result = k{{ type.cpp_global_struct_field_name() }}_Mapping.TryFindBySecond(value); if (result.has_value()) { - return result.value(); + return *result; } throw std::runtime_error(fmt::format("Invalid enum value ({}) for type {{name}}", value)); } @@ -112,7 +112,7 @@ if (value.{{ field.cpp_field_name() }}) { vb["{{ fname }}"] = {{ field.schema.parser_type('', '') }}{ - value.{{ field.cpp_field_name() }}.value() + *value.{{ field.cpp_field_name() }} }; } {% else %} @@ -136,7 +136,7 @@ { const auto result = k{{ type.cpp_global_struct_field_name() }}_Mapping.TryFindByFirst(value); if (result.has_value()) { - return Serialize(result.value(), to); + return Serialize(*result, to); } {#- TODO: text #} throw std::runtime_error("Bad enum value"); @@ -187,7 +187,7 @@ std::string ToString({{ name }} value) { const auto result = k{{ type.cpp_global_struct_field_name() }}_Mapping.TryFindByFirst(value); if (result.has_value()) { - return std::string{result.value()}; + return std::string{*result}; } {#- TODO: text #} throw std::runtime_error("Bad enum value"); diff --git a/chaotic/chaotic/back/cpp/templates/type_parsers.ipp.jinja b/chaotic/chaotic/back/cpp/templates/type_parsers.ipp.jinja index cf39a2bd5f6d..f3884319594a 100644 --- a/chaotic/chaotic/back/cpp/templates/type_parsers.ipp.jinja +++ b/chaotic/chaotic/back/cpp/templates/type_parsers.ipp.jinja @@ -19,7 +19,7 @@ static constexpr {{ userver }}::utils::TrivialSet k{{type.cpp_global_struct_field_name()}}_PropertiesNames = [](auto selector) { - return selector().template Type() + return selector().template Type() {%- for fname in type.fields -%} .Case("{{ fname }}") {%- endfor -%} @@ -39,7 +39,7 @@ {% elif type.get_py_type() == 'CppStringEnum' %} static constexpr {{ userver }}::utils::TrivialBiMap k{{ type.cpp_global_struct_field_name() }}_Mapping = [](auto selector) { - return selector().template Type<{{ type.cpp_global_name() }}, std::string>() + return selector().template Type<{{ type.cpp_global_name() }}, std::string_view>() {%- for item in type.enums %} .Case({{ type.cpp_global_name() }}::{{ item.cpp_name }}, "{{ item.raw_name }}") {%- endfor -%} @@ -114,7 +114,7 @@ const auto value = val.template As(); const auto result = k{{ type.cpp_global_struct_field_name() }}_Mapping.TryFindBySecond(value); if (result.has_value()) { - return result.value(); + return *result; } {{ userver }}::chaotic::ThrowForValue(fmt::format("Invalid enum value ({}) for type {{name}}", value), val); } @@ -126,7 +126,7 @@ const auto value = val.template As(); const auto result = k{{ type.cpp_global_struct_field_name() }}_Mapping.TryFindBySecond(value); if (result.has_value()) { - return result.value(); + return *result; } {{ userver }}::chaotic::ThrowForValue(fmt::format("Invalid enum value ({}) for type {{name}}", value), val); } diff --git a/chaotic/golden_tests/output/schemas/allof/allof.cpp b/chaotic/golden_tests/output/schemas/allof/allof.cpp index e32117ac6d74..e7250a692663 100644 --- a/chaotic/golden_tests/output/schemas/allof/allof.cpp +++ b/chaotic/golden_tests/output/schemas/allof/allof.cpp @@ -98,8 +98,7 @@ USERVER_NAMESPACE::formats::json::Value Serialize( USERVER_NAMESPACE::formats::json::ValueBuilder vb = value.extra; if (value.foo) { - vb["foo"] = - USERVER_NAMESPACE::chaotic::Primitive{value.foo.value()}; + vb["foo"] = USERVER_NAMESPACE::chaotic::Primitive{*value.foo}; } return vb.ExtractValue(); @@ -112,7 +111,7 @@ USERVER_NAMESPACE::formats::json::Value Serialize( USERVER_NAMESPACE::formats::json::ValueBuilder vb = value.extra; if (value.bar) { - vb["bar"] = USERVER_NAMESPACE::chaotic::Primitive{value.bar.value()}; + vb["bar"] = USERVER_NAMESPACE::chaotic::Primitive{*value.bar}; } return vb.ExtractValue(); @@ -144,8 +143,8 @@ USERVER_NAMESPACE::formats::json::Value Serialize( USERVER_NAMESPACE::formats::common::Type::kObject; if (value.foo) { - vb["foo"] = USERVER_NAMESPACE::chaotic::Primitive{ - value.foo.value()}; + vb["foo"] = + USERVER_NAMESPACE::chaotic::Primitive{*value.foo}; } return vb.ExtractValue(); diff --git a/chaotic/golden_tests/output/schemas/allof/allof_parsers.ipp b/chaotic/golden_tests/output/schemas/allof/allof_parsers.ipp index bad4fb318097..d7d99f44d128 100644 --- a/chaotic/golden_tests/output/schemas/allof/allof_parsers.ipp +++ b/chaotic/golden_tests/output/schemas/allof/allof_parsers.ipp @@ -6,17 +6,17 @@ namespace ns { static constexpr USERVER_NAMESPACE::utils::TrivialSet kns__AllOf__Foo__P0_PropertiesNames = [](auto selector) { - return selector().template Type().Case("foo"); + return selector().template Type().Case("foo"); }; static constexpr USERVER_NAMESPACE::utils::TrivialSet kns__AllOf__Foo__P1_PropertiesNames = [](auto selector) { - return selector().template Type().Case("bar"); + return selector().template Type().Case("bar"); }; static constexpr USERVER_NAMESPACE::utils::TrivialSet kns__AllOf_PropertiesNames = [](auto selector) { - return selector().template Type().Case("foo"); + return selector().template Type().Case("foo"); }; template diff --git a/chaotic/golden_tests/output/schemas/enum/enum.cpp b/chaotic/golden_tests/output/schemas/enum/enum.cpp index b2e2e28d4b74..75cc88fc329d 100644 --- a/chaotic/golden_tests/output/schemas/enum/enum.cpp +++ b/chaotic/golden_tests/output/schemas/enum/enum.cpp @@ -62,7 +62,7 @@ ns::Enum::Foo FromString(std::string_view value, USERVER_NAMESPACE::formats::parse::To) { const auto result = kns__Enum__Foo_Mapping.TryFindBySecond(value); if (result.has_value()) { - return result.value(); + return *result; } throw std::runtime_error( fmt::format("Invalid enum value ({}) for type ns::Enum::Foo", value)); @@ -89,7 +89,7 @@ USERVER_NAMESPACE::formats::json::Value Serialize( if (value.foo) { vb["foo"] = - USERVER_NAMESPACE::chaotic::Primitive{value.foo.value()}; + USERVER_NAMESPACE::chaotic::Primitive{*value.foo}; } return vb.ExtractValue(); @@ -98,7 +98,7 @@ USERVER_NAMESPACE::formats::json::Value Serialize( std::string ToString(ns::Enum::Foo value) { const auto result = kns__Enum__Foo_Mapping.TryFindByFirst(value); if (result.has_value()) { - return std::string{result.value()}; + return std::string{*result}; } throw std::runtime_error("Bad enum value"); } diff --git a/chaotic/golden_tests/output/schemas/enum/enum_parsers.ipp b/chaotic/golden_tests/output/schemas/enum/enum_parsers.ipp index 3bb782dddb07..9bae2765fdde 100644 --- a/chaotic/golden_tests/output/schemas/enum/enum_parsers.ipp +++ b/chaotic/golden_tests/output/schemas/enum/enum_parsers.ipp @@ -7,7 +7,7 @@ namespace ns { static constexpr USERVER_NAMESPACE::utils::TrivialBiMap kns__Enum__Foo_Mapping = [](auto selector) { return selector() - .template Type() + .template Type() .Case(ns::Enum::Foo::kOne, "one") .Case(ns::Enum::Foo::kTwo, "two") .Case(ns::Enum::Foo::kThree, "three"); @@ -15,7 +15,7 @@ static constexpr USERVER_NAMESPACE::utils::TrivialBiMap kns__Enum__Foo_Mapping = static constexpr USERVER_NAMESPACE::utils::TrivialSet kns__Enum_PropertiesNames = [](auto selector) { - return selector().template Type().Case("foo"); + return selector().template Type().Case("foo"); }; template @@ -24,7 +24,7 @@ ns::Enum::Foo Parse(Value val, const auto value = val.template As(); const auto result = kns__Enum__Foo_Mapping.TryFindBySecond(value); if (result.has_value()) { - return result.value(); + return *result; } USERVER_NAMESPACE::chaotic::ThrowForValue( fmt::format("Invalid enum value ({}) for type ns::Enum::Foo", value), diff --git a/chaotic/golden_tests/output/schemas/int/int.cpp b/chaotic/golden_tests/output/schemas/int/int.cpp index 07df11fa3e48..4bb37d5f686d 100644 --- a/chaotic/golden_tests/output/schemas/int/int.cpp +++ b/chaotic/golden_tests/output/schemas/int/int.cpp @@ -45,7 +45,7 @@ USERVER_NAMESPACE::formats::json::Value Serialize( USERVER_NAMESPACE::formats::common::Type::kObject; if (value.foo) { - vb["foo"] = USERVER_NAMESPACE::chaotic::Primitive{value.foo.value()}; + vb["foo"] = USERVER_NAMESPACE::chaotic::Primitive{*value.foo}; } return vb.ExtractValue(); diff --git a/chaotic/golden_tests/output/schemas/int/int_parsers.ipp b/chaotic/golden_tests/output/schemas/int/int_parsers.ipp index a403794d6372..1013272558ce 100644 --- a/chaotic/golden_tests/output/schemas/int/int_parsers.ipp +++ b/chaotic/golden_tests/output/schemas/int/int_parsers.ipp @@ -6,7 +6,7 @@ namespace ns { static constexpr USERVER_NAMESPACE::utils::TrivialSet kns__Int_PropertiesNames = [](auto selector) { - return selector().template Type().Case("foo"); + return selector().template Type().Case("foo"); }; template diff --git a/chaotic/golden_tests/output/schemas/oneof/oneof.cpp b/chaotic/golden_tests/output/schemas/oneof/oneof.cpp index ce154609b7f8..62593ff2267b 100644 --- a/chaotic/golden_tests/output/schemas/oneof/oneof.cpp +++ b/chaotic/golden_tests/output/schemas/oneof/oneof.cpp @@ -49,7 +49,7 @@ USERVER_NAMESPACE::formats::json::Value Serialize( if (value.foo) { vb["foo"] = USERVER_NAMESPACE::chaotic::Variant< USERVER_NAMESPACE::chaotic::Primitive, - USERVER_NAMESPACE::chaotic::Primitive>{value.foo.value()}; + USERVER_NAMESPACE::chaotic::Primitive>{*value.foo}; } return vb.ExtractValue(); diff --git a/chaotic/golden_tests/output/schemas/oneof/oneof_parsers.ipp b/chaotic/golden_tests/output/schemas/oneof/oneof_parsers.ipp index 34445fbef0cb..6c5aeac58ce5 100644 --- a/chaotic/golden_tests/output/schemas/oneof/oneof_parsers.ipp +++ b/chaotic/golden_tests/output/schemas/oneof/oneof_parsers.ipp @@ -6,7 +6,7 @@ namespace ns { static constexpr USERVER_NAMESPACE::utils::TrivialSet kns__OneOf_PropertiesNames = [](auto selector) { - return selector().template Type().Case("foo"); + return selector().template Type().Case("foo"); }; template diff --git a/chaotic/golden_tests/output/schemas/string/string.cpp b/chaotic/golden_tests/output/schemas/string/string.cpp index f0b06cf452fa..05bd74936f82 100644 --- a/chaotic/golden_tests/output/schemas/string/string.cpp +++ b/chaotic/golden_tests/output/schemas/string/string.cpp @@ -45,8 +45,7 @@ USERVER_NAMESPACE::formats::json::Value Serialize( USERVER_NAMESPACE::formats::common::Type::kObject; if (value.foo) { - vb["foo"] = - USERVER_NAMESPACE::chaotic::Primitive{value.foo.value()}; + vb["foo"] = USERVER_NAMESPACE::chaotic::Primitive{*value.foo}; } return vb.ExtractValue(); diff --git a/chaotic/golden_tests/output/schemas/string/string_parsers.ipp b/chaotic/golden_tests/output/schemas/string/string_parsers.ipp index b1c1f1e6ac11..72d535e3fdb9 100644 --- a/chaotic/golden_tests/output/schemas/string/string_parsers.ipp +++ b/chaotic/golden_tests/output/schemas/string/string_parsers.ipp @@ -6,7 +6,7 @@ namespace ns { static constexpr USERVER_NAMESPACE::utils::TrivialSet kns__String_PropertiesNames = [](auto selector) { - return selector().template Type().Case("foo"); + return selector().template Type().Case("foo"); }; template From 974b00e30623ff121a629783e0f091c62ea28356 Mon Sep 17 00:00:00 2001 From: antonyzhilin Date: Wed, 24 Jul 2024 13:19:26 +0300 Subject: [PATCH 075/629] fix docs: fix snippet paths for functional tests d3183b9e1378fd5d8be964a2976203a577da16a9 --- .mapping.json | 4 ++-- .../server/handlers/http_handler_base.hpp | 7 +++--- samples/hello_service/CMakeLists.txt | 4 ++-- samples/hello_service/main.cpp | 14 +++++------ samples/hello_service/src/hello_handler.cpp | 13 ++++++++++ .../{hello_service.cpp => hello_handler.hpp} | 24 +++++-------------- samples/hello_service/src/hello_service.hpp | 9 ------- .../docs/en/userver/tutorial/hello_service.md | 19 +++++++-------- .../docs/en/userver/tutorial/mongo_service.md | 8 +++---- .../pytest_userver/plugins/service.py | 2 +- 10 files changed, 47 insertions(+), 57 deletions(-) create mode 100644 samples/hello_service/src/hello_handler.cpp rename samples/hello_service/src/{hello_service.cpp => hello_handler.hpp} (53%) delete mode 100644 samples/hello_service/src/hello_service.hpp diff --git a/.mapping.json b/.mapping.json index 6b5726cc5fd0..243101fe450a 100644 --- a/.mapping.json +++ b/.mapping.json @@ -2922,8 +2922,8 @@ "samples/hello_service/CMakeLists.txt":"taxi/uservices/userver/samples/hello_service/CMakeLists.txt", "samples/hello_service/benchmarks/say_hello_bench.cpp":"taxi/uservices/userver/samples/hello_service/benchmarks/say_hello_bench.cpp", "samples/hello_service/main.cpp":"taxi/uservices/userver/samples/hello_service/main.cpp", - "samples/hello_service/src/hello_service.cpp":"taxi/uservices/userver/samples/hello_service/src/hello_service.cpp", - "samples/hello_service/src/hello_service.hpp":"taxi/uservices/userver/samples/hello_service/src/hello_service.hpp", + "samples/hello_service/src/hello_handler.cpp":"taxi/uservices/userver/samples/hello_service/src/hello_handler.cpp", + "samples/hello_service/src/hello_handler.hpp":"taxi/uservices/userver/samples/hello_service/src/hello_handler.hpp", "samples/hello_service/src/say_hello.cpp":"taxi/uservices/userver/samples/hello_service/src/say_hello.cpp", "samples/hello_service/src/say_hello.hpp":"taxi/uservices/userver/samples/hello_service/src/say_hello.hpp", "samples/hello_service/static_config.yaml":"taxi/uservices/userver/samples/hello_service/static_config.yaml", diff --git a/core/include/userver/server/handlers/http_handler_base.hpp b/core/include/userver/server/handlers/http_handler_base.hpp index c9b75e757d65..890c81d33325 100644 --- a/core/include/userver/server/handlers/http_handler_base.hpp +++ b/core/include/userver/server/handlers/http_handler_base.hpp @@ -41,7 +41,7 @@ class HttpHandlerMethodStatistics; class HttpHandlerStatisticsScope; // clang-format off -/// + /// @ingroup userver_components userver_http_handlers userver_base_classes /// /// @brief Base class for all the @@ -59,8 +59,9 @@ class HttpHandlerStatisticsScope; /// /// ## Example usage: /// -/// @snippet samples/hello_service/src/hello_service.cpp Hello service sample - component -/// +/// @include samples/hello_service/src/hello_handler.hpp +/// @include samples/hello_service/src/hello_handler.cpp + // clang-format on class HttpHandlerBase : public HandlerBase { diff --git a/samples/hello_service/CMakeLists.txt b/samples/hello_service/CMakeLists.txt index d5617a1a8b15..1eedf97ec9d9 100644 --- a/samples/hello_service/CMakeLists.txt +++ b/samples/hello_service/CMakeLists.txt @@ -11,8 +11,8 @@ find_package(userver COMPONENTS core REQUIRED) add_library(${PROJECT_NAME}_objs OBJECT src/say_hello.hpp src/say_hello.cpp - src/hello_service.hpp - src/hello_service.cpp + src/hello_handler.hpp + src/hello_handler.cpp ) target_link_libraries(${PROJECT_NAME}_objs userver::core) target_include_directories(${PROJECT_NAME}_objs PUBLIC src) diff --git a/samples/hello_service/main.cpp b/samples/hello_service/main.cpp index 8be077c7e503..02427fced240 100644 --- a/samples/hello_service/main.cpp +++ b/samples/hello_service/main.cpp @@ -1,15 +1,15 @@ -#include - -/// [Hello service sample - main] #include +#include #include -#include "hello_service.hpp" +// Note: this is for the purposes of tests/samples only +#include + +#include int main(int argc, char* argv[]) { - auto component_list = components::MinimalServerComponentList(); - samples::hello::AppendHello(component_list); + auto component_list = components::MinimalServerComponentList() + .Append(); return utils::DaemonMain(argc, argv, component_list); } -/// [Hello service sample - main] diff --git a/samples/hello_service/src/hello_handler.cpp b/samples/hello_service/src/hello_handler.cpp new file mode 100644 index 000000000000..52dd406392c2 --- /dev/null +++ b/samples/hello_service/src/hello_handler.cpp @@ -0,0 +1,13 @@ +#include "hello_handler.hpp" + +#include "say_hello.hpp" + +namespace samples::hello { + +std::string HelloHandler::HandleRequestThrow( + const server::http::HttpRequest& request, + server::request::RequestContext& /*request_context*/) const { + return SayHelloTo(request.GetArg("name")); +} + +} // namespace samples::hello diff --git a/samples/hello_service/src/hello_service.cpp b/samples/hello_service/src/hello_handler.hpp similarity index 53% rename from samples/hello_service/src/hello_service.cpp rename to samples/hello_service/src/hello_handler.hpp index a3b9b318493e..15e879ecf6af 100644 --- a/samples/hello_service/src/hello_service.cpp +++ b/samples/hello_service/src/hello_handler.hpp @@ -1,17 +1,14 @@ -#include - -/// [Hello service sample - component] -#include "hello_service.hpp" +#pragma once +#include #include -#include "say_hello.hpp" +// Note: this is for the purposes of tests/samples only +#include namespace samples::hello { -namespace { - -class Hello final : public server::handlers::HttpHandlerBase { +class HelloHandler final : public server::handlers::HttpHandlerBase { public: // `kName` is used as the component name in static config static constexpr std::string_view kName = "handler-hello-sample"; @@ -21,16 +18,7 @@ class Hello final : public server::handlers::HttpHandlerBase { std::string HandleRequestThrow( const server::http::HttpRequest& request, - server::request::RequestContext&) const override { - return samples::hello::SayHelloTo(request.GetArg("name")); - } + server::request::RequestContext&) const override; }; -} // namespace - -void AppendHello(components::ComponentList& component_list) { - component_list.Append(); -} - } // namespace samples::hello -/// [Hello service sample - component] diff --git a/samples/hello_service/src/hello_service.hpp b/samples/hello_service/src/hello_service.hpp deleted file mode 100644 index d255b0de54b1..000000000000 --- a/samples/hello_service/src/hello_service.hpp +++ /dev/null @@ -1,9 +0,0 @@ -#pragma once - -#include - -namespace samples::hello { - -void AppendHello(components::ComponentList& component_list); - -} // namespace samples::hello diff --git a/scripts/docs/en/userver/tutorial/hello_service.md b/scripts/docs/en/userver/tutorial/hello_service.md index 3a1028395b37..594c4c2cecae 100644 --- a/scripts/docs/en/userver/tutorial/hello_service.md +++ b/scripts/docs/en/userver/tutorial/hello_service.md @@ -53,14 +53,11 @@ would be used for HTTP response body. Otherwise if an exception `exc` derived fr `std::exception` is thrown from the function then the HTTP response code will be set to `500`. -@snippet samples/hello_service/src/hello_service.cpp Hello service sample - component +@include samples/hello_service/src/hello_handler.hpp -@warning `Handle*` functions are invoked concurrently on the same instance of the handler class. Use @ref scripts/docs/en/userver/synchronization.md "synchronization primitives" or do not modify shared data in `Handle*`. - -Note that the component in an anonymous namespace and the only way to add it -to the component lists in via calling `AppendHello` from "hello_service.hpp": +@include samples/hello_service/src/hello_handler.cpp -@snippet samples/hello_service/main.cpp Hello service sample - main +@warning `Handle*` functions are invoked concurrently on the same instance of the handler class. Use @ref scripts/docs/en/userver/synchronization.md "synchronization primitives" or do not modify shared data in `Handle*`. ### Static config @@ -82,7 +79,7 @@ Finally, we add our component to the `components::MinimalServerComponentList()`, and start the server with static configuration file passed from command line. -@snippet samples/hello_service/main.cpp Hello service sample - main +@include samples/hello_service/main.cpp ### CMake @@ -154,8 +151,8 @@ Do not forget to add the plugin in conftest.py: ## Full sources See the full example at: -* @ref samples/hello_service/src/hello_service.hpp -* @ref samples/hello_service/src/hello_service.cpp +* @ref samples/hello_service/src/hello_handler.hpp +* @ref samples/hello_service/src/hello_handler.cpp * @ref samples/hello_service/src/say_hello.hpp * @ref samples/hello_service/src/say_hello.cpp * @ref samples/hello_service/static_config.yaml @@ -173,8 +170,8 @@ See the full example at: @htmlonly
@endhtmlonly -@example samples/hello_service/src/hello_service.hpp -@example samples/hello_service/src/hello_service.cpp +@example samples/hello_service/src/hello_handler.hpp +@example samples/hello_service/src/hello_handler.cpp @example samples/hello_service/src/say_hello.hpp @example samples/hello_service/src/say_hello.cpp @example samples/hello_service/static_config.yaml diff --git a/scripts/docs/en/userver/tutorial/mongo_service.md b/scripts/docs/en/userver/tutorial/mongo_service.md index d347756998ef..2205fc598088 100644 --- a/scripts/docs/en/userver/tutorial/mongo_service.md +++ b/scripts/docs/en/userver/tutorial/mongo_service.md @@ -140,8 +140,8 @@ See the full example: * @ref samples/mongo_service/mongo_service.cpp * @ref samples/mongo_service/static_config.yaml * @ref samples/mongo_service/CMakeLists.txt -* @ref samples/mongo_service/tests/conftest.py -* @ref samples/mongo_service/tests/test_mongo.py +* @ref samples/mongo_service/testsuite/conftest.py +* @ref samples/mongo_service/testsuite/test_mongo.py ---------- @@ -153,5 +153,5 @@ See the full example: @example samples/mongo_service/mongo_service.cpp @example samples/mongo_service/static_config.yaml @example samples/mongo_service/CMakeLists.txt -@example samples/mongo_service/tests/conftest.py -@example samples/mongo_service/tests/test_mongo.py +@example samples/mongo_service/testsuite/conftest.py +@example samples/mongo_service/testsuite/test_mongo.py diff --git a/testsuite/pytest_plugins/pytest_userver/plugins/service.py b/testsuite/pytest_plugins/pytest_userver/plugins/service.py index 981a6332e54f..4ffe787c4c4a 100644 --- a/testsuite/pytest_plugins/pytest_userver/plugins/service.py +++ b/testsuite/pytest_plugins/pytest_userver/plugins/service.py @@ -55,7 +55,7 @@ def service_env(): """ Override this to pass extra environment variables to the service. - @snippet samples/redis_service/tests/conftest.py service_env + @snippet samples/redis_service/testsuite/conftest.py service_env @ingroup userver_testsuite_fixtures """ return None From 436a47151c16bd173fc6dbef07a69981241e7899 Mon Sep 17 00:00:00 2001 From: Turim Date: Wed, 24 Jul 2024 16:43:13 +0300 Subject: [PATCH 076/629] bug core: websocket HTTP connection headers MUST be case insensitive, key MUST have len16 bytes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes: https://github.com/userver-framework/userver/issues/655 Tests: протестировано CI 76322f5d8822842c6092338a7caca5d874a6313a Pull Request resolved: https://github.com/userver-framework/userver/pull/656 --- .../src/server/websocket/websocket_handler.cpp | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/core/src/server/websocket/websocket_handler.cpp b/core/src/server/websocket/websocket_handler.cpp index c538bf2ec07d..1f53398070e6 100644 --- a/core/src/server/websocket/websocket_handler.cpp +++ b/core/src/server/websocket/websocket_handler.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -33,17 +34,22 @@ std::string WebsocketHandlerBase::HandleRequestThrow( const server::http::HttpRequest& request, server::request::RequestContext& context) const { if (request.GetMethod() != server::http::HttpMethod::kGet || - request.GetHeader(USERVER_NAMESPACE::http::headers::kUpgrade) != - "websocket" || - request.GetHeader(USERVER_NAMESPACE::http::headers::kConnection) != - "Upgrade") { + !utils::StrIcaseEqual()( + request.GetHeader(USERVER_NAMESPACE::http::headers::kUpgrade), + std::string_view("websocket")) || + !utils::StrIcaseEqual()( + request.GetHeader(USERVER_NAMESPACE::http::headers::kConnection), + std::string_view("upgrade"))) { HandleNonWebsocketRequest(request, context); } const std::string& secWebsocketKey = request.GetHeader(USERVER_NAMESPACE::http::headers::kWebsocketKey); - if (secWebsocketKey.empty()) { - LOG_WARNING() << "Empty or missing Websocket Key"; + + // We are fine if `secWebsocketKey` is not properly base64-ecoded + static constexpr std::size_t kLengthOfBase64Encoded16Bytes = 24; + if (kLengthOfBase64Encoded16Bytes != secWebsocketKey.size()) { + LOG_WARNING() << "Empty or invalid Websocket Key"; throw server::handlers::ClientError(); } From 5770e5494aade9c09e420fdb45a189c92e68eff1 Mon Sep 17 00:00:00 2001 From: antonyzhilin Date: Wed, 24 Jul 2024 17:31:21 +0300 Subject: [PATCH 077/629] fix grpc: write body in case of unary response fail 70af6ef9bec61d654abc15f8feb97a8e4ad50082 --- grpc/src/ugrpc/impl/span.cpp | 2 +- .../server/middlewares/log/middleware.cpp | 29 +++++++++++++------ grpc/tests/generic_server_test.cpp | 2 +- 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/grpc/src/ugrpc/impl/span.cpp b/grpc/src/ugrpc/impl/span.cpp index 003d80add171..9cb589553ce4 100644 --- a/grpc/src/ugrpc/impl/span.cpp +++ b/grpc/src/ugrpc/impl/span.cpp @@ -10,7 +10,7 @@ USERVER_NAMESPACE_BEGIN namespace ugrpc::impl { void UpdateSpanWithStatus(tracing::Span& span, const grpc::Status& status) { - static const std::string kGrpcStatusTag{"grpc_status"}; + static const std::string kGrpcStatusTag{"grpc_code"}; span.AddTag(kGrpcStatusTag, std::string{ToString(status.error_code())}); if (!status.ok()) { span.AddTag(tracing::kErrorFlag, true); diff --git a/grpc/src/ugrpc/server/middlewares/log/middleware.cpp b/grpc/src/ugrpc/server/middlewares/log/middleware.cpp index 6989a574cd92..0a765e3f253e 100644 --- a/grpc/src/ugrpc/server/middlewares/log/middleware.cpp +++ b/grpc/src/ugrpc/server/middlewares/log/middleware.cpp @@ -13,6 +13,16 @@ namespace ugrpc::server::middlewares::log { namespace { +bool IsRequestStream(CallKind kind) { + return kind == CallKind::kRequestStream || + kind == CallKind::kBidirectionalStream; +} + +bool IsResponseStream(CallKind kind) { + return kind == CallKind::kResponseStream || + kind == CallKind::kBidirectionalStream; +} + std::string ProtobufAsJsonString(const google::protobuf::Message& message, uint64_t max_msg_size) { google::protobuf::util::JsonPrintOptions options; @@ -54,8 +64,7 @@ void Middleware::CallRequestHook(const MiddlewareCallContext& context, storage.Set(kIsFirstRequest, false); const auto call_kind = context.GetCall().GetCallKind(); - if (call_kind == CallKind::kUnaryCall || - call_kind == CallKind::kResponseStream) { + if (!IsRequestStream(call_kind)) { log_extra.Extend("type", "request"); } } @@ -67,8 +76,7 @@ void Middleware::CallResponseHook(const MiddlewareCallContext& context, auto& span = context.GetCall().GetSpan(); const auto call_kind = context.GetCall().GetCallKind(); - if (call_kind == CallKind::kUnaryCall || - call_kind == CallKind::kRequestStream) { + if (!IsResponseStream(call_kind)) { span.AddTag("grpc_type", "response"); span.AddNonInheritableTag("body", GetMessageForLogging(response, settings_)); @@ -92,17 +100,20 @@ void Middleware::Handle(MiddlewareCallContext& context) const { span.AddTag("meta_type", std::string{context.GetCall().GetCallName()}); span.AddNonInheritableTag("type", "response"); - if (call_kind == CallKind::kResponseStream || - call_kind == CallKind::kBidirectionalStream) { + if (IsResponseStream(call_kind)) { // Just like in HTTP, there must be a single trailing Span log - // with type=response and some body. We don't have a real single response + // with type=response and some `body`. We don't have a real single response // (responses are written separately, 1 log per response), so we fake // the required response log. span.AddNonInheritableTag("body", "response stream finished"); + } else { + // Write this dummy `body` in case unary response RPC fails + // (with our without status) before receiving the response. + // If the RPC finishes with OK status, `body` tag will be overwritten. + span.AddNonInheritableTag("body", "error status"); } - if (call_kind == CallKind::kRequestStream || - call_kind == CallKind::kBidirectionalStream) { + if (IsRequestStream(call_kind)) { // Just like in HTTP, there must be a single initial log // with type=request and some body. We don't have a real single request // (requests are written separately, 1 log per request), so we fake diff --git a/grpc/tests/generic_server_test.cpp b/grpc/tests/generic_server_test.cpp index e12bd4fc2fdf..9ffd69db47c1 100644 --- a/grpc/tests/generic_server_test.cpp +++ b/grpc/tests/generic_server_test.cpp @@ -108,7 +108,7 @@ UTEST_F(GenericServerLoggingTest, Logs) { EXPECT_EQ(span_log.GetTagOptional("stopwatch_name"), "grpc/sample.ugrpc.UnitTestService/SayHello") << span_log; - EXPECT_EQ(span_log.GetTagOptional("grpc_status"), "OK") << span_log; + EXPECT_EQ(span_log.GetTagOptional("grpc_code"), "OK") << span_log; } USERVER_NAMESPACE_END From 6304a833f36368159ab85ec9775ee11efa054932 Mon Sep 17 00:00:00 2001 From: Egor Paroshin Date: Wed, 24 Jul 2024 17:40:44 +0300 Subject: [PATCH 078/629] feat core: add minItems and maxItems array properties MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes: https://github.com/userver-framework/userver/issues/574 Tests: протестировано CI Pull Request resolved: #654 920219cb97938348a128b169cb8dc1ac5e77348a --- .../include/userver/yaml_config/schema.hpp | 2 + .../impl/validate_static_config.cpp | 20 +++++ .../impl/validate_static_config_test.cpp | 76 ++++++++++++++++++- universal/src/yaml_config/schema.cpp | 13 +++- 4 files changed, 109 insertions(+), 2 deletions(-) diff --git a/universal/include/userver/yaml_config/schema.hpp b/universal/include/userver/yaml_config/schema.hpp index fb0bb2470876..d05760d64e87 100644 --- a/universal/include/userver/yaml_config/schema.hpp +++ b/universal/include/userver/yaml_config/schema.hpp @@ -68,6 +68,8 @@ struct Schema final { std::optional> enum_values; std::optional minimum; std::optional maximum; + std::optional min_items; + std::optional max_items; }; Schema Parse(const formats::yaml::Value& schema, formats::parse::To); diff --git a/universal/src/yaml_config/impl/validate_static_config.cpp b/universal/src/yaml_config/impl/validate_static_config.cpp index d277c61e0d63..9039b57061ca 100644 --- a/universal/src/yaml_config/impl/validate_static_config.cpp +++ b/universal/src/yaml_config/impl/validate_static_config.cpp @@ -100,7 +100,27 @@ void ValidateObject(const YamlConfig& object, const Schema& schema) { } } +void ValidateArrayLen(const YamlConfig& array, const Schema& schema) { + if (schema.min_items && !(array.GetSize() >= *schema.min_items)) { + throw std::runtime_error(fmt::format( + "Error while validating static config against schema. " + "Expected length of {} at path '{}' to be >= {} (actual: {}).", + ToString(schema.type), array.GetPath(), *schema.min_items, + array.GetSize())); + } + + if (schema.max_items && !(array.GetSize() <= *schema.max_items)) { + throw std::runtime_error(fmt::format( + "Error while validating static config against schema. " + "Expected length of {} at path '{}' to be <= {} (actual: {}).", + ToString(schema.type), array.GetPath(), *schema.max_items, + array.GetSize())); + } +} + void ValidateArray(const YamlConfig& array, const Schema& schema) { + ValidateArrayLen(array, schema); + for (const auto& element : array) { ValidateIfPresent(element, *schema.items.value()); } diff --git a/universal/src/yaml_config/impl/validate_static_config_test.cpp b/universal/src/yaml_config/impl/validate_static_config_test.cpp index 6fdd819d37fe..f14015dd9abf 100644 --- a/universal/src/yaml_config/impl/validate_static_config_test.cpp +++ b/universal/src/yaml_config/impl/validate_static_config_test.cpp @@ -30,7 +30,8 @@ description: with incorrect field name std::runtime_error, "Schema field name must be one of ['type', 'description', " "'defaultDescription', 'additionalProperties', 'properties', 'items', " - "'enum', 'minimum', 'maximum'], but 'incorrect_filed_name' was given. " + "'enum', 'minimum', 'maximum', 'minItems', 'maxItems'], but " + "'incorrect_filed_name' was given. " "Schema path: '/'"); } @@ -267,6 +268,79 @@ additionalProperties: false "'properties.arr.items'"); } +TEST(StaticConfigValidator, MinArrayLenFailed) { + const std::string kStaticConfig = R"( +arr: [1] +)"; + + const std::string kSchema = R"( +type: object +description: simple array +additionalProperties: false +properties: + arr: + type: array + description: integer array + minItems: 2 + items: + type: integer + description: element of array +)"; + + UEXPECT_THROW_MSG( + Validate(kStaticConfig, kSchema), std::runtime_error, + "Error while validating static config against schema. " + "Expected length of array at path 'arr' to be >= 2 (actual: 1)."); +} + +TEST(StaticConfigValidator, MaxArrayLenFailed) { + const std::string kStaticConfig = R"( +arr: [1, 2, 3] +)"; + + const std::string kSchema = R"( +type: object +description: simple array +additionalProperties: false +properties: + arr: + type: array + description: integer array + maxItems: 2 + items: + type: integer + description: element of array +)"; + + UEXPECT_THROW_MSG( + Validate(kStaticConfig, kSchema), std::runtime_error, + "Error while validating static config against schema. " + "Expected length of array at path 'arr' to be <= 2 (actual: 3)."); +} + +TEST(StaticConfigValidator, ArrayLenPassed) { + const std::string kStaticConfig = R"( +arr: [1, 2, 3] +)"; + + const std::string kSchema = R"( +type: object +description: simple array +additionalProperties: false +properties: + arr: + type: array + description: integer array + minItems: 1 + maxItems: 3 + items: + type: integer + description: element of array +)"; + + UEXPECT_NO_THROW(Validate(kStaticConfig, kSchema)); +} + TEST(StaticConfigValidator, Recursive) { const std::string kStaticConfig = R"( huge-object: diff --git a/universal/src/yaml_config/schema.cpp b/universal/src/yaml_config/schema.cpp index 8c9382b32ab2..02014dd9aa71 100644 --- a/universal/src/yaml_config/schema.cpp +++ b/universal/src/yaml_config/schema.cpp @@ -27,7 +27,9 @@ void CheckFieldsNames(const formats::yaml::Value& yaml_schema) { .Case("items") .Case("enum") .Case("minimum") - .Case("maximum"); + .Case("maximum") + .Case("minItems") + .Case("maxItems"); }; for (const auto& [name, value] : Items(yaml_schema)) { @@ -71,6 +73,10 @@ void CheckSchemaStructure(const Schema& schema) { {FieldType::kInteger, FieldType::kNumber}); CheckTypeSupportsField(schema, "maximum", schema.maximum, {FieldType::kInteger, FieldType::kNumber}); + CheckTypeSupportsField(schema, "minItems", schema.min_items, + {FieldType::kArray}); + CheckTypeSupportsField(schema, "maxItems", schema.max_items, + {FieldType::kArray}); if (schema.type == FieldType::kObject) { if (!schema.properties.has_value()) { @@ -169,6 +175,9 @@ Schema Parse(const formats::yaml::Value& schema, formats::parse::To) { result.minimum = schema["minimum"].As>(); result.maximum = schema["maximum"].As>(); + result.min_items = schema["minItems"].As>(); + result.max_items = schema["maxItems"].As>(); + CheckFieldsNames(schema); CheckSchemaStructure(result); @@ -195,6 +204,8 @@ formats::yaml::Value Serialize(const Schema& schema, if (schema.enum_values) builder["enum"] = *schema.enum_values; if (schema.minimum) builder["minimum"] = *schema.minimum; if (schema.maximum) builder["maximum"] = *schema.maximum; + if (schema.min_items) builder["minItems"] = *schema.min_items; + if (schema.max_items) builder["maxItems"] = *schema.max_items; return builder.ExtractValue(); } From a3ed9a2f05d1acd49cb95c845fb21eb5a9861612 Mon Sep 17 00:00:00 2001 From: antonyzhilin Date: Wed, 24 Jul 2024 21:43:06 +0300 Subject: [PATCH 079/629] feat grpc: add docs on using generic API 6166edfddf1f6d57af106c9e110b301d53ded802 --- .mapping.json | 12 +- grpc/include/userver/ugrpc/client/fwd.hpp | 1 + grpc/include/userver/ugrpc/client/generic.hpp | 2 + .../ugrpc/server/generic_service_base.hpp | 2 + .../ugrpc/server/service_component_base.hpp | 7 +- .../ugrpc/server/service_component_base.cpp | 5 + samples/CMakeLists.txt | 3 + samples/grpc-generic-proxy/CMakeLists.txt | 20 ++++ samples/grpc-generic-proxy/config_vars.yaml | 7 ++ samples/grpc-generic-proxy/main.cpp | 49 ++++++++ .../proto/samples/greeter.proto | 15 +++ .../grpc-generic-proxy/src/proxy_service.cpp | 108 ++++++++++++++++++ .../grpc-generic-proxy/src/proxy_service.hpp | 25 ++++ samples/grpc-generic-proxy/static_config.yaml | 80 +++++++++++++ samples/grpc-generic-proxy/tests/conftest.py | 24 ++++ .../grpc-generic-proxy/tests/test_proxy.py | 96 ++++++++++++++++ samples/grpc_middleware_service/proto/ya.make | 35 ------ samples/grpc_middleware_service/tests/ya.make | 34 ------ samples/grpc_middleware_service/ya.make | 36 ------ scripts/docs/en/userver/grpc.md | 28 +++++ .../pytest_userver/plugins/grpc/mockserver.py | 6 +- 21 files changed, 482 insertions(+), 113 deletions(-) create mode 100644 samples/grpc-generic-proxy/CMakeLists.txt create mode 100644 samples/grpc-generic-proxy/config_vars.yaml create mode 100644 samples/grpc-generic-proxy/main.cpp create mode 100644 samples/grpc-generic-proxy/proto/samples/greeter.proto create mode 100644 samples/grpc-generic-proxy/src/proxy_service.cpp create mode 100644 samples/grpc-generic-proxy/src/proxy_service.hpp create mode 100644 samples/grpc-generic-proxy/static_config.yaml create mode 100644 samples/grpc-generic-proxy/tests/conftest.py create mode 100644 samples/grpc-generic-proxy/tests/test_proxy.py delete mode 100644 samples/grpc_middleware_service/proto/ya.make delete mode 100644 samples/grpc_middleware_service/tests/ya.make delete mode 100644 samples/grpc_middleware_service/ya.make diff --git a/.mapping.json b/.mapping.json index 243101fe450a..88738c728d63 100644 --- a/.mapping.json +++ b/.mapping.json @@ -2882,9 +2882,17 @@ "samples/flatbuf_service/static_config.yaml":"taxi/uservices/userver/samples/flatbuf_service/static_config.yaml", "samples/flatbuf_service/tests/conftest.py":"taxi/uservices/userver/samples/flatbuf_service/tests/conftest.py", "samples/flatbuf_service/tests/test_flatbuf.py":"taxi/uservices/userver/samples/flatbuf_service/tests/test_flatbuf.py", + "samples/grpc-generic-proxy/CMakeLists.txt":"taxi/uservices/userver/samples/grpc-generic-proxy/CMakeLists.txt", + "samples/grpc-generic-proxy/config_vars.yaml":"taxi/uservices/userver/samples/grpc-generic-proxy/config_vars.yaml", + "samples/grpc-generic-proxy/main.cpp":"taxi/uservices/userver/samples/grpc-generic-proxy/main.cpp", + "samples/grpc-generic-proxy/proto/samples/greeter.proto":"taxi/uservices/userver/samples/grpc-generic-proxy/proto/samples/greeter.proto", + "samples/grpc-generic-proxy/src/proxy_service.cpp":"taxi/uservices/userver/samples/grpc-generic-proxy/src/proxy_service.cpp", + "samples/grpc-generic-proxy/src/proxy_service.hpp":"taxi/uservices/userver/samples/grpc-generic-proxy/src/proxy_service.hpp", + "samples/grpc-generic-proxy/static_config.yaml":"taxi/uservices/userver/samples/grpc-generic-proxy/static_config.yaml", + "samples/grpc-generic-proxy/tests/conftest.py":"taxi/uservices/userver/samples/grpc-generic-proxy/tests/conftest.py", + "samples/grpc-generic-proxy/tests/test_proxy.py":"taxi/uservices/userver/samples/grpc-generic-proxy/tests/test_proxy.py", "samples/grpc_middleware_service/CMakeLists.txt":"taxi/uservices/userver/samples/grpc_middleware_service/CMakeLists.txt", "samples/grpc_middleware_service/proto/samples/greeter.proto":"taxi/uservices/userver/samples/grpc_middleware_service/proto/samples/greeter.proto", - "samples/grpc_middleware_service/proto/ya.make":"taxi/uservices/userver/samples/grpc_middleware_service/proto/ya.make", "samples/grpc_middleware_service/src/client/view.cpp":"taxi/uservices/userver/samples/grpc_middleware_service/src/client/view.cpp", "samples/grpc_middleware_service/src/client/view.hpp":"taxi/uservices/userver/samples/grpc_middleware_service/src/client/view.hpp", "samples/grpc_middleware_service/src/http_handlers/say-hello/view.cpp":"taxi/uservices/userver/samples/grpc_middleware_service/src/http_handlers/say-hello/view.cpp", @@ -2905,8 +2913,6 @@ "samples/grpc_middleware_service/static_config.yaml":"taxi/uservices/userver/samples/grpc_middleware_service/static_config.yaml", "samples/grpc_middleware_service/tests/conftest.py":"taxi/uservices/userver/samples/grpc_middleware_service/tests/conftest.py", "samples/grpc_middleware_service/tests/test_middlewares.py":"taxi/uservices/userver/samples/grpc_middleware_service/tests/test_middlewares.py", - "samples/grpc_middleware_service/tests/ya.make":"taxi/uservices/userver/samples/grpc_middleware_service/tests/ya.make", - "samples/grpc_middleware_service/ya.make":"taxi/uservices/userver/samples/grpc_middleware_service/ya.make", "samples/grpc_service/CMakeLists.txt":"taxi/uservices/userver/samples/grpc_service/CMakeLists.txt", "samples/grpc_service/main.cpp":"taxi/uservices/userver/samples/grpc_service/main.cpp", "samples/grpc_service/proto/samples/greeter.proto":"taxi/uservices/userver/samples/grpc_service/proto/samples/greeter.proto", diff --git a/grpc/include/userver/ugrpc/client/fwd.hpp b/grpc/include/userver/ugrpc/client/fwd.hpp index 6cfca840f21c..01c04772665e 100644 --- a/grpc/include/userver/ugrpc/client/fwd.hpp +++ b/grpc/include/userver/ugrpc/client/fwd.hpp @@ -12,6 +12,7 @@ class GrpcControl; namespace ugrpc::client { class ClientFactory; +class GenericClient; } // namespace ugrpc::client diff --git a/grpc/include/userver/ugrpc/client/generic.hpp b/grpc/include/userver/ugrpc/client/generic.hpp index bfd5d819af37..6f64b88e561d 100644 --- a/grpc/include/userver/ugrpc/client/generic.hpp +++ b/grpc/include/userver/ugrpc/client/generic.hpp @@ -59,6 +59,8 @@ struct GenericOptions { /// ## Example GenericClient usage with known message types /// /// @snippet grpc/tests/generic_client_test.cpp sample +/// +/// For a more complete sample, see @ref grpc_generic_api. class GenericClient final { public: GenericClient(GenericClient&&) noexcept = default; diff --git a/grpc/include/userver/ugrpc/server/generic_service_base.hpp b/grpc/include/userver/ugrpc/server/generic_service_base.hpp index e0b3b172e506..9783da837ed1 100644 --- a/grpc/include/userver/ugrpc/server/generic_service_base.hpp +++ b/grpc/include/userver/ugrpc/server/generic_service_base.hpp @@ -43,6 +43,8 @@ namespace ugrpc::server { /// ## Example GenericServiceBase usage with known message types /// /// @snippet grpc/tests/generic_server_test.cpp sample +/// +/// For a more complete sample, see @ref grpc_generic_api. class GenericServiceBase { public: /// Inherits from both @ref GenericServiceBase and @ref ServiceComponentBase. diff --git a/grpc/include/userver/ugrpc/server/service_component_base.hpp b/grpc/include/userver/ugrpc/server/service_component_base.hpp index ab581ee27f55..9882c5ee3bc1 100644 --- a/grpc/include/userver/ugrpc/server/service_component_base.hpp +++ b/grpc/include/userver/ugrpc/server/service_component_base.hpp @@ -16,6 +16,7 @@ USERVER_NAMESPACE_BEGIN namespace ugrpc::server { class ServerComponent; +class GenericServiceBase; // clang-format off @@ -43,6 +44,9 @@ class ServiceComponentBase : public components::ComponentBase { /// RegisterService with it void RegisterService(ServiceBase& service); + /// @overload + void RegisterService(GenericServiceBase& service); + private: ServerComponent& server_; ServiceConfig config_; @@ -55,7 +59,8 @@ template // NOLINTNEXTLINE(fuchsia-multiple-inheritance) class ServiceComponentBase : public server::ServiceComponentBase, public ServiceInterface { - static_assert(std::is_base_of_v); + static_assert(std::is_base_of_v || + std::is_base_of_v); public: ServiceComponentBase(const components::ComponentConfig& config, diff --git a/grpc/src/ugrpc/server/service_component_base.cpp b/grpc/src/ugrpc/server/service_component_base.cpp index f774d06a9ddb..5e8ec350c0ed 100644 --- a/grpc/src/ugrpc/server/service_component_base.cpp +++ b/grpc/src/ugrpc/server/service_component_base.cpp @@ -25,6 +25,11 @@ void ServiceComponentBase::RegisterService(ServiceBase& service) { server_.GetServer().AddService(service, std::move(config_)); } +void ServiceComponentBase::RegisterService(GenericServiceBase& service) { + UINVARIANT(!registered_.exchange(true), "Register must only be called once"); + server_.GetServer().AddService(service, std::move(config_)); +} + yaml_config::Schema ServiceComponentBase::GetStaticConfigSchema() { return yaml_config::MergeSchemas(R"( type: object diff --git a/samples/CMakeLists.txt b/samples/CMakeLists.txt index 24f9a111d050..e2c599c16b7d 100644 --- a/samples/CMakeLists.txt +++ b/samples/CMakeLists.txt @@ -22,6 +22,9 @@ if (USERVER_FEATURE_OTLP) endif() if (USERVER_FEATURE_GRPC) + add_subdirectory(grpc-generic-proxy) + add_dependencies(${PROJECT_NAME} ${PROJECT_NAME}-grpc-generic-proxy) + add_subdirectory(grpc_service) add_dependencies(${PROJECT_NAME} ${PROJECT_NAME}-grpc_service) diff --git a/samples/grpc-generic-proxy/CMakeLists.txt b/samples/grpc-generic-proxy/CMakeLists.txt new file mode 100644 index 000000000000..433fee87208b --- /dev/null +++ b/samples/grpc-generic-proxy/CMakeLists.txt @@ -0,0 +1,20 @@ +cmake_minimum_required(VERSION 3.14) +project(userver-samples-grpc-generic-proxy CXX) + +find_package(userver COMPONENTS grpc REQUIRED) + +add_executable(${PROJECT_NAME} + main.cpp + src/proxy_service.cpp +) + +target_include_directories(${PROJECT_NAME} PRIVATE src) +target_link_libraries(${PROJECT_NAME} PRIVATE userver::grpc) + +# Actually unused in the service, only needed for testsuite tests. +# We could generate just the Python bindings, but this approach is currently +# not implemented in userver_add_grpc_library. +userver_add_grpc_library(${PROJECT_NAME}-proto PROTOS samples/greeter.proto) +add_dependencies(${PROJECT_NAME} ${PROJECT_NAME}-proto) + +userver_testsuite_add_simple() diff --git a/samples/grpc-generic-proxy/config_vars.yaml b/samples/grpc-generic-proxy/config_vars.yaml new file mode 100644 index 000000000000..5820a2ef6f82 --- /dev/null +++ b/samples/grpc-generic-proxy/config_vars.yaml @@ -0,0 +1,7 @@ +testsuite-enabled: true +server-port: 8080 +monitor-port: 8081 +grpc-server-port: 8090 +# "Magical" config_vars value that will cause testsuite to override it with +# real grpc mockserver endpoint +grpc-generic-endpoint: $grpc_mockserver diff --git a/samples/grpc-generic-proxy/main.cpp b/samples/grpc-generic-proxy/main.cpp new file mode 100644 index 000000000000..5eff44d7204d --- /dev/null +++ b/samples/grpc-generic-proxy/main.cpp @@ -0,0 +1,49 @@ +// For testing purposes only, in your services write out userver:: namespace +// instead. +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +int main(int argc, char* argv[]) { + const auto component_list = + components::MinimalServerComponentList() + // Base userver components + .Append() + .Append() + // HTTP client and server are (sadly) needed for testsuite support + .Append() + .Append() + .Append() + // gRPC client setup + .Append() + .Append() + .Append() + .Append>("generic-client") + // gRPC server setup + .Append() + .Append() + .Append() + .Append() + .Append(); + + return utils::DaemonMain(argc, argv, component_list); +} diff --git a/samples/grpc-generic-proxy/proto/samples/greeter.proto b/samples/grpc-generic-proxy/proto/samples/greeter.proto new file mode 100644 index 000000000000..b239fa823b85 --- /dev/null +++ b/samples/grpc-generic-proxy/proto/samples/greeter.proto @@ -0,0 +1,15 @@ +syntax = "proto3"; + +package samples.api; + +service GreeterService { + rpc SayHello(GreetingRequest) returns(GreetingResponse) {} +} + +message GreetingRequest { + string name = 1; +} + +message GreetingResponse { + string greeting = 1; +} diff --git a/samples/grpc-generic-proxy/src/proxy_service.cpp b/samples/grpc-generic-proxy/src/proxy_service.cpp new file mode 100644 index 000000000000..67133748af9e --- /dev/null +++ b/samples/grpc-generic-proxy/src/proxy_service.cpp @@ -0,0 +1,108 @@ +#include + +#include +#include +#include + +#include +#include +#include +#include + +namespace samples { + +namespace { + +grpc::string ToGrpcString(grpc::string_ref str) { + return {str.data(), str.size()}; +} + +void ProxyRequestMetadata(const grpc::ServerContext& server_context, + grpc::ClientContext& client_context) { + // Proxy all client (request) metadata, + // add some custom metadata as well. + for (const auto& [key, value] : server_context.client_metadata()) { + client_context.AddMetadata(ToGrpcString(key), ToGrpcString(value)); + } + client_context.AddMetadata("proxy-name", "grpc-generic-proxy"); +} + +void ProxyTrailingResponseMetadata(const grpc::ClientContext& client_context, + grpc::ServerContext& server_context) { + // Proxy all server (response) trailing metadata, + // add some custom metadata as well. + for (const auto& [key, value] : client_context.GetServerTrailingMetadata()) { + server_context.AddTrailingMetadata(ToGrpcString(key), ToGrpcString(value)); + } + server_context.AddTrailingMetadata("proxy-name", "grpc-generic-proxy"); +} + +} // namespace + +ProxyService::ProxyService(const components::ComponentConfig& config, + const components::ComponentContext& context) + : ugrpc::server::GenericServiceBase::Component(config, context), + client_(context + .FindComponent>("generic-client") + .GetClient()) {} + +void ProxyService::Handle(Call& call) { + // In this example we proxy any unary RPC to client_, adding some metadata. + + grpc::ByteBuffer request_bytes; + // Read might throw on a broken RPC, just rethrow then. + if (!call.Read(request_bytes)) { + // The client has already called WritesDone. + // We expect exactly 1 request, so that's an error for us. + call.FinishWithError(grpc::Status{grpc::StatusCode::INVALID_ARGUMENT, + "Expected exactly 1 request, given: 0"}); + return; + } + + grpc::ByteBuffer ignored_request_bytes; + // Wait until the client calls WritesDone before proceeding so that we know + // that no misuse will occur later. For unary RPCs, clients will essentially + // call WritesDone implicitly. + if (call.Read(ignored_request_bytes)) { + call.FinishWithError( + grpc::Status{grpc::StatusCode::INVALID_ARGUMENT, + "Expected exactly 1 request, given: at least 2"}); + return; + } + + auto client_context = std::make_unique(); + ProxyRequestMetadata(call.GetContext(), *client_context); + + // Deadline propagation will work, as we've registered the DP middleware + // in the config of grpc-server component. + // Optionally, we can set an additional timeout using GenericOptions::qos. + auto client_rpc = client_.UnaryCall(call.GetCallName(), request_bytes, + std::move(client_context)); + + grpc::ByteBuffer response_bytes; + try { + response_bytes = client_rpc.Finish(); + } catch (const ugrpc::client::ErrorWithStatus& ex) { + // Proxy the error returned from client. + ProxyTrailingResponseMetadata(client_rpc.GetContext(), call.GetContext()); + call.FinishWithError(ex.GetStatus()); + return; + } catch (const ugrpc::client::RpcError& ex) { + // Either the upstream client has cancelled our server RPC, or a network + // failure has occurred, or the deadline has expired. See: + // * ugrpc::client::RpcInterruptedError + // * ugrpc::client::RpcCancelledError + LOG_WARNING() << "Client RPC has failed: " << ex; + call.FinishWithError(grpc::Status{grpc::StatusCode::UNAVAILABLE, + "Failed to proxy the request"}); + return; + } + + ProxyTrailingResponseMetadata(client_rpc.GetContext(), call.GetContext()); + + // WriteAndFinish might throw on a broken RPC, just rethrow then. + call.WriteAndFinish(response_bytes); +} + +} // namespace samples diff --git a/samples/grpc-generic-proxy/src/proxy_service.hpp b/samples/grpc-generic-proxy/src/proxy_service.hpp new file mode 100644 index 000000000000..d655d078cb01 --- /dev/null +++ b/samples/grpc-generic-proxy/src/proxy_service.hpp @@ -0,0 +1,25 @@ +#pragma once + +// For testing purposes only, in your services write out userver:: namespace +// instead. +#include + +#include +#include + +namespace samples { + +class ProxyService final : public ugrpc::server::GenericServiceBase::Component { + public: + static constexpr std::string_view kName = "proxy-service"; + + ProxyService(const components::ComponentConfig& config, + const components::ComponentContext& context); + + void Handle(Call& call) override; + + private: + ugrpc::client::GenericClient& client_; +}; + +} // namespace samples diff --git a/samples/grpc-generic-proxy/static_config.yaml b/samples/grpc-generic-proxy/static_config.yaml new file mode 100644 index 000000000000..a07eeb6037bb --- /dev/null +++ b/samples/grpc-generic-proxy/static_config.yaml @@ -0,0 +1,80 @@ +# yaml +components_manager: + components: + # Base userver components + logging: + fs-task-processor: fs-task-processor + loggers: + default: + file_path: '@stderr' + level: debug + overflow_behavior: discard + testsuite-support: + congestion-control: + + # HTTP client and server are (sadly) needed for testsuite support + server: + load-enabled: $testsuite-enabled + listener: + port: $server-port + task_processor: main-task-processor + listener-monitor: + port: $monitor-port + task_processor: monitor-task-processor + http-client: + load-enabled: $testsuite-enabled + fs-task-processor: fs-task-processor + dns-client: + load-enabled: $testsuite-enabled + fs-task-processor: fs-task-processor + tests-control: + load-enabled: $testsuite-enabled + path: /tests/{action} + method: POST + task_processor: main-task-processor + testpoint-timeout: 10s + testpoint-url: mockserver/testpoint + throttling_enabled: false + + # gRPC client setup (ClientFactoryComponent and SimpleClientComponent) + grpc-client-factory: + task-processor: grpc-blocking-task-processor + middlewares: + - grpc-client-logging + - grpc-client-deadline-propagation + + grpc-client-logging: + grpc-client-deadline-propagation: + + generic-client: + endpoint: $grpc-generic-endpoint + + # gRPC server setup (ServerComponent and ProxyService) + grpc-server: + port: $grpc-server-port + service-defaults: + task-processor: main-task-processor + middlewares: + - grpc-server-logging + - grpc-server-deadline-propagation + - grpc-server-congestion-control + + grpc-server-logging: + grpc-server-deadline-propagation: + grpc-server-congestion-control: + + proxy-service: + + default_task_processor: main-task-processor + + task_processors: + main-task-processor: + worker_threads: 4 + monitor-task-processor: + worker_threads: 1 + thread_name: mon-worker + fs-task-processor: + worker_threads: 2 + grpc-blocking-task-processor: + worker_threads: 2 + thread_name: grpc-worker diff --git a/samples/grpc-generic-proxy/tests/conftest.py b/samples/grpc-generic-proxy/tests/conftest.py new file mode 100644 index 000000000000..b97d36616d8b --- /dev/null +++ b/samples/grpc-generic-proxy/tests/conftest.py @@ -0,0 +1,24 @@ +import pytest +import samples.greeter_pb2_grpc as greeter_services # noqa: E402, E501 + +pytest_plugins = ['pytest_userver.plugins.grpc'] + + +@pytest.fixture(name='mock_grpc_greeter_session', scope='session') +def _mock_grpc_greeter_session(grpc_mockserver, create_grpc_mock): + mock = create_grpc_mock(greeter_services.GreeterServiceServicer) + greeter_services.add_GreeterServiceServicer_to_server( + mock.servicer, grpc_mockserver, + ) + return mock + + +@pytest.fixture +def mock_grpc_greeter(mock_grpc_greeter_session): + with mock_grpc_greeter_session.mock() as mock: + yield mock + + +@pytest.fixture +def grpc_client(grpc_channel): + return greeter_services.GreeterServiceStub(grpc_channel) diff --git a/samples/grpc-generic-proxy/tests/test_proxy.py b/samples/grpc-generic-proxy/tests/test_proxy.py new file mode 100644 index 000000000000..7703f4bf6418 --- /dev/null +++ b/samples/grpc-generic-proxy/tests/test_proxy.py @@ -0,0 +1,96 @@ +import grpc +import grpc.aio +import pytest +import samples.greeter_pb2 as greeter_protos # noqa: E402, E501 + + +async def test_basic(grpc_client, mock_grpc_greeter, service_client): + @mock_grpc_greeter('SayHello') + async def mock_say_hello(request, context): + return greeter_protos.GreetingResponse( + greeting=f'Hello, {request.name} from mockserver!', + ) + + # TODO(TAXICOMMON-8865) make it implicit, like when calling HTTP handlers + await service_client.update_server_state() + + request = greeter_protos.GreetingRequest(name='Python') + response = await grpc_client.SayHello(request) + assert mock_say_hello.times_called == 1 + assert response.greeting == 'Hello, Python from mockserver!' + + +async def test_request_metadata( + grpc_client, mock_grpc_greeter, service_client, +): + @mock_grpc_greeter('SayHello') + async def mock_say_hello(request, context): + req_metadata = grpc.aio.Metadata(*context.invocation_metadata()) + if ( + req_metadata.get('routing-param', '') != 'request-meta-value' + or req_metadata.get('auth-key', '') != 'secret-stuff' + ): + raise ValueError('Failed to proxy the required metadata') + if req_metadata.get('proxy-name', '') != 'grpc-generic-proxy': + raise ValueError('Proxy failed to add custom metadata') + + return greeter_protos.GreetingResponse(greeting='Hi!') + + # TODO(TAXICOMMON-8865) make it implicit, like when calling HTTP handlers + await service_client.update_server_state() + + request = greeter_protos.GreetingRequest(name='Python') + response = await grpc_client.SayHello( + request, + metadata=( + ('routing-param', 'request-meta-value'), + ('auth-key', 'secret-stuff'), + ), + ) + assert mock_say_hello.times_called == 1 + assert response.greeting == 'Hi!' + + +async def test_trailing_response_metadata( + grpc_client, mock_grpc_greeter, service_client, +): + @mock_grpc_greeter('SayHello') + async def mock_say_hello(request, context): + context.set_trailing_metadata( + (('response-meta-key', 'response-meta-value'),), + ) + return greeter_protos.GreetingResponse(greeting='Hi!') + + # TODO(TAXICOMMON-8865) make it implicit, like when calling HTTP handlers + await service_client.update_server_state() + + request = greeter_protos.GreetingRequest(name='Python') + call = grpc_client.SayHello(request) + response = await call + assert mock_say_hello.times_called == 1 + assert response.greeting == 'Hi!' + resp_metadata = await call.trailing_metadata() + assert resp_metadata.get('response-meta-key', '') == 'response-meta-value' + + +async def test_error(grpc_client, mock_grpc_greeter, service_client): + @mock_grpc_greeter('SayHello') + async def mock_say_hello(request, context): + context.set_code(grpc.StatusCode.UNAVAILABLE) + context.set_details(f'Failed to greet {request.name}') + context.set_trailing_metadata( + (('response-meta-key', 'response-meta-value'),), + ) + return greeter_protos.GreetingResponse() + + # TODO(TAXICOMMON-8865) make it implicit, like when calling HTTP handlers + await service_client.update_server_state() + + request = greeter_protos.GreetingRequest(name='Python') + call = grpc_client.SayHello(request) + with pytest.raises(grpc.aio.AioRpcError) as e: + _ = await call + assert e.value.code() == grpc.StatusCode.UNAVAILABLE + assert e.value.details() == 'Failed to greet Python' + resp_metadata = await call.trailing_metadata() + assert resp_metadata.get('response-meta-key', '') == 'response-meta-value' diff --git a/samples/grpc_middleware_service/proto/ya.make b/samples/grpc_middleware_service/proto/ya.make deleted file mode 100644 index 588fd42a9caf..000000000000 --- a/samples/grpc_middleware_service/proto/ya.make +++ /dev/null @@ -1,35 +0,0 @@ -PROTO_LIBRARY(userver-functional-test-service-grpc-proto) - -SUBSCRIBER(g:taxi-common) - -EXCLUDE_TAGS(GO_PROTO JAVA_PROTO) - -SRCDIR(taxi/uservices/userver/samples/grpc_middleware_service/proto) - -SRCS( - samples/greeter.proto -) - -PROTO_NAMESPACE(taxi/uservices/userver/samples/grpc_middleware_service/proto) -PY_NAMESPACE(.) - -GRPC() - -CPP_PROTO_PLUGIN2( - usrv_service - taxi/uservices/userver-arc-utils/scripts/grpc/arc-generate-service - _service.usrv.pb.hpp - _service.usrv.pb.cpp - DEPS taxi/uservices/userver/grpc -) -CPP_PROTO_PLUGIN2( - usrv_client - taxi/uservices/userver-arc-utils/scripts/grpc/arc-generate-client - _client.usrv.pb.hpp - _client.usrv.pb.cpp - DEPS taxi/uservices/userver/grpc -) - -END() - - diff --git a/samples/grpc_middleware_service/tests/ya.make b/samples/grpc_middleware_service/tests/ya.make deleted file mode 100644 index 7b7ef0b89fac..000000000000 --- a/samples/grpc_middleware_service/tests/ya.make +++ /dev/null @@ -1,34 +0,0 @@ -SUBSCRIBER(g:taxi-common) - -PY3TEST() - -SIZE(SMALL) - -TEST_SRCS( - conftest.py - taxi/uservices/userver-arc-utils/functional_tests/basic/conftest.py - test_middlewares.py -) - -PEERDIR( - contrib/python/grpcio - contrib/python/yandex-taxi-testsuite - taxi/uservices/userver/testsuite/pytest_plugins/pytest_userver - taxi/uservices/userver/testsuite/pytest_plugins/pytest_userver/plugins/grpc - taxi/uservices/userver/samples/grpc_middleware_service/proto - contrib/python/psycopg2 - contrib/python/requests -) - -DEPENDS( - taxi/uservices/userver/samples/grpc_middleware_service -) - -DATA( - arcadia/taxi/uservices/userver/samples/grpc_middleware_service -) - -CONFTEST_LOAD_POLICY_LOCAL() -TEST_CWD(/) - -END() diff --git a/samples/grpc_middleware_service/ya.make b/samples/grpc_middleware_service/ya.make deleted file mode 100644 index 66b0bb975ff8..000000000000 --- a/samples/grpc_middleware_service/ya.make +++ /dev/null @@ -1,36 +0,0 @@ -PROGRAM(userver-functional-test-service) - -ALLOCATOR(J) - -SUBSCRIBER(g:taxi-common) - -SRCDIR(taxi/uservices/userver/samples/grpc_middleware_service/src) - -ADDINCL( - taxi/uservices/userver/samples/grpc_middleware_service/src -) - -PEERDIR( - contrib/libs/grpc - contrib/libs/grpc/grpcpp_channelz - contrib/libs/protobuf - taxi/uservices/userver/samples/grpc_middleware_service/proto - taxi/uservices/userver/core - taxi/uservices/userver/grpc -) - -SRCS( - main.cpp - client/view.cpp - service/view.cpp - http_handlers/say-hello/view.cpp - middlewares/client/component.cpp - middlewares/client/middleware.cpp - middlewares/server/component.cpp - middlewares/server/middleware.cpp - middlewares/auth.cpp -) - -END() - -RECURSE_FOR_TESTS(tests) diff --git a/scripts/docs/en/userver/grpc.md b/scripts/docs/en/userver/grpc.md index f7bc080544d4..faef9cd24bf0 100644 --- a/scripts/docs/en/userver/grpc.md +++ b/scripts/docs/en/userver/grpc.md @@ -169,6 +169,34 @@ Use ugrpc::server::MiddlewareBase and ugrpc::client::MiddlewareBase to implement new middlewares. +@anchor grpc_generic_api +## Generic API + +gRPC generic API allows to call and accept RPCs with dynamic service and method names. +The other side will see this as a normal RPC, it does not need to use generic API. + +Intended mainly for use in proxies. Metadata can be used to proxy the request without parsing it. + +See details in: + +* @ref ugrpc::client::GenericClient ; +* @ref ugrpc::server::GenericServiceBase . + +Full example showing the usage of both: + +* @include samples/grpc-generic-proxy/src/proxy_service.hpp +* @include samples/grpc-generic-proxy/src/proxy_service.cpp +* @include samples/grpc-generic-proxy/main.cpp +* @include samples/grpc-generic-proxy/static_config.yaml +* @include samples/grpc-generic-proxy/config_vars.yaml +* @include samples/grpc-generic-proxy/CMakeLists.txt + +Based on: + +* grpcpp [generic stub](https://grpc.github.io/grpc/cpp/grpcpp_2generic_2generic__stub_8h.html); +* grpcpp [generic service](https://grpc.github.io/grpc/cpp/grpcpp_2generic_2async__generic__service_8h.html). + + ## Metrics * Client metrics are put inside `grpc.client.by-destination {grpc_destination=FULL_SERVICE_NAME/METHOD_NAME}` diff --git a/testsuite/pytest_plugins/pytest_userver/plugins/grpc/mockserver.py b/testsuite/pytest_plugins/pytest_userver/plugins/grpc/mockserver.py index 68eac29e753f..48059d29e7f5 100644 --- a/testsuite/pytest_plugins/pytest_userver/plugins/grpc/mockserver.py +++ b/testsuite/pytest_plugins/pytest_userver/plugins/grpc/mockserver.py @@ -145,11 +145,9 @@ async def run_method(self, *args, **kwargs): method = mock.get(name, None) if method is not None: - call = method(*args, **kwargs) + return await method(*args, **kwargs) else: - call = default_method(self, *args, **kwargs) - - return await call + return default_method(self, *args, **kwargs) @functools.wraps(default_method) async def run_stream_method(self, *args, **kwargs): From 9791546b48de469c3abb5e594acd4b606312b2d7 Mon Sep 17 00:00:00 2001 From: segoon Date: Wed, 24 Jul 2024 21:51:28 +0300 Subject: [PATCH 080/629] bug chaotic: fix oneof w/ discriminator handling 66ba9eb7c2738686ff81785708ad09dcc2ab0642 --- .mapping.json | 6 + chaotic/chaotic/back/cpp/renderer.py | 5 +- chaotic/chaotic/back/cpp/translator.py | 5 + .../output/schemas/allof/allof.hpp | 3 +- .../output/schemas/allof/allof_fwd.hpp | 2 +- .../golden_tests/output/schemas/enum/enum.hpp | 3 +- .../output/schemas/enum/enum_fwd.hpp | 2 +- .../golden_tests/output/schemas/int/int.hpp | 3 +- .../output/schemas/int/int_fwd.hpp | 2 +- .../output/schemas/oneof/oneof.hpp | 3 +- .../output/schemas/oneof/oneof_fwd.hpp | 2 +- .../oneofdiscriminator/oneofdiscriminator.cpp | 143 ++++++++++++++++++ .../oneofdiscriminator/oneofdiscriminator.hpp | 124 +++++++++++++++ .../oneofdiscriminator_fwd.hpp | 11 ++ .../oneofdiscriminator_parsers.ipp | 89 +++++++++++ .../output/schemas/string/string.hpp | 3 +- .../output/schemas/string/string_fwd.hpp | 2 +- .../oneofdiscriminator.yaml | 31 ++++ .../userver/chaotic/type_bundle_cpp.hpp | 1 + .../schemas/oneofdiscriminator.yaml | 30 ++++ 20 files changed, 459 insertions(+), 11 deletions(-) create mode 100644 chaotic/golden_tests/output/schemas/oneofdiscriminator/oneofdiscriminator.cpp create mode 100644 chaotic/golden_tests/output/schemas/oneofdiscriminator/oneofdiscriminator.hpp create mode 100644 chaotic/golden_tests/output/schemas/oneofdiscriminator/oneofdiscriminator_fwd.hpp create mode 100644 chaotic/golden_tests/output/schemas/oneofdiscriminator/oneofdiscriminator_parsers.ipp create mode 100644 chaotic/golden_tests/schemas/oneofdiscriminator/oneofdiscriminator.yaml create mode 100644 chaotic/integration_tests/schemas/oneofdiscriminator.yaml diff --git a/.mapping.json b/.mapping.json index 88738c728d63..408e7411df6d 100644 --- a/.mapping.json +++ b/.mapping.json @@ -67,6 +67,10 @@ "chaotic/golden_tests/output/schemas/oneof/oneof.hpp":"taxi/uservices/userver/chaotic/golden_tests/output/schemas/oneof/oneof.hpp", "chaotic/golden_tests/output/schemas/oneof/oneof_fwd.hpp":"taxi/uservices/userver/chaotic/golden_tests/output/schemas/oneof/oneof_fwd.hpp", "chaotic/golden_tests/output/schemas/oneof/oneof_parsers.ipp":"taxi/uservices/userver/chaotic/golden_tests/output/schemas/oneof/oneof_parsers.ipp", + "chaotic/golden_tests/output/schemas/oneofdiscriminator/oneofdiscriminator.cpp":"taxi/uservices/userver/chaotic/golden_tests/output/schemas/oneofdiscriminator/oneofdiscriminator.cpp", + "chaotic/golden_tests/output/schemas/oneofdiscriminator/oneofdiscriminator.hpp":"taxi/uservices/userver/chaotic/golden_tests/output/schemas/oneofdiscriminator/oneofdiscriminator.hpp", + "chaotic/golden_tests/output/schemas/oneofdiscriminator/oneofdiscriminator_fwd.hpp":"taxi/uservices/userver/chaotic/golden_tests/output/schemas/oneofdiscriminator/oneofdiscriminator_fwd.hpp", + "chaotic/golden_tests/output/schemas/oneofdiscriminator/oneofdiscriminator_parsers.ipp":"taxi/uservices/userver/chaotic/golden_tests/output/schemas/oneofdiscriminator/oneofdiscriminator_parsers.ipp", "chaotic/golden_tests/output/schemas/string/string.cpp":"taxi/uservices/userver/chaotic/golden_tests/output/schemas/string/string.cpp", "chaotic/golden_tests/output/schemas/string/string.hpp":"taxi/uservices/userver/chaotic/golden_tests/output/schemas/string/string.hpp", "chaotic/golden_tests/output/schemas/string/string_fwd.hpp":"taxi/uservices/userver/chaotic/golden_tests/output/schemas/string/string_fwd.hpp", @@ -75,6 +79,7 @@ "chaotic/golden_tests/schemas/enum/enum.yaml":"taxi/uservices/userver/chaotic/golden_tests/schemas/enum/enum.yaml", "chaotic/golden_tests/schemas/int/int.yaml":"taxi/uservices/userver/chaotic/golden_tests/schemas/int/int.yaml", "chaotic/golden_tests/schemas/oneof/oneof.yaml":"taxi/uservices/userver/chaotic/golden_tests/schemas/oneof/oneof.yaml", + "chaotic/golden_tests/schemas/oneofdiscriminator/oneofdiscriminator.yaml":"taxi/uservices/userver/chaotic/golden_tests/schemas/oneofdiscriminator/oneofdiscriminator.yaml", "chaotic/golden_tests/schemas/string/string.yaml":"taxi/uservices/userver/chaotic/golden_tests/schemas/string/string.yaml", "chaotic/include/userver/chaotic/array.hpp":"taxi/uservices/userver/chaotic/include/userver/chaotic/array.hpp", "chaotic/include/userver/chaotic/convert.hpp":"taxi/uservices/userver/chaotic/include/userver/chaotic/convert.hpp", @@ -146,6 +151,7 @@ "chaotic/integration_tests/schemas/object_single_field.yaml":"taxi/uservices/userver/chaotic/integration_tests/schemas/object_single_field.yaml", "chaotic/integration_tests/schemas/one_of.yaml":"taxi/uservices/userver/chaotic/integration_tests/schemas/one_of.yaml", "chaotic/integration_tests/schemas/oneof_array_oneof.yaml":"taxi/uservices/userver/chaotic/integration_tests/schemas/oneof_array_oneof.yaml", + "chaotic/integration_tests/schemas/oneofdiscriminator.yaml":"taxi/uservices/userver/chaotic/integration_tests/schemas/oneofdiscriminator.yaml", "chaotic/integration_tests/schemas/pattern.yaml":"taxi/uservices/userver/chaotic/integration_tests/schemas/pattern.yaml", "chaotic/integration_tests/schemas/recursion.yaml":"taxi/uservices/userver/chaotic/integration_tests/schemas/recursion.yaml", "chaotic/integration_tests/schemas/uuid.yaml":"taxi/uservices/userver/chaotic/integration_tests/schemas/uuid.yaml", diff --git a/chaotic/chaotic/back/cpp/renderer.py b/chaotic/chaotic/back/cpp/renderer.py index bca5976abe69..2b794cee34ca 100644 --- a/chaotic/chaotic/back/cpp/renderer.py +++ b/chaotic/chaotic/back/cpp/renderer.py @@ -37,7 +37,10 @@ def get_current_namespace() -> str: def close_namespace() -> str: if current_namespace: - return '}' * (current_namespace.count('::') + 1) + data = '' + for name in reversed(current_namespace.split('::')): + data += '} //' + name + '\n' + return data else: return '' diff --git a/chaotic/chaotic/back/cpp/translator.py b/chaotic/chaotic/back/cpp/translator.py index 4ac7060d7fdd..608f11f3cecf 100644 --- a/chaotic/chaotic/back/cpp/translator.py +++ b/chaotic/chaotic/back/cpp/translator.py @@ -75,6 +75,11 @@ def mark_as_only_json(type_: cpp_types.CppType, reason: str) -> None: mark_as_only_json(parent, reason) for type_ in self.parent: + if isinstance(type_, cpp_types.CppVariantWithDiscriminator): + mark_as_only_json( + type_, + f'{type_.raw_cpp_type} has JSON-specific field "extra"', + ) if isinstance(type_, cpp_types.CppStruct): assert isinstance(type_, cpp_types.CppStruct) diff --git a/chaotic/golden_tests/output/schemas/allof/allof.hpp b/chaotic/golden_tests/output/schemas/allof/allof.hpp index b6aa69a93ef1..b20c6747de40 100644 --- a/chaotic/golden_tests/output/schemas/allof/allof.hpp +++ b/chaotic/golden_tests/output/schemas/allof/allof.hpp @@ -10,8 +10,9 @@ #include namespace ns { -namespace impl {} +namespace impl {} // namespace impl } // namespace ns + namespace ns { struct AllOf { diff --git a/chaotic/golden_tests/output/schemas/allof/allof_fwd.hpp b/chaotic/golden_tests/output/schemas/allof/allof_fwd.hpp index a3f5fd1726e6..fffa47dff96a 100644 --- a/chaotic/golden_tests/output/schemas/allof/allof_fwd.hpp +++ b/chaotic/golden_tests/output/schemas/allof/allof_fwd.hpp @@ -4,4 +4,4 @@ namespace ns { struct AllOf; -} +} // namespace ns diff --git a/chaotic/golden_tests/output/schemas/enum/enum.hpp b/chaotic/golden_tests/output/schemas/enum/enum.hpp index 23bca7c50e5d..958e032f16c6 100644 --- a/chaotic/golden_tests/output/schemas/enum/enum.hpp +++ b/chaotic/golden_tests/output/schemas/enum/enum.hpp @@ -9,8 +9,9 @@ #include namespace ns { -namespace impl {} +namespace impl {} // namespace impl } // namespace ns + namespace ns { struct Enum { diff --git a/chaotic/golden_tests/output/schemas/enum/enum_fwd.hpp b/chaotic/golden_tests/output/schemas/enum/enum_fwd.hpp index 8bf7eb0dfa9c..fe76c3d643b8 100644 --- a/chaotic/golden_tests/output/schemas/enum/enum_fwd.hpp +++ b/chaotic/golden_tests/output/schemas/enum/enum_fwd.hpp @@ -4,4 +4,4 @@ namespace ns { struct Enum; -} +} // namespace ns diff --git a/chaotic/golden_tests/output/schemas/int/int.hpp b/chaotic/golden_tests/output/schemas/int/int.hpp index 5f2a376e97a2..9bf57aeab473 100644 --- a/chaotic/golden_tests/output/schemas/int/int.hpp +++ b/chaotic/golden_tests/output/schemas/int/int.hpp @@ -9,8 +9,9 @@ #include namespace ns { -namespace impl {} +namespace impl {} // namespace impl } // namespace ns + namespace ns { struct Int { diff --git a/chaotic/golden_tests/output/schemas/int/int_fwd.hpp b/chaotic/golden_tests/output/schemas/int/int_fwd.hpp index 133c60ca6ca4..8c187ed77a1e 100644 --- a/chaotic/golden_tests/output/schemas/int/int_fwd.hpp +++ b/chaotic/golden_tests/output/schemas/int/int_fwd.hpp @@ -4,4 +4,4 @@ namespace ns { struct Int; -} +} // namespace ns diff --git a/chaotic/golden_tests/output/schemas/oneof/oneof.hpp b/chaotic/golden_tests/output/schemas/oneof/oneof.hpp index c959bdd774a3..e405a01b8239 100644 --- a/chaotic/golden_tests/output/schemas/oneof/oneof.hpp +++ b/chaotic/golden_tests/output/schemas/oneof/oneof.hpp @@ -12,8 +12,9 @@ #include namespace ns { -namespace impl {} +namespace impl {} // namespace impl } // namespace ns + namespace ns { struct OneOf { diff --git a/chaotic/golden_tests/output/schemas/oneof/oneof_fwd.hpp b/chaotic/golden_tests/output/schemas/oneof/oneof_fwd.hpp index ffb346fa80b2..178a3d41fa74 100644 --- a/chaotic/golden_tests/output/schemas/oneof/oneof_fwd.hpp +++ b/chaotic/golden_tests/output/schemas/oneof/oneof_fwd.hpp @@ -4,4 +4,4 @@ namespace ns { struct OneOf; -} +} // namespace ns diff --git a/chaotic/golden_tests/output/schemas/oneofdiscriminator/oneofdiscriminator.cpp b/chaotic/golden_tests/output/schemas/oneofdiscriminator/oneofdiscriminator.cpp new file mode 100644 index 000000000000..aa50b83c4c75 --- /dev/null +++ b/chaotic/golden_tests/output/schemas/oneofdiscriminator/oneofdiscriminator.cpp @@ -0,0 +1,143 @@ +#include "oneofdiscriminator.hpp" + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "oneofdiscriminator_parsers.ipp" + +namespace ns { + +bool operator==(const ns::A& lhs, const ns::A& rhs) { + return lhs.type == rhs.type && lhs.a_prop == rhs.a_prop && + lhs.extra == rhs.extra && + + true; +} + +USERVER_NAMESPACE::logging::LogHelper& operator<<( + USERVER_NAMESPACE::logging::LogHelper& lh, const ns::A& value) { + return lh << ToString(USERVER_NAMESPACE::formats::json::ValueBuilder(value) + .ExtractValue()); +} + +A Parse(USERVER_NAMESPACE::formats::json::Value json, + USERVER_NAMESPACE::formats::parse::To to) { + return Parse(json, to); +} + +/* Parse(USERVER_NAMESPACE::formats::yaml::Value, To) was not generated: + * ns::A has JSON-specific field "extra" */ + +/* Parse(USERVER_NAMESPACE::yaml_config::Value, To) was not generated: + * ns::A has JSON-specific field "extra" */ + +USERVER_NAMESPACE::formats::json::Value Serialize( + [[maybe_unused]] const ns::A& value, + USERVER_NAMESPACE::formats::serialize::To< + USERVER_NAMESPACE::formats::json::Value>) { + USERVER_NAMESPACE::formats::json::ValueBuilder vb = value.extra; + + if (value.type) { + vb["type"] = + USERVER_NAMESPACE::chaotic::Primitive{*value.type}; + } + + if (value.a_prop) { + vb["a_prop"] = USERVER_NAMESPACE::chaotic::Primitive{*value.a_prop}; + } + + return vb.ExtractValue(); +} + +bool operator==(const ns::B& lhs, const ns::B& rhs) { + return lhs.type == rhs.type && lhs.b_prop == rhs.b_prop && + lhs.extra == rhs.extra && + + true; +} + +USERVER_NAMESPACE::logging::LogHelper& operator<<( + USERVER_NAMESPACE::logging::LogHelper& lh, const ns::B& value) { + return lh << ToString(USERVER_NAMESPACE::formats::json::ValueBuilder(value) + .ExtractValue()); +} + +B Parse(USERVER_NAMESPACE::formats::json::Value json, + USERVER_NAMESPACE::formats::parse::To to) { + return Parse(json, to); +} + +/* Parse(USERVER_NAMESPACE::formats::yaml::Value, To) was not generated: + * ns::B has JSON-specific field "extra" */ + +/* Parse(USERVER_NAMESPACE::yaml_config::Value, To) was not generated: + * ns::B has JSON-specific field "extra" */ + +USERVER_NAMESPACE::formats::json::Value Serialize( + [[maybe_unused]] const ns::B& value, + USERVER_NAMESPACE::formats::serialize::To< + USERVER_NAMESPACE::formats::json::Value>) { + USERVER_NAMESPACE::formats::json::ValueBuilder vb = value.extra; + + if (value.type) { + vb["type"] = + USERVER_NAMESPACE::chaotic::Primitive{*value.type}; + } + + if (value.b_prop) { + vb["b_prop"] = USERVER_NAMESPACE::chaotic::Primitive{*value.b_prop}; + } + + return vb.ExtractValue(); +} + +bool operator==(const ns::OneOfDiscriminator& lhs, + const ns::OneOfDiscriminator& rhs) { + return lhs.foo == rhs.foo && true; +} + +USERVER_NAMESPACE::logging::LogHelper& operator<<( + USERVER_NAMESPACE::logging::LogHelper& lh, + const ns::OneOfDiscriminator& value) { + return lh << ToString(USERVER_NAMESPACE::formats::json::ValueBuilder(value) + .ExtractValue()); +} + +OneOfDiscriminator Parse( + USERVER_NAMESPACE::formats::json::Value json, + USERVER_NAMESPACE::formats::parse::To to) { + return Parse(json, to); +} + +/* Parse(USERVER_NAMESPACE::formats::yaml::Value, To) + * was not generated: ns::OneOfDiscriminator@Foo has JSON-specific field "extra" + */ + +/* Parse(USERVER_NAMESPACE::yaml_config::Value, To) was + * not generated: ns::OneOfDiscriminator@Foo has JSON-specific field "extra" */ + +USERVER_NAMESPACE::formats::json::Value Serialize( + [[maybe_unused]] const ns::OneOfDiscriminator& value, + USERVER_NAMESPACE::formats::serialize::To< + USERVER_NAMESPACE::formats::json::Value>) { + USERVER_NAMESPACE::formats::json::ValueBuilder vb = + USERVER_NAMESPACE::formats::common::Type::kObject; + + if (value.foo) { + vb["foo"] = USERVER_NAMESPACE::chaotic::OneOfWithDiscriminator< + &ns::impl::kns__OneOfDiscriminator__Foo_Settings, + USERVER_NAMESPACE::chaotic::Primitive, + USERVER_NAMESPACE::chaotic::Primitive>{*value.foo}; + } + + return vb.ExtractValue(); +} + +} // namespace ns diff --git a/chaotic/golden_tests/output/schemas/oneofdiscriminator/oneofdiscriminator.hpp b/chaotic/golden_tests/output/schemas/oneofdiscriminator/oneofdiscriminator.hpp new file mode 100644 index 000000000000..afc6709e828b --- /dev/null +++ b/chaotic/golden_tests/output/schemas/oneofdiscriminator/oneofdiscriminator.hpp @@ -0,0 +1,124 @@ +#pragma once + +#include "oneofdiscriminator_fwd.hpp" + +#include +#include +#include +#include +#include +#include + +#include + +namespace ns { +namespace impl {} // namespace impl +} // namespace ns + +namespace ns { + +struct A { + std::optional type{}; + std::optional a_prop{}; + + USERVER_NAMESPACE::formats::json::Value extra; +}; + +bool operator==(const ns::A& lhs, const ns::A& rhs); + +USERVER_NAMESPACE::logging::LogHelper& operator<<( + USERVER_NAMESPACE::logging::LogHelper& lh, const ns::A& value); + +A Parse(USERVER_NAMESPACE::formats::json::Value json, + USERVER_NAMESPACE::formats::parse::To); + +/* Parse(USERVER_NAMESPACE::formats::yaml::Value, To) was not generated: + * ns::A has JSON-specific field "extra" */ + +/* Parse(USERVER_NAMESPACE::yaml_config::Value, To) was not generated: + * ns::A has JSON-specific field "extra" */ + +USERVER_NAMESPACE::formats::json::Value Serialize( + const ns::A& value, USERVER_NAMESPACE::formats::serialize::To< + USERVER_NAMESPACE::formats::json::Value>); + +} // namespace ns + +namespace ns { +namespace impl {} // namespace impl +} // namespace ns + +namespace ns { + +struct B { + std::optional type{}; + std::optional b_prop{}; + + USERVER_NAMESPACE::formats::json::Value extra; +}; + +bool operator==(const ns::B& lhs, const ns::B& rhs); + +USERVER_NAMESPACE::logging::LogHelper& operator<<( + USERVER_NAMESPACE::logging::LogHelper& lh, const ns::B& value); + +B Parse(USERVER_NAMESPACE::formats::json::Value json, + USERVER_NAMESPACE::formats::parse::To); + +/* Parse(USERVER_NAMESPACE::formats::yaml::Value, To) was not generated: + * ns::B has JSON-specific field "extra" */ + +/* Parse(USERVER_NAMESPACE::yaml_config::Value, To) was not generated: + * ns::B has JSON-specific field "extra" */ + +USERVER_NAMESPACE::formats::json::Value Serialize( + const ns::B& value, USERVER_NAMESPACE::formats::serialize::To< + USERVER_NAMESPACE::formats::json::Value>); + +} // namespace ns + +namespace ns { +namespace impl { + +[[maybe_unused]] static constexpr USERVER_NAMESPACE::chaotic::OneOfSettings + kns__OneOfDiscriminator__Foo_Settings = { + "type", USERVER_NAMESPACE::utils::TrivialSet([](auto selector) { + return selector().template Type().Case("aaa").Case( + "bbb"); + })}; + +} // namespace impl +} // namespace ns + +namespace ns { + +struct OneOfDiscriminator { + using Foo = std::variant; + + std::optional foo{}; +}; + +bool operator==(const ns::OneOfDiscriminator& lhs, + const ns::OneOfDiscriminator& rhs); + +USERVER_NAMESPACE::logging::LogHelper& operator<<( + USERVER_NAMESPACE::logging::LogHelper& lh, + const ns::OneOfDiscriminator& value); + +OneOfDiscriminator Parse( + USERVER_NAMESPACE::formats::json::Value json, + USERVER_NAMESPACE::formats::parse::To); + +/* Parse(USERVER_NAMESPACE::formats::yaml::Value, To) + * was not generated: ns::OneOfDiscriminator@Foo has JSON-specific field "extra" + */ + +/* Parse(USERVER_NAMESPACE::yaml_config::Value, To) was + * not generated: ns::OneOfDiscriminator@Foo has JSON-specific field "extra" */ + +USERVER_NAMESPACE::formats::json::Value Serialize( + const ns::OneOfDiscriminator& value, + USERVER_NAMESPACE::formats::serialize::To< + USERVER_NAMESPACE::formats::json::Value>); + +} // namespace ns diff --git a/chaotic/golden_tests/output/schemas/oneofdiscriminator/oneofdiscriminator_fwd.hpp b/chaotic/golden_tests/output/schemas/oneofdiscriminator/oneofdiscriminator_fwd.hpp new file mode 100644 index 000000000000..f47dfb5d0e3f --- /dev/null +++ b/chaotic/golden_tests/output/schemas/oneofdiscriminator/oneofdiscriminator_fwd.hpp @@ -0,0 +1,11 @@ +#pragma once + +namespace ns { + +struct A; + +struct B; + +struct OneOfDiscriminator; + +} // namespace ns diff --git a/chaotic/golden_tests/output/schemas/oneofdiscriminator/oneofdiscriminator_parsers.ipp b/chaotic/golden_tests/output/schemas/oneofdiscriminator/oneofdiscriminator_parsers.ipp new file mode 100644 index 000000000000..982bcc7521cf --- /dev/null +++ b/chaotic/golden_tests/output/schemas/oneofdiscriminator/oneofdiscriminator_parsers.ipp @@ -0,0 +1,89 @@ +#pragma once + +#include "oneofdiscriminator.hpp" + +namespace ns { + +static constexpr USERVER_NAMESPACE::utils::TrivialSet kns__A_PropertiesNames = + [](auto selector) { + return selector().template Type().Case("type").Case( + "a_prop"); + }; + +template +ns::A Parse(Value value, USERVER_NAMESPACE::formats::parse::To) { + value.CheckNotMissing(); + value.CheckObjectOrNull(); + + ns::A res; + + res.type = value["type"] + .template As>>(); + res.a_prop = + value["a_prop"] + .template As< + std::optional>>(); + + res.extra = USERVER_NAMESPACE::chaotic::ExtractAdditionalPropertiesTrue( + value, kns__A_PropertiesNames); + + return res; +} + +static constexpr USERVER_NAMESPACE::utils::TrivialSet kns__B_PropertiesNames = + [](auto selector) { + return selector().template Type().Case("type").Case( + "b_prop"); + }; + +template +ns::B Parse(Value value, USERVER_NAMESPACE::formats::parse::To) { + value.CheckNotMissing(); + value.CheckObjectOrNull(); + + ns::B res; + + res.type = value["type"] + .template As>>(); + res.b_prop = + value["b_prop"] + .template As< + std::optional>>(); + + res.extra = USERVER_NAMESPACE::chaotic::ExtractAdditionalPropertiesTrue( + value, kns__B_PropertiesNames); + + return res; +} + +static constexpr USERVER_NAMESPACE::utils::TrivialSet + kns__OneOfDiscriminator_PropertiesNames = [](auto selector) { + return selector().template Type().Case("foo"); + }; + +template +ns::OneOfDiscriminator Parse( + Value value, + USERVER_NAMESPACE::formats::parse::To) { + value.CheckNotMissing(); + value.CheckObjectOrNull(); + + ns::OneOfDiscriminator res; + + res.foo = + value["foo"] + .template As< + std::optional, + USERVER_NAMESPACE::chaotic::Primitive>>>(); + + USERVER_NAMESPACE::chaotic::ValidateNoAdditionalProperties( + value, kns__OneOfDiscriminator_PropertiesNames); + + return res; +} + +} // namespace ns diff --git a/chaotic/golden_tests/output/schemas/string/string.hpp b/chaotic/golden_tests/output/schemas/string/string.hpp index ce28acfba972..260fd142afd8 100644 --- a/chaotic/golden_tests/output/schemas/string/string.hpp +++ b/chaotic/golden_tests/output/schemas/string/string.hpp @@ -9,8 +9,9 @@ #include namespace ns { -namespace impl {} +namespace impl {} // namespace impl } // namespace ns + namespace ns { struct String { diff --git a/chaotic/golden_tests/output/schemas/string/string_fwd.hpp b/chaotic/golden_tests/output/schemas/string/string_fwd.hpp index 2c341bafb13d..66cc4a99ebc2 100644 --- a/chaotic/golden_tests/output/schemas/string/string_fwd.hpp +++ b/chaotic/golden_tests/output/schemas/string/string_fwd.hpp @@ -4,4 +4,4 @@ namespace ns { struct String; -} +} // namespace ns diff --git a/chaotic/golden_tests/schemas/oneofdiscriminator/oneofdiscriminator.yaml b/chaotic/golden_tests/schemas/oneofdiscriminator/oneofdiscriminator.yaml new file mode 100644 index 000000000000..ecdd08d35b40 --- /dev/null +++ b/chaotic/golden_tests/schemas/oneofdiscriminator/oneofdiscriminator.yaml @@ -0,0 +1,31 @@ +components: + schemas: + OneOfDiscriminator: + type: object + additionalProperties: false + properties: + foo: + oneOf: + - $ref: '#/components/schemas/A' + - $ref: '#/components/schemas/B' + discriminator: + propertyName: type + mapping: + aaa: '#/components/schemas/A' + bbb: '#/components/schemas/B' + A: + type: object + additionalProperties: true + properties: + type: + type: string + a_prop: + type: integer + B: + type: object + additionalProperties: true + properties: + type: + type: string + b_prop: + type: integer diff --git a/chaotic/include/userver/chaotic/type_bundle_cpp.hpp b/chaotic/include/userver/chaotic/type_bundle_cpp.hpp index c37272db9c86..514c799164cd 100644 --- a/chaotic/include/userver/chaotic/type_bundle_cpp.hpp +++ b/chaotic/include/userver/chaotic/type_bundle_cpp.hpp @@ -3,5 +3,6 @@ #include #include #include +#include #include #include diff --git a/chaotic/integration_tests/schemas/oneofdiscriminator.yaml b/chaotic/integration_tests/schemas/oneofdiscriminator.yaml new file mode 100644 index 000000000000..c320cd3ee12b --- /dev/null +++ b/chaotic/integration_tests/schemas/oneofdiscriminator.yaml @@ -0,0 +1,30 @@ +definitions: + OneOfDiscriminator: + type: object + additionalProperties: false + properties: + foo: + oneOf: + - $ref: '#/definitions/A' + - $ref: '#/definitions/B' + discriminator: + propertyName: type + mapping: + aaa: '#/definitions/A' + bbb: '#/definitions/B' + A: + type: object + additionalProperties: true + properties: + type: + type: string + a_prop: + type: integer + B: + type: object + additionalProperties: true + properties: + type: + type: string + b_prop: + type: integer From 5012b68ec7614e7dacead6547b592e13a7eb614a Mon Sep 17 00:00:00 2001 From: antoshkka Date: Thu, 25 Jul 2024 10:49:02 +0300 Subject: [PATCH 081/629] feat docs: show all inherited members of a class as if those members were ordinary class members MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tests: протестировано локально faaa51e92e3f50c1803a334100bb74f97a6ae5e0 --- scripts/docs/doxygen.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/docs/doxygen.conf b/scripts/docs/doxygen.conf index 9a2733def415..f99bb1538536 100644 --- a/scripts/docs/doxygen.conf +++ b/scripts/docs/doxygen.conf @@ -155,7 +155,7 @@ ALWAYS_DETAILED_SEC = NO # operators of the base classes will not be shown. # The default value is: NO. -INLINE_INHERITED_MEMB = NO +INLINE_INHERITED_MEMB = YES # If the FULL_PATH_NAMES tag is set to YES, doxygen will prepend the full path # before files name in the file list and in the header files. If set to NO the From b6b9b36da7ae6b860a669bce5fee7c765dfe3674 Mon Sep 17 00:00:00 2001 From: antonyzhilin Date: Thu, 25 Jul 2024 13:13:41 +0300 Subject: [PATCH 082/629] refactor cmake: don't code-generate Find*.cmake MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `external-deps` is exterminated, rewritten in pure cmake. As a result: 1. Clarity of `Find*` is increased, the code is not hidden behind py + jinja 2. Removes a venv для `external-deps` => in some cases Python dependency is not needed at all for release builds 3. Slight speed-up of the first cmake configure (~2 seconds) 4. `Find*` can be further refactored and improved. Now it's much easier to do compared to `FindHelper.jinja` a249e6b2fc0b0d8b4e5b860de3067418fdffbd4f --- .gitignore | 1 - .mapping.json | 65 ++-- CMakeLists.txt | 22 +- clickhouse/CMakeLists.txt | 2 +- cmake/DetectVersion.cmake | 2 + cmake/ModuleHelpers.cmake | 352 ++++++++++++++++++ cmake/UserverSetupEnvironment.cmake | 2 +- cmake/install/Config.cmake.in | 6 +- cmake/install/userver-chaotic-config.cmake | 2 - cmake/install/userver-clickhouse-config.cmake | 1 - cmake/install/userver-core-config.cmake | 4 +- cmake/install/userver-grpc-config.cmake | 1 - cmake/install/userver-kafka-config.cmake | 1 - cmake/install/userver-mongo-config.cmake | 1 - cmake/install/userver-mysql-config.cmake | 1 - cmake/install/userver-otlp-config.cmake | 2 - cmake/install/userver-rabbitmq-config.cmake | 1 - cmake/install/userver-redis-config.cmake | 1 - cmake/install/userver-universal-config.cmake | 1 - cmake/install/userver-ydb-config.cmake | 1 - cmake/modules/FindBrotli.cmake | 24 ++ cmake/modules/FindCrypto.cmake | 12 + cmake/modules/FindCryptoPP.cmake | 20 + cmake/modules/FindGit.cmake | 10 + cmake/modules/FindGrpcChannelz.cmake | 17 + cmake/modules/FindGssApi.cmake | 17 + cmake/modules/FindHiredis.cmake | 19 + cmake/modules/FindJemalloc.cmake | 25 ++ cmake/modules/FindLibEv.cmake | 17 + cmake/modules/FindNghttp2.cmake | 16 + cmake/modules/FindPostgreSQLInternal.cmake | 88 +++++ cmake/modules/FindPythonDev.cmake | 29 ++ cmake/modules/FindRdKafka.cmake | 28 ++ cmake/modules/FindSSL.cmake | 12 + cmake/modules/FindUserverGBench.cmake | 23 ++ cmake/modules/FindUserverGrpc.cmake | 32 ++ cmake/modules/Findbson.cmake | 21 ++ cmake/modules/Findc-ares.cmake | 19 + cmake/modules/Findcctz.cmake | 19 + cmake/modules/Findclickhouse-cpp.cmake | 16 + cmake/modules/Findfmt.cmake | 19 + cmake/modules/Findlibgflags.cmake | 23 ++ cmake/modules/Findlibintl.cmake | 14 + cmake/modules/Findlibmariadb.cmake | 16 + cmake/modules/Findlibsnappy.cmake | 23 ++ cmake/modules/Findlibyamlcpp.cmake | 20 + cmake/modules/Findlibz.cmake | 19 + cmake/modules/Findlibzip.cmake | 19 + cmake/modules/Findlibzstd.cmake | 22 ++ cmake/modules/Findmongoc.cmake | 21 ++ core/CMakeLists.txt | 6 +- external-deps/Brotli.yaml | 21 -- external-deps/CryptoPP.yaml | 26 -- external-deps/Git.yaml | 16 - external-deps/GrpcChannelz.yaml | 23 -- external-deps/GssApi.yaml | 22 -- external-deps/Hiredis.yaml | 22 -- external-deps/Jemalloc.yaml | 27 -- external-deps/LibEv.yaml | 19 - external-deps/Nghttp2.yaml | 17 - external-deps/OpenSSL.yaml | 38 -- external-deps/PostgreSQLInternal.yaml | 87 ----- external-deps/README.md | 4 - external-deps/RdKafka.yaml | 31 -- external-deps/UserverGBench.yaml | 28 -- external-deps/UserverGrpc.yaml | 31 -- external-deps/bson.yaml | 23 -- external-deps/c-ares.yaml | 24 -- external-deps/cctz.yaml | 23 -- external-deps/clickhouse-cpp.yaml | 14 - external-deps/fmt.yaml | 22 -- external-deps/libgflags.yaml | 26 -- external-deps/libintl.yaml | 12 - external-deps/libmariadb.yaml | 17 - external-deps/libsnappy.yaml | 26 -- external-deps/libyamlcpp.yaml | 22 -- external-deps/libz.yaml | 25 -- external-deps/libzip.yaml | 24 -- external-deps/libzstd.yaml | 25 -- external-deps/mongoc.yaml | 23 -- external-deps/python-dev.yaml | 33 -- grpc/CMakeLists.txt | 2 +- kafka/CMakeLists.txt | 2 +- mongo/CMakeLists.txt | 4 +- mysql/CMakeLists.txt | 2 +- postgresql/pq-extra/CMakeLists.txt | 4 +- redis/CMakeLists.txt | 2 +- scripts/external_deps/__init__.py | 0 scripts/external_deps/cmake_generator.py | 161 -------- scripts/external_deps/requirements.txt | 3 - .../external_deps/templates/FindHelper.jinja | 231 ------------ universal/CMakeLists.txt | 13 +- 92 files changed, 1069 insertions(+), 1241 deletions(-) create mode 100644 cmake/ModuleHelpers.cmake create mode 100644 cmake/modules/FindBrotli.cmake create mode 100644 cmake/modules/FindCrypto.cmake create mode 100644 cmake/modules/FindCryptoPP.cmake create mode 100644 cmake/modules/FindGit.cmake create mode 100644 cmake/modules/FindGrpcChannelz.cmake create mode 100644 cmake/modules/FindGssApi.cmake create mode 100644 cmake/modules/FindHiredis.cmake create mode 100644 cmake/modules/FindJemalloc.cmake create mode 100644 cmake/modules/FindLibEv.cmake create mode 100644 cmake/modules/FindNghttp2.cmake create mode 100644 cmake/modules/FindPostgreSQLInternal.cmake create mode 100644 cmake/modules/FindPythonDev.cmake create mode 100644 cmake/modules/FindRdKafka.cmake create mode 100644 cmake/modules/FindSSL.cmake create mode 100644 cmake/modules/FindUserverGBench.cmake create mode 100644 cmake/modules/FindUserverGrpc.cmake create mode 100644 cmake/modules/Findbson.cmake create mode 100644 cmake/modules/Findc-ares.cmake create mode 100644 cmake/modules/Findcctz.cmake create mode 100644 cmake/modules/Findclickhouse-cpp.cmake create mode 100644 cmake/modules/Findfmt.cmake create mode 100644 cmake/modules/Findlibgflags.cmake create mode 100644 cmake/modules/Findlibintl.cmake create mode 100644 cmake/modules/Findlibmariadb.cmake create mode 100644 cmake/modules/Findlibsnappy.cmake create mode 100644 cmake/modules/Findlibyamlcpp.cmake create mode 100644 cmake/modules/Findlibz.cmake create mode 100644 cmake/modules/Findlibzip.cmake create mode 100644 cmake/modules/Findlibzstd.cmake create mode 100644 cmake/modules/Findmongoc.cmake delete mode 100644 external-deps/Brotli.yaml delete mode 100644 external-deps/CryptoPP.yaml delete mode 100644 external-deps/Git.yaml delete mode 100644 external-deps/GrpcChannelz.yaml delete mode 100644 external-deps/GssApi.yaml delete mode 100644 external-deps/Hiredis.yaml delete mode 100644 external-deps/Jemalloc.yaml delete mode 100644 external-deps/LibEv.yaml delete mode 100644 external-deps/Nghttp2.yaml delete mode 100644 external-deps/OpenSSL.yaml delete mode 100644 external-deps/PostgreSQLInternal.yaml delete mode 100644 external-deps/README.md delete mode 100644 external-deps/RdKafka.yaml delete mode 100644 external-deps/UserverGBench.yaml delete mode 100644 external-deps/UserverGrpc.yaml delete mode 100644 external-deps/bson.yaml delete mode 100644 external-deps/c-ares.yaml delete mode 100644 external-deps/cctz.yaml delete mode 100644 external-deps/clickhouse-cpp.yaml delete mode 100644 external-deps/fmt.yaml delete mode 100644 external-deps/libgflags.yaml delete mode 100644 external-deps/libintl.yaml delete mode 100644 external-deps/libmariadb.yaml delete mode 100644 external-deps/libsnappy.yaml delete mode 100644 external-deps/libyamlcpp.yaml delete mode 100644 external-deps/libz.yaml delete mode 100644 external-deps/libzip.yaml delete mode 100644 external-deps/libzstd.yaml delete mode 100644 external-deps/mongoc.yaml delete mode 100644 external-deps/python-dev.yaml delete mode 100644 scripts/external_deps/__init__.py delete mode 100644 scripts/external_deps/cmake_generator.py delete mode 100644 scripts/external_deps/requirements.txt delete mode 100644 scripts/external_deps/templates/FindHelper.jinja diff --git a/.gitignore b/.gitignore index 15eb9ed9d223..77a54fdb5827 100644 --- a/.gitignore +++ b/.gitignore @@ -11,7 +11,6 @@ .cores/ /build*/ /cmake-build-*/ -/cmake/cmake_generated/ /docs/ /third_party/ CMakeLists.txt.user diff --git a/.mapping.json b/.mapping.json index 408e7411df6d..bd9a3f4d69e8 100644 --- a/.mapping.json +++ b/.mapping.json @@ -331,6 +331,7 @@ "cmake/GetUserverVersion.cmake":"taxi/uservices/userver/cmake/GetUserverVersion.cmake", "cmake/GrpcTargets.cmake":"taxi/uservices/userver/cmake/GrpcTargets.cmake", "cmake/IncludeWhatYouUse.cmake":"taxi/uservices/userver/cmake/IncludeWhatYouUse.cmake", + "cmake/ModuleHelpers.cmake":"taxi/uservices/userver/cmake/ModuleHelpers.cmake", "cmake/PrepareInstall.cmake":"taxi/uservices/userver/cmake/PrepareInstall.cmake", "cmake/README.md":"taxi/uservices/userver/cmake/README.md", "cmake/RequireLTO.cmake":"taxi/uservices/userver/cmake/RequireLTO.cmake", @@ -379,6 +380,36 @@ "cmake/install/userver-redis-config.cmake":"taxi/uservices/userver/cmake/install/userver-redis-config.cmake", "cmake/install/userver-universal-config.cmake":"taxi/uservices/userver/cmake/install/userver-universal-config.cmake", "cmake/install/userver-ydb-config.cmake":"taxi/uservices/userver/cmake/install/userver-ydb-config.cmake", + "cmake/modules/FindBrotli.cmake":"taxi/uservices/userver/cmake/modules/FindBrotli.cmake", + "cmake/modules/FindCrypto.cmake":"taxi/uservices/userver/cmake/modules/FindCrypto.cmake", + "cmake/modules/FindCryptoPP.cmake":"taxi/uservices/userver/cmake/modules/FindCryptoPP.cmake", + "cmake/modules/FindGit.cmake":"taxi/uservices/userver/cmake/modules/FindGit.cmake", + "cmake/modules/FindGrpcChannelz.cmake":"taxi/uservices/userver/cmake/modules/FindGrpcChannelz.cmake", + "cmake/modules/FindGssApi.cmake":"taxi/uservices/userver/cmake/modules/FindGssApi.cmake", + "cmake/modules/FindHiredis.cmake":"taxi/uservices/userver/cmake/modules/FindHiredis.cmake", + "cmake/modules/FindJemalloc.cmake":"taxi/uservices/userver/cmake/modules/FindJemalloc.cmake", + "cmake/modules/FindLibEv.cmake":"taxi/uservices/userver/cmake/modules/FindLibEv.cmake", + "cmake/modules/FindNghttp2.cmake":"taxi/uservices/userver/cmake/modules/FindNghttp2.cmake", + "cmake/modules/FindPostgreSQLInternal.cmake":"taxi/uservices/userver/cmake/modules/FindPostgreSQLInternal.cmake", + "cmake/modules/FindPythonDev.cmake":"taxi/uservices/userver/cmake/modules/FindPythonDev.cmake", + "cmake/modules/FindRdKafka.cmake":"taxi/uservices/userver/cmake/modules/FindRdKafka.cmake", + "cmake/modules/FindSSL.cmake":"taxi/uservices/userver/cmake/modules/FindSSL.cmake", + "cmake/modules/FindUserverGBench.cmake":"taxi/uservices/userver/cmake/modules/FindUserverGBench.cmake", + "cmake/modules/FindUserverGrpc.cmake":"taxi/uservices/userver/cmake/modules/FindUserverGrpc.cmake", + "cmake/modules/Findbson.cmake":"taxi/uservices/userver/cmake/modules/Findbson.cmake", + "cmake/modules/Findc-ares.cmake":"taxi/uservices/userver/cmake/modules/Findc-ares.cmake", + "cmake/modules/Findcctz.cmake":"taxi/uservices/userver/cmake/modules/Findcctz.cmake", + "cmake/modules/Findclickhouse-cpp.cmake":"taxi/uservices/userver/cmake/modules/Findclickhouse-cpp.cmake", + "cmake/modules/Findfmt.cmake":"taxi/uservices/userver/cmake/modules/Findfmt.cmake", + "cmake/modules/Findlibgflags.cmake":"taxi/uservices/userver/cmake/modules/Findlibgflags.cmake", + "cmake/modules/Findlibintl.cmake":"taxi/uservices/userver/cmake/modules/Findlibintl.cmake", + "cmake/modules/Findlibmariadb.cmake":"taxi/uservices/userver/cmake/modules/Findlibmariadb.cmake", + "cmake/modules/Findlibsnappy.cmake":"taxi/uservices/userver/cmake/modules/Findlibsnappy.cmake", + "cmake/modules/Findlibyamlcpp.cmake":"taxi/uservices/userver/cmake/modules/Findlibyamlcpp.cmake", + "cmake/modules/Findlibz.cmake":"taxi/uservices/userver/cmake/modules/Findlibz.cmake", + "cmake/modules/Findlibzip.cmake":"taxi/uservices/userver/cmake/modules/Findlibzip.cmake", + "cmake/modules/Findlibzstd.cmake":"taxi/uservices/userver/cmake/modules/Findlibzstd.cmake", + "cmake/modules/Findmongoc.cmake":"taxi/uservices/userver/cmake/modules/Findmongoc.cmake", "cmake/sanitize-macos.blacklist.txt":"taxi/uservices/userver/cmake/sanitize-macos.blacklist.txt", "cmake/sanitize.blacklist.txt":"taxi/uservices/userver/cmake/sanitize.blacklist.txt", "conan/README.md":"taxi/uservices/userver/conan/README.md", @@ -1687,36 +1718,6 @@ "core/utest/src/utest/test_case_macros_test.cpp":"taxi/uservices/userver/core/utest/src/utest/test_case_macros_test.cpp", "core/utest/src/utest/utest.cpp":"taxi/uservices/userver/core/utest/src/utest/utest.cpp", "core/utest/src/utils/statistics/testing.cpp":"taxi/uservices/userver/core/utest/src/utils/statistics/testing.cpp", - "external-deps/Brotli.yaml":"taxi/uservices/userver/external-deps/Brotli.yaml", - "external-deps/CryptoPP.yaml":"taxi/uservices/userver/external-deps/CryptoPP.yaml", - "external-deps/Git.yaml":"taxi/uservices/userver/external-deps/Git.yaml", - "external-deps/GrpcChannelz.yaml":"taxi/uservices/userver/external-deps/GrpcChannelz.yaml", - "external-deps/GssApi.yaml":"taxi/uservices/userver/external-deps/GssApi.yaml", - "external-deps/Hiredis.yaml":"taxi/uservices/userver/external-deps/Hiredis.yaml", - "external-deps/Jemalloc.yaml":"taxi/uservices/userver/external-deps/Jemalloc.yaml", - "external-deps/LibEv.yaml":"taxi/uservices/userver/external-deps/LibEv.yaml", - "external-deps/Nghttp2.yaml":"taxi/uservices/userver/external-deps/Nghttp2.yaml", - "external-deps/OpenSSL.yaml":"taxi/uservices/userver/external-deps/OpenSSL.yaml", - "external-deps/PostgreSQLInternal.yaml":"taxi/uservices/userver/external-deps/PostgreSQLInternal.yaml", - "external-deps/README.md":"taxi/uservices/userver/external-deps/README.md", - "external-deps/RdKafka.yaml":"taxi/uservices/userver/external-deps/RdKafka.yaml", - "external-deps/UserverGBench.yaml":"taxi/uservices/userver/external-deps/UserverGBench.yaml", - "external-deps/UserverGrpc.yaml":"taxi/uservices/userver/external-deps/UserverGrpc.yaml", - "external-deps/bson.yaml":"taxi/uservices/userver/external-deps/bson.yaml", - "external-deps/c-ares.yaml":"taxi/uservices/userver/external-deps/c-ares.yaml", - "external-deps/cctz.yaml":"taxi/uservices/userver/external-deps/cctz.yaml", - "external-deps/clickhouse-cpp.yaml":"taxi/uservices/userver/external-deps/clickhouse-cpp.yaml", - "external-deps/fmt.yaml":"taxi/uservices/userver/external-deps/fmt.yaml", - "external-deps/libgflags.yaml":"taxi/uservices/userver/external-deps/libgflags.yaml", - "external-deps/libintl.yaml":"taxi/uservices/userver/external-deps/libintl.yaml", - "external-deps/libmariadb.yaml":"taxi/uservices/userver/external-deps/libmariadb.yaml", - "external-deps/libsnappy.yaml":"taxi/uservices/userver/external-deps/libsnappy.yaml", - "external-deps/libyamlcpp.yaml":"taxi/uservices/userver/external-deps/libyamlcpp.yaml", - "external-deps/libz.yaml":"taxi/uservices/userver/external-deps/libz.yaml", - "external-deps/libzip.yaml":"taxi/uservices/userver/external-deps/libzip.yaml", - "external-deps/libzstd.yaml":"taxi/uservices/userver/external-deps/libzstd.yaml", - "external-deps/mongoc.yaml":"taxi/uservices/userver/external-deps/mongoc.yaml", - "external-deps/python-dev.yaml":"taxi/uservices/userver/external-deps/python-dev.yaml", "grpc/CMakeLists.txt":"taxi/uservices/userver/grpc/CMakeLists.txt", "grpc/README.md":"taxi/uservices/userver/grpc/README.md", "grpc/benchmarks/base.cpp":"taxi/uservices/userver/grpc/benchmarks/base.cpp", @@ -3311,10 +3312,6 @@ "scripts/docs/scripts/styledBtn.js":"taxi/uservices/userver/scripts/docs/scripts/styledBtn.js", "scripts/docs/scripts/telegramLanguage.js":"taxi/uservices/userver/scripts/docs/scripts/telegramLanguage.js", "scripts/docs/scripts/toc.js":"taxi/uservices/userver/scripts/docs/scripts/toc.js", - "scripts/external_deps/__init__.py":"taxi/uservices/userver/scripts/external_deps/__init__.py", - "scripts/external_deps/cmake_generator.py":"taxi/uservices/userver/scripts/external_deps/cmake_generator.py", - "scripts/external_deps/requirements.txt":"taxi/uservices/userver/scripts/external_deps/requirements.txt", - "scripts/external_deps/templates/FindHelper.jinja":"taxi/uservices/userver/scripts/external_deps/templates/FindHelper.jinja", "scripts/grpc/__init__.py":"taxi/uservices/userver/scripts/grpc/__init__.py", "scripts/grpc/generator.py":"taxi/uservices/userver/scripts/grpc/generator.py", "scripts/grpc/protoc_usrv_plugin.sh":"taxi/uservices/userver/scripts/grpc/protoc_usrv_plugin.sh", diff --git a/CMakeLists.txt b/CMakeLists.txt index a20f23465b0b..15399298f254 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -141,6 +141,7 @@ if(USERVER_INSTALL) include(GNUInstallDirs) endif() +include(ModuleHelpers) include(GetUserverVersion) include(AddGoogleTests) include(FindPackageRequired) @@ -149,27 +150,6 @@ include(UserverTestsuite) include(CheckCompileFlags) include(CMakePackageConfigHelpers) -message(STATUS "Generating cmake files ...") -userver_venv_setup( - NAME userver-cmake - PYTHON_OUTPUT_VAR USERVER_CMAKE_GEN_PYTHON_BINARY - REQUIREMENTS "${USERVER_ROOT_DIR}/scripts/external_deps/requirements.txt" -) -execute_process( - COMMAND - "${USERVER_CMAKE_GEN_PYTHON_BINARY}" - -u "${USERVER_ROOT_DIR}/scripts/external_deps/cmake_generator.py" - --repo-dir "${USERVER_ROOT_DIR}" - --build-dir "${CMAKE_BINARY_DIR}" - RESULT_VARIABLE RESULT - WORKING_DIRECTORY "${USERVER_ROOT_DIR}" -) -if(RESULT) - message(FATAL_ERROR - "Generating cmake files failed with exit code: ${RESULT}" - ) -endif(RESULT) - set(USERVER_THIRD_PARTY_DIRS ${USERVER_ROOT_DIR}/third_party CACHE INTERNAL "") init_debian_depends() diff --git a/clickhouse/CMakeLists.txt b/clickhouse/CMakeLists.txt index 7319f0e42800..af6e20a1e825 100644 --- a/clickhouse/CMakeLists.txt +++ b/clickhouse/CMakeLists.txt @@ -51,7 +51,7 @@ _userver_install_targets(COMPONENT clickhouse TARGETS ${PROJECT_NAME}) _userver_directory_install(COMPONENT clickhouse FILES "${USERVER_ROOT_DIR}/cmake/install/userver-clickhouse-config.cmake" - "${CMAKE_BINARY_DIR}/cmake_generated/Findclickhouse-cpp.cmake" + "${USERVER_ROOT_DIR}/cmake/modules/Findclickhouse-cpp.cmake" DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/userver ) diff --git a/cmake/DetectVersion.cmake b/cmake/DetectVersion.cmake index 5b162b9b6075..d496501c723b 100644 --- a/cmake/DetectVersion.cmake +++ b/cmake/DetectVersion.cmake @@ -1,3 +1,5 @@ +include_guard(GLOBAL) + macro(userver_version_strip_trash VERSION_OUTPUT_VAR) STRING(REGEX REPLACE "^1:" "" ${VERSION_OUTPUT_VAR} "${${VERSION_OUTPUT_VAR}}") endmacro() diff --git a/cmake/ModuleHelpers.cmake b/cmake/ModuleHelpers.cmake new file mode 100644 index 000000000000..0703019539aa --- /dev/null +++ b/cmake/ModuleHelpers.cmake @@ -0,0 +1,352 @@ +include_guard(GLOBAL) + +cmake_policy(SET CMP0054 NEW) + +macro(_userver_module_begin) + set(options) + set(oneValueArgs + # Target name, also used for package name by default + NAME + # Custom package name; NAME is used by default + PACKAGE_NAME + # For multi-target packages + COMMON_NAME + VERSION + ) + set(multiValueArgs + DEBIAN_NAMES + FORMULA_NAMES + RPM_NAMES + PACMAN_NAMES + PKG_NAMES + # For version detection of manually installed packages and unknown + # package managers. + PKG_CONFIG_NAMES + ) + + cmake_parse_arguments( + ARG "${options}" "${oneValueArgs}" "${multiValueArgs}" "${ARGN}") + + set(name "${ARG_NAME}") + + if(ARG_VERSION) + if(NOT ${name}_FIND_VERSION OR "${${name}_FIND_VERSION}" VERSION_LESS "${ARG_VERSION}") + set("${name}_FIND_VERSION" "${ARG_VERSION}") + endif() + endif() + + if(NOT USERVER_CHECK_PACKAGE_VERSIONS) + unset("${name}_FIND_VERSION") + endif() + + if(TARGET "${name}") + if(NOT ${name}_FIND_VERSION) + set("${name}_FOUND" ON) + set("${name}_SKIP_USERVER_FIND" ON) + return() + endif() + + if(${name}_VERSION) + if(${name}_FIND_VERSION VERSION_LESS_EQUAL ${name}_VERSION) + set("${name}_FOUND" ON) + set("${name}_SKIP_USERVER_FIND" ON) + return() + else() + message(FATAL_ERROR + "Already using version ${${name}_VERSION}" + "of ${name} when version ${${name}_FIND_VERSION} " + "was requested." + ) + endif() + endif() + endif() + + set(FULL_ERROR_MESSAGE "Could not find `${name}` package.") + if(ARG_DEBIAN_NAMES) + list(JOIN ARG_DEBIAN_NAMES " " package_names_joined) + string(APPEND FULL_ERROR_MESSAGE "\\n\\tDebian: sudo apt update && sudo apt install ${package_names_joined}") + endif() + if(ARG_FORMULA_NAMES) + list(JOIN ARG_FORMULA_NAMES " " package_names_joined) + string(APPEND FULL_ERROR_MESSAGE "\\n\\tMacOS: brew install ${package_names_joined}") + endif() + if(ARG_RPM_NAMES) + list(JOIN ARG_RPM_NAMES " " package_names_joined) + string(APPEND FULL_ERROR_MESSAGE "\\n\\tFedora: sudo dnf install ${package_names_joined}") + endif() + if(ARG_PACMAN_NAMES) + list(JOIN ARG_PACMAN_NAMES " " package_names_joined) + string(APPEND FULL_ERROR_MESSAGE "\\n\\tArchLinux: sudo pacman -S ${package_names_joined}") + endif() + if(ARG_PKG_NAMES) + list(JOIN ARG_PKG_NAMES " " package_names_joined) + string(APPEND FULL_ERROR_MESSAGE "\\n\\tFreeBSD: pkg install ${package_names_joined}") + endif() + string(APPEND FULL_ERROR_MESSAGE "\\n") + + set("${ARG_NAME}_LIBRARIES") + set("${ARG_NAME}_INCLUDE_DIRS") + set("${ARG_NAME}_EXECUTABLE") +endmacro() + +macro(_userver_module_find_part) + # Also uses ARGs left over from _userver_find_module_begin + + set(options) + set(oneValueArgs + PART_TYPE + ) + set(multiValueArgs + NAMES + PATHS + PATH_SUFFIXES + ) + + cmake_parse_arguments( + ARG "${options}" "${oneValueArgs}" "${multiValueArgs}" "${ARGN}") + + if(${ARG_NAME}_SKIP_USERVER_FIND) + return() + endif() + + if("${ARG_PART_TYPE}" STREQUAL "library") + set(variable "${ARG_NAME}_LIBRARIES") + elseif("${ARG_PART_TYPE}" STREQUAL "path") + set(variable "${ARG_NAME}_INCLUDE_DIRS") + elseif("${ARG_PART_TYPE}" STREQUAL "program") + set(variable "${ARG_NAME}_EXECUTABLE") + else() + message(FATAL_ERROR "Invalid PART_TYPE") + endif() + + list(JOIN ARG_NAMES _ names_joined) + string(REPLACE / _ names_joined "${names_joined}") + string(REPLACE . _ names_joined "${names_joined}") + set(mangled_name "${variable}_${names_joined}") + + set(find_command_args + "${mangled_name}" + NAMES ${ARG_NAMES} + PATH_SUFFIXES ${ARG_PATH_SUFFIXES} + PATHS ${ARG_PATHS} + ) + if("${ARG_PART_TYPE}" STREQUAL "library") + find_library(${find_command_args}) + elseif("${ARG_PART_TYPE}" STREQUAL "path") + find_path(${find_command_args}) + elseif("${ARG_PART_TYPE}" STREQUAL "program") + find_program(${find_command_args}) + else() + message(FATAL_ERROR "Invalid PART_TYPE") + endif() + list(APPEND "${variable}" "${${mangled_name}}") + + unset(variable) + unset(names_joined) + unset(mangled_name) + unset(find_command_args) +endmacro() + +macro(_userver_module_find_library) + set(options) + set(oneValueArgs) + set(multiValueArgs + NAMES + PATHS + PATH_SUFFIXES + ) + + cmake_parse_arguments( + ARG "${options}" "${oneValueArgs}" "${multiValueArgs}" "${ARGN}") + + _userver_module_find_part( + PART_TYPE library + NAMES ${ARG_NAMES} + PATHS ${ARG_PATHS} + PATH_SUFFIXES ${ARG_PATH_SUFFIXES} + ) +endmacro() + +macro(_userver_module_find_include) + set(options) + set(oneValueArgs) + set(multiValueArgs + NAMES + PATHS + PATH_SUFFIXES + ) + + cmake_parse_arguments( + ARG "${options}" "${oneValueArgs}" "${multiValueArgs}" "${ARGN}") + + _userver_module_find_part( + PART_TYPE path + NAMES ${ARG_NAMES} + PATHS ${ARG_PATHS} + PATH_SUFFIXES ${ARG_PATH_SUFFIXES} + ) +endmacro() + +macro(_userver_module_find_program) + set(options) + set(oneValueArgs) + set(multiValueArgs + NAMES + PATHS + PATH_SUFFIXES + ) + + cmake_parse_arguments( + ARG "${options}" "${oneValueArgs}" "${multiValueArgs}" "${ARGN}") + + _userver_module_find_part( + PART_TYPE program + NAMES ${ARG_NAMES} + PATHS ${ARG_PATHS} + PATH_SUFFIXES ${ARG_PATH_SUFFIXES} + ) +endmacro() + +macro(_userver_module_end) + # Uses ARGs left over from _userver_find_module_begin + + if(${ARG_NAME}_SKIP_USERVER_FIND) + return() + endif() + + include(FindPackageHandleStandardArgs) + include(DetectVersion) + + set(name "${ARG_NAME}") + if(ARG_PACKAGE_NAME) + set(current_package_name "${ARG_PACKAGE_NAME}") + else() + set(current_package_name "${ARG_NAME}") + endif() + set(libraries_variable "${ARG_NAME}_LIBRARIES") + set(includes_variable "${ARG_NAME}_INCLUDE_DIRS") + set(programs_variable "${ARG_NAME}_EXECUTABLE") + + if(${name}_VERSION) + set("${current_package_name}_VERSION" "${${name}_VERSION}") + endif() + + # If it is possible detect the version as the dependent CMake files may rely on it + if(NOT ${current_package_name}_VERSION) + if(ARG_DEBIAN_NAMES AND UNIX AND NOT APPLE) + list(GET ARG_DEBIAN_NAMES 0 first_package_name) + deb_version("${current_package_name}_VERSION" "${first_package_name}") + endif() + + if(ARG_RPM_NAMES AND UNIX AND NOT APPLE) + list(GET ARG_RPM_NAMES 0 first_package_name) + rpm_version("${current_package_name}_VERSION" "${first_package_name}") + endif() + + if(ARG_PACMAN_NAMES AND UNIX AND NOT APPLE) + list(GET ARG_PACMAN_NAMES 0 first_package_name) + pacman_version("${current_package_name}_VERSION" "${first_package_name}") + endif() + + if(ARG_PKG_NAMES AND UNIX AND NOT APPLE) + list(GET ARG_PKG_NAMES 0 first_package_name) + pkg_version("${current_package_name}_VERSION" "${first_package_name}") + endif() + + if(ARG_FORMULA_NAMES AND APPLE) + list(GET ARG_FORMULA_NAMES 0 first_package_name) + brew_version("${current_package_name}_VERSION" "${first_package_name}") + endif() + + if(ARG_PKG_CONFIG_NAMES) + list(GET ARG_PKG_CONFIG_NAMES 0 first_package_name) + pkg_config_version("${current_package_name}_VERSION" "${first_package_name}") + endif() + endif() + + set(required_vars) + # Important to check this way, because NOTFOUND still means that this + # kind of dependencies exist. + if(NOT "${${libraries_variable}}" STREQUAL "") + list(APPEND required_vars "${libraries_variable}") + endif() + if(NOT "${${includes_variable}}" STREQUAL "") + list(APPEND required_vars "${includes_variable}") + endif() + if(NOT "${${programs_variable}}" STREQUAL "") + list(APPEND required_vars "${programs_variable}") + endif() + if(required_vars) + find_package_handle_standard_args( + "${current_package_name}" + REQUIRED_VARS + ${required_vars} + FAIL_MESSAGE + "${FULL_ERROR_MESSAGE}" + ) + mark_as_advanced(${required_vars}) + else() + # Forward to another CMake module, add nice error messages if missing. + if(ARG_COMMON_NAME) + set(wrapped_package_name "${ARG_COMMON_NAME}") + else() + set(wrapped_package_name "${current_package_name}") + endif() + set(find_command_args "${wrapped_package_name}") + if(ARG_VERSION) + list(APPEND find_command_args "${ARG_VERSION}") + endif() + if(ARG_COMMON_NAME) + list(APPEND find_command_args COMPONENTS "${name}") + endif() + find_package(${find_command_args}) + set("${name}_FOUND" "${${wrapped_package_name}_FOUND}") + endif() + + if(NOT ${name}_FOUND) + if(${name}_FIND_REQUIRED) + message(FATAL_ERROR "${FULL_ERROR_MESSAGE}. Required version is at least ${${name}_FIND_VERSION}") + endif() + return() + endif() + + if(${name}_FIND_VERSION) + if("${${current_package_name}_VERSION}" VERSION_LESS "${${name}_FIND_VERSION}") + message(STATUS + "Version of ${name} is ${${current_package_name}_VERSION}. " + "Required version is at least ${${name}_FIND_VERSION}. " + "Ignoring found ${name}. " + "Note: Set -DUSERVER_CHECK_PACKAGE_VERSIONS=0 to skip package " + "version checks if the package is fine." + ) + set("${name}_FOUND" OFF) + return() + endif() + endif() + + if(ARG_COMMON_NAME + OR (NOT "${${libraries_variable}}" STREQUAL "") + OR (NOT "${${includes_variable}}" STREQUAL "")) + if(NOT TARGET "${name}") + add_library("${name}" INTERFACE IMPORTED GLOBAL) + + if(ARG_COMMON_NAME AND TARGET "${ARG_COMMON_NAME}::${name}") + target_link_libraries("${name}" INTERFACE "${ARG_COMMON_NAME}::${name}") + endif() + + if(NOT "${${includes_variable}}" STREQUAL "") + target_include_directories("${name}" INTERFACE ${${includes_variable}}) + message(STATUS "${name} include directories: ${${includes_variable}}") + endif() + + if(NOT "${${libraries_variable}}" STREQUAL "") + target_link_libraries("${name}" INTERFACE ${${libraries_variable}}) + message(STATUS "${name} libraries: ${${libraries_variable}}") + endif() + endif() + + if(${current_package_name}_VERSION) + set("${name}_VERSION" "${${current_package_name}_VERSION}" CACHE STRING "Version of the ${name}") + endif() + endif() +endmacro() diff --git a/cmake/UserverSetupEnvironment.cmake b/cmake/UserverSetupEnvironment.cmake index 96deb2300aa4..ff0bfbcf8bf0 100644 --- a/cmake/UserverSetupEnvironment.cmake +++ b/cmake/UserverSetupEnvironment.cmake @@ -33,8 +33,8 @@ function(_userver_setup_environment_impl) set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${USERVER_CMAKE_DIR}" + "${USERVER_CMAKE_DIR}/modules" "${CMAKE_BINARY_DIR}" - "${CMAKE_BINARY_DIR}/cmake_generated" PARENT_SCOPE ) set(CMAKE_PREFIX_PATH diff --git a/cmake/install/Config.cmake.in b/cmake/install/Config.cmake.in index 05e544e0b240..838d28329bd2 100644 --- a/cmake/install/Config.cmake.in +++ b/cmake/install/Config.cmake.in @@ -40,10 +40,8 @@ set(USERVER_TESTSUITE_DIR "${USERVER_CMAKE_DIR}/testsuite") set(USERVER_IMPL_ORIGINAL_CXX_STANDARD @CMAKE_CXX_STANDARD@) set(USERVER_IMPL_FEATURE_JEMALLOC @USERVER_FEATURE_JEMALLOC@) -set(CMAKE_MODULE_PATH - ${CMAKE_MODULE_PATH} - "${USERVER_CMAKE_DIR}" -) +list(APPEND CMAKE_MODULE_PATH "${USERVER_CMAKE_DIR}") +include("${USERVER_CMAKE_DIR}/ModuleHelpers.cmake") foreach(ITEM_COMPONENT IN LISTS userver_FIND_COMPONENTS) include("${USERVER_CMAKE_DIR}/userver-${ITEM_COMPONENT}-config.cmake") diff --git a/cmake/install/userver-chaotic-config.cmake b/cmake/install/userver-chaotic-config.cmake index 798fefdaee99..8779e8aab2fd 100644 --- a/cmake/install/userver-chaotic-config.cmake +++ b/cmake/install/userver-chaotic-config.cmake @@ -8,8 +8,6 @@ find_package(userver REQUIRED COMPONENTS core ) -list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/..") - set_property(GLOBAL PROPERTY userver_chaotic_extra_args "-I ${CMAKE_CURRENT_LIST_DIR}/../../../include") set(USERVER_CHAOTIC_SCRIPTS_PATH "${USERVER_CMAKE_DIR}/chaotic") include(ChaoticGen) diff --git a/cmake/install/userver-clickhouse-config.cmake b/cmake/install/userver-clickhouse-config.cmake index 773152135817..1c8df1c204d9 100644 --- a/cmake/install/userver-clickhouse-config.cmake +++ b/cmake/install/userver-clickhouse-config.cmake @@ -8,7 +8,6 @@ find_package(userver REQUIRED COMPONENTS core ) -list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/..") include("${USERVER_CMAKE_DIR}/Findclickhouse-cpp.cmake") set(userver_clickhouse_FOUND TRUE) diff --git a/cmake/install/userver-core-config.cmake b/cmake/install/userver-core-config.cmake index e15c08c9eb0f..ef374d60345d 100644 --- a/cmake/install/userver-core-config.cmake +++ b/cmake/install/userver-core-config.cmake @@ -14,9 +14,7 @@ find_package(Boost REQUIRED COMPONENTS ) find_package(CURL "7.68" REQUIRED) -find_package(ZLIB REQUIRED) - -list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/..") +find_package(ZLIB REQUIRED) find_package(Nghttp2 REQUIRED) find_package(LibEv REQUIRED) diff --git a/cmake/install/userver-grpc-config.cmake b/cmake/install/userver-grpc-config.cmake index cd97b4a36d6e..7d0aca039917 100644 --- a/cmake/install/userver-grpc-config.cmake +++ b/cmake/install/userver-grpc-config.cmake @@ -10,7 +10,6 @@ find_package(userver REQUIRED COMPONENTS set(USERVER_GRPC_SCRIPTS_PATH "${USERVER_CMAKE_DIR}/grpc") -list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/..") include("${USERVER_CMAKE_DIR}/GrpcTargets.cmake") include_directories(${USERVER_CMAKE_DIR}/proto_generated/proto) diff --git a/cmake/install/userver-kafka-config.cmake b/cmake/install/userver-kafka-config.cmake index 1a5dd11c33ae..c1cca2b19fd2 100644 --- a/cmake/install/userver-kafka-config.cmake +++ b/cmake/install/userver-kafka-config.cmake @@ -8,7 +8,6 @@ find_package(userver REQUIRED COMPONENTS core ) -list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/..") include("${USERVER_CMAKE_DIR}/FindRdKafka.cmake") set(userver_kafka_FOUND TRUE) diff --git a/cmake/install/userver-mongo-config.cmake b/cmake/install/userver-mongo-config.cmake index ddb253541a91..1be38559f93c 100644 --- a/cmake/install/userver-mongo-config.cmake +++ b/cmake/install/userver-mongo-config.cmake @@ -8,7 +8,6 @@ find_package(userver REQUIRED COMPONENTS core ) -list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/..") include("${USERVER_CMAKE_DIR}/Findbson.cmake") include("${USERVER_CMAKE_DIR}/Findmongoc.cmake") diff --git a/cmake/install/userver-mysql-config.cmake b/cmake/install/userver-mysql-config.cmake index 1043f2a15cc5..b4f67384daa8 100644 --- a/cmake/install/userver-mysql-config.cmake +++ b/cmake/install/userver-mysql-config.cmake @@ -8,7 +8,6 @@ find_package(userver REQUIRED COMPONENTS core ) -list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/..") include("${USERVER_CMAKE_DIR}/Findlibmariadb.cmake") set(userver_mysql_FOUND TRUE) diff --git a/cmake/install/userver-otlp-config.cmake b/cmake/install/userver-otlp-config.cmake index bc4d6c09995c..88e32f053347 100644 --- a/cmake/install/userver-otlp-config.cmake +++ b/cmake/install/userver-otlp-config.cmake @@ -8,6 +8,4 @@ find_package(userver REQUIRED COMPONENTS core ) -list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/..") - set(userver_otlp_FOUND TRUE) diff --git a/cmake/install/userver-rabbitmq-config.cmake b/cmake/install/userver-rabbitmq-config.cmake index 9e976e01d792..81e86365c836 100644 --- a/cmake/install/userver-rabbitmq-config.cmake +++ b/cmake/install/userver-rabbitmq-config.cmake @@ -8,7 +8,6 @@ find_package(userver REQUIRED COMPONENTS core ) -list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/..") include("${USERVER_CMAKE_DIR}/SetupAmqpCPP.cmake") set(userver_rabbitmq_FOUND TRUE) diff --git a/cmake/install/userver-redis-config.cmake b/cmake/install/userver-redis-config.cmake index 0b221be55340..44724380a926 100644 --- a/cmake/install/userver-redis-config.cmake +++ b/cmake/install/userver-redis-config.cmake @@ -8,7 +8,6 @@ find_package(userver REQUIRED COMPONENTS core ) -list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/..") include("${USERVER_CMAKE_DIR}/FindHiredis.cmake") set(userver_redis_FOUND TRUE) diff --git a/cmake/install/userver-universal-config.cmake b/cmake/install/userver-universal-config.cmake index e06dbd951fe4..832a61259b97 100644 --- a/cmake/install/userver-universal-config.cmake +++ b/cmake/install/userver-universal-config.cmake @@ -21,7 +21,6 @@ if(NOT TARGET fmt) add_library(fmt ALIAS fmt::fmt) endif() -list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/..") find_package(cctz REQUIRED) find_package(CryptoPP REQUIRED) find_package(libyamlcpp REQUIRED) diff --git a/cmake/install/userver-ydb-config.cmake b/cmake/install/userver-ydb-config.cmake index e642309f3fda..10e56b9835e5 100644 --- a/cmake/install/userver-ydb-config.cmake +++ b/cmake/install/userver-ydb-config.cmake @@ -9,7 +9,6 @@ find_package(userver REQUIRED COMPONENTS core ) -list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/..") # include("${USERVER_CMAKE_DIR}/ydb-cpp-sdk.cmake") set(userver_ydb_FOUND TRUE) diff --git a/cmake/modules/FindBrotli.cmake b/cmake/modules/FindBrotli.cmake new file mode 100644 index 000000000000..84639052621f --- /dev/null +++ b/cmake/modules/FindBrotli.cmake @@ -0,0 +1,24 @@ +_userver_module_begin( + NAME Brotli + DEBIAN_NAMES libbrotli-dev + FORMULA_NAMES brotli + PACMAN_NAMES brotli +) + +_userver_module_find_include( + NAMES brotli/decode.h +) + +_userver_module_find_include( + NAMES brotli/encode.h +) + +_userver_module_find_library( + NAMES brotlidec +) + +_userver_module_find_library( + NAMES brotlienc +) + +_userver_module_end() diff --git a/cmake/modules/FindCrypto.cmake b/cmake/modules/FindCrypto.cmake new file mode 100644 index 000000000000..3ea31c4cc6af --- /dev/null +++ b/cmake/modules/FindCrypto.cmake @@ -0,0 +1,12 @@ +_userver_module_begin( + NAME Crypto + COMMON_NAME OpenSSL + PACKAGE_NAME OpenSSL + DEBIAN_NAMES libssl-dev + FORMULA_NAMES openssl + RPM_NAMES openssl-devel + PACMAN_NAMES openssl + PKG_CONFIG_NAMES openssl +) + +_userver_module_end() diff --git a/cmake/modules/FindCryptoPP.cmake b/cmake/modules/FindCryptoPP.cmake new file mode 100644 index 000000000000..ce8148fb41c0 --- /dev/null +++ b/cmake/modules/FindCryptoPP.cmake @@ -0,0 +1,20 @@ +_userver_module_begin( + NAME CryptoPP + DEBIAN_NAMES libcrypto++-dev + FORMULA_NAMES cryptopp + RPM_NAMES cryptopp-devel + PACMAN_NAMES crypto++ + PKG_CONFIG_NAMES libcrypto++ +) + +_userver_module_find_include( + NAMES cryptopp/cryptlib.h + PATH_SUFFIXES include +) + +_userver_module_find_library( + NAMES cryptlib cryptopp + PATH_SUFFIXES lib +) + +_userver_module_end() diff --git a/cmake/modules/FindGit.cmake b/cmake/modules/FindGit.cmake new file mode 100644 index 000000000000..cfae4ce9dc88 --- /dev/null +++ b/cmake/modules/FindGit.cmake @@ -0,0 +1,10 @@ +_userver_module_begin( + NAME Git +) + +_userver_module_find_program( + NAMES git eg + PATH_SUFFIXES Git/cmd Git/bin +) + +_userver_module_end() diff --git a/cmake/modules/FindGrpcChannelz.cmake b/cmake/modules/FindGrpcChannelz.cmake new file mode 100644 index 000000000000..675ab3fc7f30 --- /dev/null +++ b/cmake/modules/FindGrpcChannelz.cmake @@ -0,0 +1,17 @@ +_userver_module_begin( + NAME GrpcChannelz + DEBIAN_NAMES libgrpc-dev libgrpc++-dev libgrpc++1 protobuf-compiler-grpc + FORMULA_NAMES grpc + PACMAN_NAMES grpc + PKG_CONFIG_NAMES grpc++ +) + +_userver_module_find_include( + NAMES grpc/grpc.h +) + +_userver_module_find_library( + NAMES grpcpp_channelz +) + +_userver_module_end() diff --git a/cmake/modules/FindGssApi.cmake b/cmake/modules/FindGssApi.cmake new file mode 100644 index 000000000000..600984cb644a --- /dev/null +++ b/cmake/modules/FindGssApi.cmake @@ -0,0 +1,17 @@ +_userver_module_begin( + NAME GssApi + DEBIAN_NAMES libkrb5-dev + FORMULA_NAMES krb5 + PACMAN_NAMES krb5 +) + +_userver_module_find_include( + NAMES gssapi.h + PATH_SUFFIXES gssapi +) + +_userver_module_find_library( + NAMES gssapi_krb5 gssapi +) + +_userver_module_end() diff --git a/cmake/modules/FindHiredis.cmake b/cmake/modules/FindHiredis.cmake new file mode 100644 index 000000000000..1169245166cf --- /dev/null +++ b/cmake/modules/FindHiredis.cmake @@ -0,0 +1,19 @@ +_userver_module_begin( + NAME Hiredis + VERSION 0.13.3 + DEBIAN_NAMES libhiredis-dev + FORMULA_NAMES hiredis + RPM_NAMES hiredis-devel + PACMAN_NAMES hiredis + PKG_NAMES hiredis +) + +_userver_module_find_include( + NAMES hiredis/hiredis.h +) + +_userver_module_find_library( + NAMES hiredis +) + +_userver_module_end() diff --git a/cmake/modules/FindJemalloc.cmake b/cmake/modules/FindJemalloc.cmake new file mode 100644 index 000000000000..eb1ab5c49c22 --- /dev/null +++ b/cmake/modules/FindJemalloc.cmake @@ -0,0 +1,25 @@ +if(USERVER_SANITIZE) + message(FATAL_ERROR + "jemalloc is not compatible with sanitizers, " + "please skip it for USERVER_SANITIZE-enabled builds" + ) +endif() + +_userver_module_begin( + NAME Jemalloc + DEBIAN_NAMES libjemalloc-dev + FORMULA_NAMES jemalloc + RPM_NAMES jemalloc-devel + PACMAN_NAMES jemalloc + PKG_CONFIG_NAMES jemalloc +) + +_userver_module_find_include( + NAMES jemalloc/jemalloc.h +) + +_userver_module_find_library( + NAMES jemalloc +) + +_userver_module_end() diff --git a/cmake/modules/FindLibEv.cmake b/cmake/modules/FindLibEv.cmake new file mode 100644 index 000000000000..e25cbf40b3e0 --- /dev/null +++ b/cmake/modules/FindLibEv.cmake @@ -0,0 +1,17 @@ +_userver_module_begin( + NAME LibEv + DEBIAN_NAMES libev-dev + FORMULA_NAMES libev + RPM_NAMES libev-devel + PACMAN_NAMES libev +) + +_userver_module_find_include( + NAMES ev.h +) + +_userver_module_find_library( + NAMES ev +) + +_userver_module_end() diff --git a/cmake/modules/FindNghttp2.cmake b/cmake/modules/FindNghttp2.cmake new file mode 100644 index 000000000000..268463e7b993 --- /dev/null +++ b/cmake/modules/FindNghttp2.cmake @@ -0,0 +1,16 @@ +_userver_module_begin( + NAME Nghttp2 + DEBIAN_NAMES libnghttp2-dev + FORMULA_NAMES nghttp2 + PACMAN_NAMES libnghttp2 +) + +_userver_module_find_include( + NAMES nghttp2/nghttp2.h +) + +_userver_module_find_library( + NAMES nghttp2 +) + +_userver_module_end() diff --git a/cmake/modules/FindPostgreSQLInternal.cmake b/cmake/modules/FindPostgreSQLInternal.cmake new file mode 100644 index 000000000000..747f0962a305 --- /dev/null +++ b/cmake/modules/FindPostgreSQLInternal.cmake @@ -0,0 +1,88 @@ +_userver_module_begin( + NAME PostgreSQLInternal + DEBIAN_NAMES libpq-dev postgresql-server-dev-16 + FORMULA_NAMES postgresql@16 + RPM_NAMES postgresql-server-devel postgresql-static + PKG_NAMES postgresql16-server +) + +_userver_module_find_include( + NAMES postgres_fe.h + PATHS + ${USERVER_PG_SERVER_INCLUDE_DIR} + /usr/include/postgresql/12/server + /usr/include/postgresql/13/server + /usr/include/postgresql/14/server + /usr/include/postgresql/15/server + /usr/include/postgresql/16/server + /usr/include/postgresql/17/server + /usr/include/postgresql/18/server + /usr/include/postgresql/19/server + /usr/include/postgresql/20/server + /usr/local/include/postgresql/server # FreeBSD + /usr/include/postgresql/server # Manjaro + PATH_SUFFIXES + pgsql/server # postgresql-server-devel +) + +_userver_module_find_include( + NAMES libpq-int.h + PATHS + ${USERVER_PG_INCLUDE_DIR}/internal + ${USERVER_PG_INCLUDE_DIR}/postgresql/internal + /usr/local/include/postgresql/internal # FreeBSD + PATH_SUFFIXES + postgresql/internal # libpq-dev + pgsql/internal # postgresql-private-devel +) + +_userver_module_find_include( + NAMES libpq-fe.h + PATHS + ${USERVER_PG_INCLUDE_DIR} + /usr/local/include # FreeBSD + PATH_SUFFIXES + postgresql + pgsql +) + +_userver_module_find_library( + NAMES libpq.a # must be a static library as we use internal symbols + PATHS + ${USERVER_PG_LIBRARY_DIR} + /usr/local/lib # FreeBSD +) + +_userver_module_find_library( + NAMES libpgcommon.a + PATHS + ${USERVER_PG_SERVER_LIBRARY_DIR} + ${USERVER_PG_LIBRARY_DIR} + /usr/lib/postgresql/12/lib + /usr/lib/postgresql/13/lib + /usr/lib/postgresql/14/lib + /usr/lib/postgresql/15/lib + /usr/lib/postgresql/16/lib + /usr/lib/postgresql/17/lib + /usr/lib/postgresql/18/lib + /usr/lib/postgresql/19/lib + /usr/lib/postgresql/20/lib +) + +_userver_module_find_library( + NAMES libpgport.a + PATHS + ${USERVER_PG_SERVER_LIBRARY_DIR} + ${USERVER_PG_LIBRARY_DIR} + /usr/lib/postgresql/12/lib + /usr/lib/postgresql/13/lib + /usr/lib/postgresql/14/lib + /usr/lib/postgresql/15/lib + /usr/lib/postgresql/16/lib + /usr/lib/postgresql/17/lib + /usr/lib/postgresql/18/lib + /usr/lib/postgresql/19/lib + /usr/lib/postgresql/20/lib +) + +_userver_module_end() diff --git a/cmake/modules/FindPythonDev.cmake b/cmake/modules/FindPythonDev.cmake new file mode 100644 index 000000000000..823c2575a0fe --- /dev/null +++ b/cmake/modules/FindPythonDev.cmake @@ -0,0 +1,29 @@ +_userver_module_begin( + NAME PythonDev + DEBIAN_NAMES python3-dev + FORMULA_NAMES python3-dev + RPM_NAMES python3-dev + PACMAN_NAMES python +) + +_userver_module_find_include( + NAMES Python.h + PATHS + /usr/include/python3m + /usr/include/python3.6m + /usr/include/python3.7m + /usr/include/python3.8m + /usr/include/python3.9m + /usr/include/python3 + /usr/include/python3.6 + /usr/include/python3.7 + /usr/include/python3.8 + /usr/include/python3.9 + /usr/include/python3.10 + /usr/include/python3.11 + /usr/include/python3.12 + /usr/include/python3.13 + /usr/include/python3.14 +) + +_userver_module_end() diff --git a/cmake/modules/FindRdKafka.cmake b/cmake/modules/FindRdKafka.cmake new file mode 100644 index 000000000000..1d49074e268f --- /dev/null +++ b/cmake/modules/FindRdKafka.cmake @@ -0,0 +1,28 @@ +set(RDKAFKA_BUILD_STATIC ON) +set(RDKAFKA_BUILD_EXAMPLES OFF) +set(RDKAFKA_BUILD_TESTS OFF) +set(WITH_SSL ON) +set(WITH_SASL ON) +set(WITH_ZLIB OFF) +set(WITH_ZSTD OFF) +set(WITH_LIBDL OFF) +set(ENABLE_LZ4_EXT OFF) + +_userver_module_begin( + NAME RdKafka + VERSION 2.4.0 + DEBIAN_NAMES librdkafka-dev + FORMULA_NAMES librdkafka + PACMAN_NAMES librdkafka + PKG_CONFIG_NAMES rdkafka +) + +_userver_module_find_include( + NAMES librdkafka/rdkafka.h +) + +_userver_module_find_library( + NAMES librdkafka.a +) + +_userver_module_end() diff --git a/cmake/modules/FindSSL.cmake b/cmake/modules/FindSSL.cmake new file mode 100644 index 000000000000..c590d78e36e2 --- /dev/null +++ b/cmake/modules/FindSSL.cmake @@ -0,0 +1,12 @@ +_userver_module_begin( + NAME SSL + COMMON_NAME OpenSSL + PACKAGE_NAME OpenSSL + DEBIAN_NAMES libssl-dev + FORMULA_NAMES openssl + RPM_NAMES openssl-devel + PACMAN_NAMES openssl + PKG_CONFIG_NAMES libssl +) + +_userver_module_end() diff --git a/cmake/modules/FindUserverGBench.cmake b/cmake/modules/FindUserverGBench.cmake new file mode 100644 index 000000000000..a656e7e67198 --- /dev/null +++ b/cmake/modules/FindUserverGBench.cmake @@ -0,0 +1,23 @@ +_userver_module_begin( + NAME UserverGBench + VERSION 1.5 # libbenchmark_main.a appeared after 1.3 + DEBIAN_NAMES libbenchmark-dev + FORMULA_NAMES google-benchmark + RPM_NAMES google-benchmark-devel + PACMAN_NAMES benchmark + PKG_CONFIG_NAMES benchmark +) + +_userver_module_find_include( + NAMES benchmark/benchmark.h +) + +_userver_module_find_library( + NAMES benchmark_main +) + +_userver_module_find_library( + NAMES benchmark +) + +_userver_module_end() diff --git a/cmake/modules/FindUserverGrpc.cmake b/cmake/modules/FindUserverGrpc.cmake new file mode 100644 index 000000000000..63d2a8a24d37 --- /dev/null +++ b/cmake/modules/FindUserverGrpc.cmake @@ -0,0 +1,32 @@ +_userver_module_begin( + NAME UserverGrpc + DEBIAN_NAMES libgrpc-dev libgrpc++-dev libgrpc++1 protobuf-compiler-grpc + FORMULA_NAMES grpc + PACMAN_NAMES grpc + PKG_CONFIG_NAMES grpc++ +) + +_userver_module_find_include( + NAMES grpc/grpc.h +) + +_userver_module_find_library( + NAMES grpc +) + +_userver_module_find_library( + NAMES grpc++ +) + +_userver_module_find_library( + NAMES gpr +) + +_userver_module_find_library( + NAMES + absl + absl_synchronization + grpc # fallback, old versions of gRPC do not link with absl +) + +_userver_module_end() diff --git a/cmake/modules/Findbson.cmake b/cmake/modules/Findbson.cmake new file mode 100644 index 000000000000..22f4b0f43b3d --- /dev/null +++ b/cmake/modules/Findbson.cmake @@ -0,0 +1,21 @@ +_userver_module_begin( + NAME bson + VERSION 1.16.0 + DEBIAN_NAMES libmongoc-dev + FORMULA_NAMES mongo-c-driver + RPM_NAMES mongo-c-driver-devel + PACMAN_NAMES mongo-c-driver +) + +_userver_module_find_include( + NAMES bson/bson.h + PATHS + /usr/include/libbson-1.0 + /usr/local/opt/mongo-c-driver/include/libbson-1.0 +) + +_userver_module_find_library( + NAMES bson bson-1.0 +) + +_userver_module_end() diff --git a/cmake/modules/Findc-ares.cmake b/cmake/modules/Findc-ares.cmake new file mode 100644 index 000000000000..6bcb20437bc2 --- /dev/null +++ b/cmake/modules/Findc-ares.cmake @@ -0,0 +1,19 @@ +_userver_module_begin( + NAME c-ares + VERSION 1.16.0 # ares_getaddrinfo appeared only in 1.16 + DEBIAN_NAMES libc-ares-dev + FORMULA_NAMES c-ares + RPM_NAMES c-ares-devel + PACMAN_NAMES c-ares + PKG_CONFIG_NAMES libcares +) + +_userver_module_find_include( + NAMES ares.h +) + +_userver_module_find_library( + NAMES cares +) + +_userver_module_end() diff --git a/cmake/modules/Findcctz.cmake b/cmake/modules/Findcctz.cmake new file mode 100644 index 000000000000..001b31377217 --- /dev/null +++ b/cmake/modules/Findcctz.cmake @@ -0,0 +1,19 @@ +_userver_module_begin( + NAME cctz + VERSION 2.3 + DEBIAN_NAMES libcctz-dev + FORMULA_NAMES cctz + RPM_NAMES cctz-devel + PACMAN_NAMES cctz +) + +_userver_module_find_include( + NAMES cctz/civil_time.h +) + +_userver_module_find_library( + NAMES cctz + PATHS /usr/lib/x86_64-linux-gnu +) + +_userver_module_end() diff --git a/cmake/modules/Findclickhouse-cpp.cmake b/cmake/modules/Findclickhouse-cpp.cmake new file mode 100644 index 000000000000..617c0dc2b802 --- /dev/null +++ b/cmake/modules/Findclickhouse-cpp.cmake @@ -0,0 +1,16 @@ +_userver_module_begin( + NAME clickhouse-cpp +) + +_userver_module_find_include( + NAMES clickhouse/block.h + PATH_SUFFIXES + clickhouse-cpp + yandex/clickhouse-cpp +) + +_userver_module_find_library( + NAMES libclickhouse-cpp-lib.so +) + +_userver_module_end() diff --git a/cmake/modules/Findfmt.cmake b/cmake/modules/Findfmt.cmake new file mode 100644 index 000000000000..ebba6fc9dd3d --- /dev/null +++ b/cmake/modules/Findfmt.cmake @@ -0,0 +1,19 @@ +_userver_module_begin( + NAME fmt + VERSION 7.1 + DEBIAN_NAMES libfmt-dev + FORMULA_NAMES fmt + RPM_NAMES fmt-devel + PACMAN_NAMES fmt + PKG_CONFIG_NAMES fmt +) + +_userver_module_find_include( + NAMES fmt/format.h +) + +_userver_module_find_library( + NAMES fmt +) + +_userver_module_end() diff --git a/cmake/modules/Findlibgflags.cmake b/cmake/modules/Findlibgflags.cmake new file mode 100644 index 000000000000..64ede3830a0e --- /dev/null +++ b/cmake/modules/Findlibgflags.cmake @@ -0,0 +1,23 @@ +_userver_module_begin( + NAME libgflags + DEBIAN_NAMES libgflags-dev + FORMULA_NAMES gflags + RPM_NAMES libgflags-dev + PACMAN_NAMES gflags +) + +_userver_module_find_include( + NAMES + gflags/gflags.h + gflags/gflags_completions.h + gflags/gflags_declare.h + gflags/gflags_gflags.h + PATH_SUFFIXES include +) + +_userver_module_find_library( + NAMES gflags + PATH_SUFFIXES lib +) + +_userver_module_end() diff --git a/cmake/modules/Findlibintl.cmake b/cmake/modules/Findlibintl.cmake new file mode 100644 index 000000000000..6f02d1f1601c --- /dev/null +++ b/cmake/modules/Findlibintl.cmake @@ -0,0 +1,14 @@ +_userver_module_begin( + NAME libintl +) + +_userver_module_find_include( + NAMES stdio.h +) + +_userver_module_find_library( + NAMES libintl.a # FreeBSD + PATHS /usr/local/lib +) + +_userver_module_end() diff --git a/cmake/modules/Findlibmariadb.cmake b/cmake/modules/Findlibmariadb.cmake new file mode 100644 index 000000000000..0b38409a136c --- /dev/null +++ b/cmake/modules/Findlibmariadb.cmake @@ -0,0 +1,16 @@ +_userver_module_begin( + NAME libmariadb + VERSION 3.0.3 + DEBIAN_NAMES libmariadb-dev +) + +_userver_module_find_include( + NAMES mariadb/mysql.h +) + +_userver_module_find_library( + NAMES mariadb + PATH_SUFFIXES lib +) + +_userver_module_end() diff --git a/cmake/modules/Findlibsnappy.cmake b/cmake/modules/Findlibsnappy.cmake new file mode 100644 index 000000000000..225685291c2b --- /dev/null +++ b/cmake/modules/Findlibsnappy.cmake @@ -0,0 +1,23 @@ +_userver_module_begin( + NAME libsnappy + DEBIAN_NAMES libsnappy-dev + FORMULA_NAMES snappy + RPM_NAMES libsnappy-dev + PACMAN_NAMES snappy +) + +_userver_module_find_include( + NAMES + snappy-c.h + snappy-sinksource.h + snappy-stubs-public.h + snappy.h + PATH_SUFFIXES include +) + +_userver_module_find_library( + NAMES snappy + PATH_SUFFIXES lib +) + +_userver_module_end() diff --git a/cmake/modules/Findlibyamlcpp.cmake b/cmake/modules/Findlibyamlcpp.cmake new file mode 100644 index 000000000000..3c12d660ed3c --- /dev/null +++ b/cmake/modules/Findlibyamlcpp.cmake @@ -0,0 +1,20 @@ +_userver_module_begin( + NAME libyamlcpp + DEBIAN_NAMES libyaml-cpp-dev + FORMULA_NAMES yaml-cpp + PACMAN_NAMES yaml-cpp +) + +_userver_module_find_include( + NAMES + yaml-cpp/yaml.h + yaml-cpp/node.h + PATH_SUFFIXES include +) + +_userver_module_find_library( + NAMES yaml-cpp + PATH_SUFFIXES lib +) + +_userver_module_end() diff --git a/cmake/modules/Findlibz.cmake b/cmake/modules/Findlibz.cmake new file mode 100644 index 000000000000..5bc95388f061 --- /dev/null +++ b/cmake/modules/Findlibz.cmake @@ -0,0 +1,19 @@ +_userver_module_begin( + NAME libz + DEBIAN_NAMES zlib1g-dev + FORMULA_NAMES zlib + RPM_NAMES zlib1g-dev + PACMAN_NAMES zlib +) + +_userver_module_find_include( + NAMES zlib.h zconf.h + PATH_SUFFIXES include +) + +_userver_module_find_library( + NAMES zlib1g z + PATH_SUFFIXES lib +) + +_userver_module_end() diff --git a/cmake/modules/Findlibzip.cmake b/cmake/modules/Findlibzip.cmake new file mode 100644 index 000000000000..f0c04b5fa42d --- /dev/null +++ b/cmake/modules/Findlibzip.cmake @@ -0,0 +1,19 @@ +_userver_module_begin( + NAME libzip + DEBIAN_NAMES libbz2-dev + FORMULA_NAMES libbz2 + RPM_NAMES libbz2-dev + PACMAN_NAMES bzip2 +) + +_userver_module_find_include( + NAMES bzlib.h + PATH_SUFFIXES include +) + +_userver_module_find_library( + NAMES bz2 libbz2 + PATH_SUFFIXES lib +) + +_userver_module_end() diff --git a/cmake/modules/Findlibzstd.cmake b/cmake/modules/Findlibzstd.cmake new file mode 100644 index 000000000000..76b1ebeaa362 --- /dev/null +++ b/cmake/modules/Findlibzstd.cmake @@ -0,0 +1,22 @@ +_userver_module_begin( + NAME libzstd + DEBIAN_NAMES libzstd-dev + FORMULA_NAMES libzstd + RPM_NAMES libzstd-dev + PACMAN_NAMES zstd +) + +_userver_module_find_include( + NAMES + zdict.h + zstd.h + zstd_errors.h + PATH_SUFFIXES include +) + +_userver_module_find_library( + NAMES zstd + PATH_SUFFIXES lib +) + +_userver_module_end() diff --git a/cmake/modules/Findmongoc.cmake b/cmake/modules/Findmongoc.cmake new file mode 100644 index 000000000000..252c44aaefa3 --- /dev/null +++ b/cmake/modules/Findmongoc.cmake @@ -0,0 +1,21 @@ +_userver_module_begin( + NAME mongoc + VERSION 1.16.0 + DEBIAN_NAMES libmongoc-dev + FORMULA_NAMES mongo-c-driver + RPM_NAMES mongo-c-driver-devel + PACMAN_NAMES mongo-c-driver +) + +_userver_module_find_include( + NAMES mongoc/mongoc.h + PATHS + /usr/include/libmongoc-1.0 + /usr/local/opt/mongo-c/include/libmongoc-1.0 +) + +_userver_module_find_library( + NAMES mongoc mongoc-1.0 +) + +_userver_module_end() diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index efe9034ecf28..cab10587f941 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -294,9 +294,9 @@ _userver_install_targets(COMPONENT core TARGETS ${PROJECT_NAME}) _userver_directory_install(COMPONENT core FILES "${USERVER_ROOT_DIR}/cmake/UserverTestsuite.cmake" "${USERVER_ROOT_DIR}/cmake/install/userver-core-config.cmake" - "${CMAKE_BINARY_DIR}/cmake_generated/Findc-ares.cmake" - "${CMAKE_BINARY_DIR}/cmake_generated/FindNghttp2.cmake" - "${CMAKE_BINARY_DIR}/cmake_generated/FindLibEv.cmake" + "${USERVER_ROOT_DIR}/cmake/modules/Findc-ares.cmake" + "${USERVER_ROOT_DIR}/cmake/modules/FindNghttp2.cmake" + "${USERVER_ROOT_DIR}/cmake/modules/FindLibEv.cmake" DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/userver ) diff --git a/external-deps/Brotli.yaml b/external-deps/Brotli.yaml deleted file mode 100644 index 638f75c136fa..000000000000 --- a/external-deps/Brotli.yaml +++ /dev/null @@ -1,21 +0,0 @@ -name: Brotli - -debian-names: - - libbrotli-dev -formula-name: brotli -pacman-names: - - brotli - -libraries: - find: - - names: - - brotlidec - - names: - - brotlienc - -includes: - find: - - names: - - brotli/decode.h - - names: - - brotli/encode.h diff --git a/external-deps/CryptoPP.yaml b/external-deps/CryptoPP.yaml deleted file mode 100644 index 35c3a80e4764..000000000000 --- a/external-deps/CryptoPP.yaml +++ /dev/null @@ -1,26 +0,0 @@ -name: CryptoPP - -includes: - find: - - names: - - cryptopp/cryptlib.h - path-suffixes: - - include - -libraries: - find: - - names: - - cryptlib - - cryptopp - path-suffixes: - - lib - -debian-names: - - libcrypto++-dev -formula-name: cryptopp -rpm-names: - - cryptopp-devel -pacman-names: - - crypto++ -pkg-config-names: - - libcrypto++ diff --git a/external-deps/Git.yaml b/external-deps/Git.yaml deleted file mode 100644 index ce3db801b464..000000000000 --- a/external-deps/Git.yaml +++ /dev/null @@ -1,16 +0,0 @@ -name: Git - -includes: - enabled: false - -libraries: - enabled: false - -programs: - find: - - names: - - git - - eg - path-suffixes: - - Git/cmd - - Git/bin diff --git a/external-deps/GrpcChannelz.yaml b/external-deps/GrpcChannelz.yaml deleted file mode 100644 index c6d1c795893e..000000000000 --- a/external-deps/GrpcChannelz.yaml +++ /dev/null @@ -1,23 +0,0 @@ -name: GrpcChannelz - -debian-names: - - libgrpc-dev - - libgrpc++-dev - - libgrpc++1 - - protobuf-compiler-grpc - -formula-name: grpc -pacman-names: - - grpc -pkg-config-names: - - grpc++ - -libraries: - find: - - names: - - grpcpp_channelz - -includes: - find: - - names: - - grpc/grpc.h diff --git a/external-deps/GssApi.yaml b/external-deps/GssApi.yaml deleted file mode 100644 index 1bfcae119ce9..000000000000 --- a/external-deps/GssApi.yaml +++ /dev/null @@ -1,22 +0,0 @@ -name: GssApi - -debian-names: - - libkrb5-dev - -formula-name: krb5 - -pacman-names: - - krb5 - -includes: - find: - - names: - - gssapi.h - path-suffixes: - - gssapi - -libraries: - find: - - names: - - gssapi_krb5 - - gssapi diff --git a/external-deps/Hiredis.yaml b/external-deps/Hiredis.yaml deleted file mode 100644 index cef2cebcb2af..000000000000 --- a/external-deps/Hiredis.yaml +++ /dev/null @@ -1,22 +0,0 @@ -name: Hiredis -version: 0.13.3 - -includes: - find: - - names: - - hiredis/hiredis.h - -libraries: - find: - - names: - - hiredis - -debian-names: - - libhiredis-dev -formula-name: hiredis -rpm-names: - - hiredis-devel -pacman-names: - - hiredis -pkg-names: - - hiredis diff --git a/external-deps/Jemalloc.yaml b/external-deps/Jemalloc.yaml deleted file mode 100644 index 558ae8dff1a6..000000000000 --- a/external-deps/Jemalloc.yaml +++ /dev/null @@ -1,27 +0,0 @@ -name: Jemalloc - -checks: - - expression: USERVER_SANITIZE - error: >- - jemalloc is not compatible with sanitizers, please skip it for SANITIZE-enabled - builds - -debian-names: - - libjemalloc-dev -formula-name: jemalloc -rpm-names: - - jemalloc-devel -pacman-names: - - jemalloc -pkg-config-names: - - jemalloc - -libraries: - find: - - names: - - jemalloc - -includes: - find: - - names: - - jemalloc/jemalloc.h diff --git a/external-deps/LibEv.yaml b/external-deps/LibEv.yaml deleted file mode 100644 index c96e9bf75e44..000000000000 --- a/external-deps/LibEv.yaml +++ /dev/null @@ -1,19 +0,0 @@ -name: LibEv - -includes: - find: - - names: - - ev.h - -libraries: - find: - - names: - - ev - -debian-names: - - libev-dev -formula-name: libev -rpm-names: - - libev-devel -pacman-names: - - libev diff --git a/external-deps/Nghttp2.yaml b/external-deps/Nghttp2.yaml deleted file mode 100644 index ac48235b03cf..000000000000 --- a/external-deps/Nghttp2.yaml +++ /dev/null @@ -1,17 +0,0 @@ -name: Nghttp2 - -includes: - find: - - names: - - nghttp2/nghttp2.h - -libraries: - find: - - names: - - nghttp2 - -debian-names: - - libnghttp2-dev -formula-name: nghttp2 -pacman-names: - - libnghttp2 diff --git a/external-deps/OpenSSL.yaml b/external-deps/OpenSSL.yaml deleted file mode 100644 index c8f3d2babba3..000000000000 --- a/external-deps/OpenSSL.yaml +++ /dev/null @@ -1,38 +0,0 @@ -common-name: OpenSSL -partials: - - name: Crypto - package-name: OpenSSL - - debian-names: - - libssl-dev - formula-name: openssl - rpm-names: - - openssl-devel - pacman-names: - - openssl - pkg-config-names: - - openssl - - libraries: - enabled: false - - includes: - enabled: false - - name: SSL - package-name: OpenSSL - - debian-names: - - libssl-dev - formula-name: openssl - rpm-names: - - openssl-devel - pacman-names: - - openssl - pkg-config-names: - - libssl - - libraries: - enabled: false - - includes: - enabled: false diff --git a/external-deps/PostgreSQLInternal.yaml b/external-deps/PostgreSQLInternal.yaml deleted file mode 100644 index 102e763418f8..000000000000 --- a/external-deps/PostgreSQLInternal.yaml +++ /dev/null @@ -1,87 +0,0 @@ -name: PostgreSQLInternal - - -includes: - find: - - names: - - postgres_fe.h - paths: - - ${USERVER_PG_SERVER_INCLUDE_DIR} - - /usr/include/postgresql/12/server - - /usr/include/postgresql/13/server - - /usr/include/postgresql/14/server - - /usr/include/postgresql/15/server - - /usr/include/postgresql/16/server - - /usr/include/postgresql/17/server - - /usr/include/postgresql/18/server - - /usr/include/postgresql/19/server - - /usr/include/postgresql/20/server - - /usr/local/include/postgresql/server # FreeBSD - - /usr/include/postgresql/server # Manjaro - path-suffixes: - - pgsql/server # postgresql-server-devel - - - names: - - libpq-int.h - paths: - - ${USERVER_PG_INCLUDE_DIR}/internal - - ${USERVER_PG_INCLUDE_DIR}/postgresql/internal - - /usr/local/include/postgresql/internal # FreeBSD - path-suffixes: - - postgresql/internal # libpq-dev - - pgsql/internal # postgresql-private-devel - - - names: - - libpq-fe.h - paths: - - ${USERVER_PG_INCLUDE_DIR} - - /usr/local/include # FreeBSD - path-suffixes: - - postgresql - - pgsql - -libraries: - find: - - names: - - libpq.a # must be a static library as we use internal symbols - paths: - - ${USERVER_PG_LIBRARY_DIR} - - /usr/local/lib # FreeBSD - - names: - - libpgcommon.a - paths: - - ${USERVER_PG_SERVER_LIBRARY_DIR} - - ${USERVER_PG_LIBRARY_DIR} - - /usr/lib/postgresql/12/lib - - /usr/lib/postgresql/13/lib - - /usr/lib/postgresql/14/lib - - /usr/lib/postgresql/15/lib - - /usr/lib/postgresql/16/lib - - /usr/lib/postgresql/17/lib - - /usr/lib/postgresql/18/lib - - /usr/lib/postgresql/19/lib - - /usr/lib/postgresql/20/lib - - names: - - libpgport.a - paths: - - ${USERVER_PG_SERVER_LIBRARY_DIR} - - ${USERVER_PG_LIBRARY_DIR} - - /usr/lib/postgresql/12/lib - - /usr/lib/postgresql/13/lib - - /usr/lib/postgresql/14/lib - - /usr/lib/postgresql/15/lib - - /usr/lib/postgresql/16/lib - - /usr/lib/postgresql/17/lib - - /usr/lib/postgresql/18/lib - - /usr/lib/postgresql/19/lib - - /usr/lib/postgresql/20/lib - -debian-names: - - libpq-dev - - postgresql-server-dev-16 -formula-name: postgresql@16 -rpm-names: - - postgresql-server-devel - - postgresql-static -pkg-names: - - postgresql16-server diff --git a/external-deps/README.md b/external-deps/README.md deleted file mode 100644 index 41f80ee266f5..000000000000 --- a/external-deps/README.md +++ /dev/null @@ -1,4 +0,0 @@ -# userver: External Dependencies - -YAML files that describe where to find third-party headers and libraries. Those -files are used to generate CMake scripts by the `scripts/external_deps`. diff --git a/external-deps/RdKafka.yaml b/external-deps/RdKafka.yaml deleted file mode 100644 index 58408e6cda8e..000000000000 --- a/external-deps/RdKafka.yaml +++ /dev/null @@ -1,31 +0,0 @@ -name: RdKafka -version: 2.4.0 - -includes: - find: - - names: - - librdkafka/rdkafka.h - -libraries: - find: - - names: - - librdkafka.a - -debian-names: - - librdkafka-dev -formula-name: librdkafka -pkg-config-names: - - rdkafka -pacman-names: - - librdkafka - -extra-cmake-vars: - RDKAFKA_BUILD_STATIC: "ON" - RDKAFKA_BUILD_EXAMPLES: "OFF" - RDKAFKA_BUILD_TESTS: "OFF" - WITH_SSL: "ON" - WITH_SASL: "ON" - WITH_ZLIB: "OFF" - WITH_ZSTD: "OFF" - WITH_LIBDL: "OFF" - ENABLE_LZ4_EXT: "OFF" diff --git a/external-deps/UserverGBench.yaml b/external-deps/UserverGBench.yaml deleted file mode 100644 index 9f66ef32ab81..000000000000 --- a/external-deps/UserverGBench.yaml +++ /dev/null @@ -1,28 +0,0 @@ -name: UserverGBench -version: '1.5' # libbenchmark_main.a appeared after 1.3 - -debian-names: - - libbenchmark-dev - -rpm-names: - - google-benchmark-devel - -formula-name: google-benchmark - -pacman-names: - - benchmark - -pkg-config-names: - - benchmark - -libraries: - find: - - names: - - benchmark_main - - names: - - benchmark - -includes: - find: - - names: - - benchmark/benchmark.h diff --git a/external-deps/UserverGrpc.yaml b/external-deps/UserverGrpc.yaml deleted file mode 100644 index 4e3d68511c15..000000000000 --- a/external-deps/UserverGrpc.yaml +++ /dev/null @@ -1,31 +0,0 @@ -name: UserverGrpc - -debian-names: - - libgrpc-dev - - libgrpc++-dev - - libgrpc++1 - - protobuf-compiler-grpc - -formula-name: grpc -pacman-names: - - grpc -pkg-config-names: - - grpc++ - -libraries: - find: - - names: - - grpc - - names: - - grpc++ - - names: - - gpr - - names: - - absl - - absl_synchronization - - grpc # fallback, old versions of gRPC do not link with absl - -includes: - find: - - names: - - grpc/grpc.h diff --git a/external-deps/bson.yaml b/external-deps/bson.yaml deleted file mode 100644 index c4327c33522d..000000000000 --- a/external-deps/bson.yaml +++ /dev/null @@ -1,23 +0,0 @@ -name: bson -version: 1.16.0 - -libraries: - find: - - names: - - bson - - bson-1.0 -includes: - find: - - names: - - bson/bson.h - paths: - - /usr/include/libbson-1.0 - - /usr/local/opt/mongo-c-driver/include/libbson-1.0 - -debian-names: - - libmongoc-dev -formula-name: mongo-c-driver -rpm-names: - - mongo-c-driver-devel -pacman-names: - - mongo-c-driver diff --git a/external-deps/c-ares.yaml b/external-deps/c-ares.yaml deleted file mode 100644 index c08e0bc7d64a..000000000000 --- a/external-deps/c-ares.yaml +++ /dev/null @@ -1,24 +0,0 @@ -name: c-ares -version: 1.16.0 # ares_getaddrinfo appeared only in 1.16 -type: find-helper - -debian-names: - - libc-ares-dev - -formula-name: c-ares -rpm-names: - - c-ares-devel -pacman-names: - - c-ares -pkg-config-names: - - libcares - -libraries: - find: - - names: - - cares - -includes: - find: - - names: - - ares.h diff --git a/external-deps/cctz.yaml b/external-deps/cctz.yaml deleted file mode 100644 index 478fd81d5e98..000000000000 --- a/external-deps/cctz.yaml +++ /dev/null @@ -1,23 +0,0 @@ -name: cctz -version: '2.3' - -debian-names: - - libcctz-dev - -formula-name: cctz -rpm-names: - - cctz-devel -pacman-names: - - cctz - -libraries: - find: - - names: - - cctz - paths: - - /usr/lib/x86_64-linux-gnu - -includes: - find: - - names: - - cctz/civil_time.h diff --git a/external-deps/clickhouse-cpp.yaml b/external-deps/clickhouse-cpp.yaml deleted file mode 100644 index bf6c18425b7a..000000000000 --- a/external-deps/clickhouse-cpp.yaml +++ /dev/null @@ -1,14 +0,0 @@ -name: clickhouse-cpp - -libraries: - find: - - names: - - libclickhouse-cpp-lib.so - -includes: - find: - - names: - - clickhouse/block.h - path-suffixes: - - clickhouse-cpp - - yandex/clickhouse-cpp diff --git a/external-deps/fmt.yaml b/external-deps/fmt.yaml deleted file mode 100644 index ffe065daf088..000000000000 --- a/external-deps/fmt.yaml +++ /dev/null @@ -1,22 +0,0 @@ -name: fmt -version: '7.1' # on update also adjust the version in core/CMakeLists.txt - -includes: - find: - - names: - - fmt/format.h - -libraries: - find: - - names: - - fmt - -debian-names: - - libfmt-dev -formula-name: fmt -rpm-names: - - fmt-devel -pacman-names: - - fmt -pkg-config-names: - - fmt diff --git a/external-deps/libgflags.yaml b/external-deps/libgflags.yaml deleted file mode 100644 index e3ea1c5b0d68..000000000000 --- a/external-deps/libgflags.yaml +++ /dev/null @@ -1,26 +0,0 @@ -name: libgflags - -includes: - find: - - names: - - gflags/gflags.h - - gflags/gflags_completions.h - - gflags/gflags_declare.h - - gflags/gflags_gflags.h - path-suffixes: - - include - -libraries: - find: - - names: - - gflags - path-suffixes: - - lib - -debian-names: - - libgflags-dev -formula-name: gflags -rpm-names: - - libgflags-dev -pacman-names: - - gflags diff --git a/external-deps/libintl.yaml b/external-deps/libintl.yaml deleted file mode 100644 index 9a1e86c53f78..000000000000 --- a/external-deps/libintl.yaml +++ /dev/null @@ -1,12 +0,0 @@ -name: libintl - -includes: - find: - - names: - - stdio.h -libraries: - find: - - names: - - libintl.a # FreeBSD - paths: - - /usr/local/lib diff --git a/external-deps/libmariadb.yaml b/external-deps/libmariadb.yaml deleted file mode 100644 index ae90f18e0622..000000000000 --- a/external-deps/libmariadb.yaml +++ /dev/null @@ -1,17 +0,0 @@ -name: libmariadb -version: 3.0.3 - -includes: - find: - - names: - - mariadb/mysql.h - -libraries: - find: - - names: - - mariadb - path-suffixes: - - lib - -debian-names: - - libmariadb-dev diff --git a/external-deps/libsnappy.yaml b/external-deps/libsnappy.yaml deleted file mode 100644 index 590ed7dd42c3..000000000000 --- a/external-deps/libsnappy.yaml +++ /dev/null @@ -1,26 +0,0 @@ -name: libsnappy - -includes: - find: - - names: - - snappy-c.h - - snappy-sinksource.h - - snappy-stubs-public.h - - snappy.h - path-suffixes: - - include - -libraries: - find: - - names: - - snappy - path-suffixes: - - lib - -debian-names: - - libsnappy-dev -formula-name: snappy -rpm-names: - - libsnappy-dev -pacman-names: - - snappy diff --git a/external-deps/libyamlcpp.yaml b/external-deps/libyamlcpp.yaml deleted file mode 100644 index a51bf9f5348d..000000000000 --- a/external-deps/libyamlcpp.yaml +++ /dev/null @@ -1,22 +0,0 @@ -name: libyamlcpp - -includes: - find: - - names: - - yaml-cpp/yaml.h - - yaml-cpp/node.h - path-suffixes: - - include - -libraries: - find: - - names: - - yaml-cpp - path-suffixes: - - lib - -debian-names: - - libyaml-cpp-dev -formula-name: yaml-cpp -pacman-names: - - yaml-cpp diff --git a/external-deps/libz.yaml b/external-deps/libz.yaml deleted file mode 100644 index 6da7abd6d49d..000000000000 --- a/external-deps/libz.yaml +++ /dev/null @@ -1,25 +0,0 @@ -name: libz - -includes: - find: - - names: - - zlib.h - - zconf.h - path-suffixes: - - include - -libraries: - find: - - names: - - zlib1g - - z - path-suffixes: - - lib - -debian-names: - - zlib1g-dev -formula-name: zlib -rpm-names: - - zlib1g-dev -pacman-names: - - zlib diff --git a/external-deps/libzip.yaml b/external-deps/libzip.yaml deleted file mode 100644 index f6312f293df5..000000000000 --- a/external-deps/libzip.yaml +++ /dev/null @@ -1,24 +0,0 @@ -name: libzip - -includes: - find: - - names: - - bzlib.h - path-suffixes: - - include - -libraries: - find: - - names: - - bz2 - - libbz2 - path-suffixes: - - lib - -debian-names: - - libbz2-dev -formula-name: libbz2 -rpm-names: - - libbz2-dev -pacman-names: - - bzip2 diff --git a/external-deps/libzstd.yaml b/external-deps/libzstd.yaml deleted file mode 100644 index 42d201665cb4..000000000000 --- a/external-deps/libzstd.yaml +++ /dev/null @@ -1,25 +0,0 @@ -name: libzstd - -includes: - find: - - names: - - zdict.h - - zstd.h - - zstd_errors.h - path-suffixes: - - include - -libraries: - find: - - names: - - zstd - path-suffixes: - - lib - -debian-names: - - libzstd-dev -formula-name: libzstd -rpm-names: - - libzstd-dev -pacman-names: - - zstd diff --git a/external-deps/mongoc.yaml b/external-deps/mongoc.yaml deleted file mode 100644 index 23cabd30917e..000000000000 --- a/external-deps/mongoc.yaml +++ /dev/null @@ -1,23 +0,0 @@ -name: mongoc -version: 1.16.0 - -libraries: - find: - - names: - - mongoc - - mongoc-1.0 -includes: - find: - - names: - - mongoc/mongoc.h - paths: - - /usr/include/libmongoc-1.0 - - /usr/local/opt/mongo-c/include/libmongoc-1.0 - -debian-names: - - libmongoc-dev -formula-name: mongo-c-driver -rpm-names: - - mongo-c-driver-devel -pacman-names: - - mongo-c-driver diff --git a/external-deps/python-dev.yaml b/external-deps/python-dev.yaml deleted file mode 100644 index 7a7bc7534db2..000000000000 --- a/external-deps/python-dev.yaml +++ /dev/null @@ -1,33 +0,0 @@ -name: PythonDev - -includes: - find: - - names: - - Python.h - paths: - - /usr/include/python3m - - /usr/include/python3.6m - - /usr/include/python3.7m - - /usr/include/python3.8m - - /usr/include/python3.9m - - /usr/include/python3 - - /usr/include/python3.6 - - /usr/include/python3.7 - - /usr/include/python3.8 - - /usr/include/python3.9 - - /usr/include/python3.10 - - /usr/include/python3.11 - - /usr/include/python3.12 - - /usr/include/python3.13 - - /usr/include/python3.14 - -libraries: - enabled: false - -debian-names: - - python3-dev -formula-name: python3-dev -rpm-names: - - python3-dev -pacman-names: - - python diff --git a/grpc/CMakeLists.txt b/grpc/CMakeLists.txt index 9425b32cf1c0..bfc1ce3b8b6c 100644 --- a/grpc/CMakeLists.txt +++ b/grpc/CMakeLists.txt @@ -189,7 +189,7 @@ _userver_directory_install(COMPONENT grpc FILES "${USERVER_ROOT_DIR}/cmake/SetupGrpc.cmake" "${USERVER_ROOT_DIR}/cmake/GrpcTargets.cmake" "${USERVER_ROOT_DIR}/cmake/install/userver-grpc-config.cmake" - "${CMAKE_BINARY_DIR}/cmake_generated/FindUserverGrpc.cmake" + "${USERVER_ROOT_DIR}/cmake/modules/FindUserverGrpc.cmake" DIRECTORY "${USERVER_ROOT_DIR}/scripts/grpc" DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/userver diff --git a/kafka/CMakeLists.txt b/kafka/CMakeLists.txt index 5b20bb79fc15..99d8bc5f5586 100644 --- a/kafka/CMakeLists.txt +++ b/kafka/CMakeLists.txt @@ -45,7 +45,7 @@ _userver_install_targets(COMPONENT kafka TARGETS ${PROJECT_NAME}) _userver_directory_install(COMPONENT kafka FILES "${USERVER_ROOT_DIR}/cmake/install/userver-kafka-config.cmake" - "${CMAKE_BINARY_DIR}/cmake_generated/FindRdKafka.cmake" + "${USERVER_ROOT_DIR}/cmake/modules/FindRdKafka.cmake" DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/userver ) diff --git a/mongo/CMakeLists.txt b/mongo/CMakeLists.txt index c8cf61d90653..fabc9028129b 100644 --- a/mongo/CMakeLists.txt +++ b/mongo/CMakeLists.txt @@ -62,8 +62,8 @@ _userver_install_targets(COMPONENT mongo TARGETS ${PROJECT_NAME}) _userver_directory_install(COMPONENT mongo FILES "${USERVER_ROOT_DIR}/cmake/install/userver-mongo-config.cmake" - "${CMAKE_BINARY_DIR}/cmake_generated/Findmongoc.cmake" - "${CMAKE_BINARY_DIR}/cmake_generated/Findbson.cmake" + "${USERVER_ROOT_DIR}/cmake/modules/Findmongoc.cmake" + "${USERVER_ROOT_DIR}/cmake/modules/Findbson.cmake" DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/userver ) diff --git a/mysql/CMakeLists.txt b/mysql/CMakeLists.txt index b97810a9cb05..cddca21d1973 100644 --- a/mysql/CMakeLists.txt +++ b/mysql/CMakeLists.txt @@ -43,7 +43,7 @@ _userver_install_targets(COMPONENT mysql TARGETS ${PROJECT_NAME}) _userver_directory_install(COMPONENT mysql FILES "${USERVER_ROOT_DIR}/cmake/install/userver-mysql-config.cmake" - "${CMAKE_BINARY_DIR}/cmake_generated/Findlibmariadb.cmake" + "${USERVER_ROOT_DIR}/cmake/modules/Findlibmariadb.cmake" DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/userver ) diff --git a/postgresql/pq-extra/CMakeLists.txt b/postgresql/pq-extra/CMakeLists.txt index 433d0e906895..dc56b43b0536 100644 --- a/postgresql/pq-extra/CMakeLists.txt +++ b/postgresql/pq-extra/CMakeLists.txt @@ -53,7 +53,7 @@ _userver_install_targets(COMPONENT postgres TARGETS ${PROJECT_NAME}) _userver_directory_install(COMPONENT postgres FILES "${USERVER_ROOT_DIR}/cmake/SetupPostgresqlDeps.cmake" - "${CMAKE_BINARY_DIR}/cmake_generated/FindPostgreSQLInternal.cmake" - "${CMAKE_BINARY_DIR}/cmake_generated/FindGssApi.cmake" + "${USERVER_ROOT_DIR}/cmake/modules/FindPostgreSQLInternal.cmake" + "${USERVER_ROOT_DIR}/cmake/modules/FindGssApi.cmake" DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/userver ) diff --git a/redis/CMakeLists.txt b/redis/CMakeLists.txt index caaac0df022d..3103e94abe4c 100644 --- a/redis/CMakeLists.txt +++ b/redis/CMakeLists.txt @@ -88,7 +88,7 @@ _userver_install_targets(COMPONENT redis TARGETS ${PROJECT_NAME}) _userver_directory_install(COMPONENT redis FILES "${USERVER_ROOT_DIR}/cmake/install/userver-redis-config.cmake" - "${CMAKE_BINARY_DIR}/cmake_generated/FindHiredis.cmake" + "${USERVER_ROOT_DIR}/cmake/modules/FindHiredis.cmake" DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/userver ) diff --git a/scripts/external_deps/__init__.py b/scripts/external_deps/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/scripts/external_deps/cmake_generator.py b/scripts/external_deps/cmake_generator.py deleted file mode 100644 index 944e591bff5d..000000000000 --- a/scripts/external_deps/cmake_generator.py +++ /dev/null @@ -1,161 +0,0 @@ -import argparse -import os - -import jinja2 -import voluptuous -import yaml - - -FIND_HELPER_TYPE = 'find-helper' - -EXTERNAL_DEPS_TYPE = [FIND_HELPER_TYPE] - -LIB_SCHEMA = voluptuous.Schema( - { - voluptuous.Required('name'): str, - 'type': voluptuous.Any(None, *EXTERNAL_DEPS_TYPE), - 'package-name': str, - 'debian-names': [str], - 'debian-binary-depends': [str], - 'formula-name': str, - 'rpm-names': [str], - 'pacman-names': [str], - 'pkg-names': [str], - # For version detection of manually installed packages and unknown - # package managers. - 'pkg-config-names': [str], - 'version': voluptuous.Any(str, int), - 'extra-cmake-vars': {str: str}, - 'includes': { - 'enabled': bool, - 'variable': str, - 'find': [{'path-suffixes': list, 'paths': list, 'names': list}], - }, - 'libraries': { - 'enabled': bool, - 'variable': str, - 'find': [{'path-suffixes': list, 'paths': list, 'names': list}], - }, - 'programs': { - 'enabled': bool, - 'variable': str, - 'find': [{'path-suffixes': list, 'paths': list, 'names': list}], - }, - 'fail-message': str, - 'compile-definitions': {'names': list}, - 'checks': [{'expression': str, 'error': str}], - }, -) -CONFIG_SCHEMA = voluptuous.Schema( - voluptuous.Any( - LIB_SCHEMA, - { - voluptuous.Required('common-name'): str, - voluptuous.Required('partials'): list, - }, - ), -) - - -def _load_yaml(path): - with open(path, encoding='utf-8') as fin: - return yaml.load(fin, getattr(yaml, 'CSafeLoader', yaml.SafeLoader)) - - -def generate_cmake(name: str, value, renderer: jinja2.Environment): - result: dict = {} - - fail_message = value.get('fail-message') - if fail_message: - fail_message = fail_message.replace('\n', ' ') - - assert '::' not in name, f'Should be no "::" in "{name}"' - filename = f'Find{name}.cmake' - cmake_type = value.get('type', FIND_HELPER_TYPE) - - assert cmake_type == FIND_HELPER_TYPE - result[filename] = renderer.get_template('FindHelper.jinja').render( - { - 'name': name, - 'package_name': value.get('package-name'), - 'common_name': value.get('common-name'), - 'debian_names': value.get('debian-names'), - 'formula_name': value.get('formula-name'), - 'rpm_names': value.get('rpm-names'), - 'pacman_names': value.get('pacman-names'), - 'pkg_names': value.get('pkg-names'), - 'pkg_config_names': value.get('pkg-config-names'), - 'version': value.get('version'), - 'extra_cmake_vars': value.get('extra-cmake-vars', {}), - 'includes': value.get('includes'), - 'libraries': value.get('libraries'), - 'programs': value.get('programs'), - 'fail_message': fail_message, - 'virtual': value.get('virtual', False), - 'compile_definitions': value.get('compile-definitions'), - 'checks': value.get('checks', []), - }, - ) - - return result - - -def parse_deps_files(dependencies_dir: str) -> dict: - dependencies: dict = {} - - for path, _, files in os.walk(dependencies_dir): - for file_name in files: - _, config_ext = os.path.splitext(file_name) - if config_ext != '.yaml': - continue - file_path = os.path.join(path, file_name) - config = _load_yaml(file_path) - CONFIG_SCHEMA(config) - if 'common-name' in config: - for partial in config.get('partials'): - dependencies[partial['name']] = { - 'common-name': config['common-name'], - **partial, - } - else: - dependencies[config['name']] = config - - return dependencies - - -def main(argv=None): - parser = argparse.ArgumentParser() - parser.add_argument('--repo-dir', required=True) - parser.add_argument('--build-dir', required=True) - parser.add_argument('--log-level', default='WARNING') - - args = parser.parse_args(argv) - - dependencies_dir = os.path.join(args.repo_dir, 'external-deps') - dependencies = parse_deps_files(dependencies_dir) - - cmake_generated_path = os.path.join(args.build_dir, 'cmake_generated') - os.makedirs(cmake_generated_path, exist_ok=True) - - template_path = os.path.join( - args.repo_dir, 'scripts', 'external_deps', 'templates', - ) - renderer = jinja2.Environment( - trim_blocks=True, - keep_trailing_newline=True, - loader=jinja2.FileSystemLoader([template_path]), - undefined=jinja2.StrictUndefined, - extensions=['jinja2.ext.loopcontrols'], - # We do not generate HTML pages. The output could be shown in web by - # CI/CD which should escape the messages on its own. - autoescape=False, - ) - for key, value in dependencies.items(): - for filename, data in generate_cmake(key, value, renderer).items(): - file_path = os.path.join(cmake_generated_path, filename) - with open(file_path, 'w') as output_file: - output_file.write(data) - - -if __name__ == '__main__': - main() diff --git a/scripts/external_deps/requirements.txt b/scripts/external_deps/requirements.txt deleted file mode 100644 index ca0f078ff2fa..000000000000 --- a/scripts/external_deps/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -voluptuous >= 0.11.1 -Jinja2 >= 2.10 -PyYAML >= 3.13 diff --git a/scripts/external_deps/templates/FindHelper.jinja b/scripts/external_deps/templates/FindHelper.jinja deleted file mode 100644 index a1293d9bf1e5..000000000000 --- a/scripts/external_deps/templates/FindHelper.jinja +++ /dev/null @@ -1,231 +0,0 @@ -# AUTOGENERATED, DON'T CHANGE THIS FILE! -# -# Generated via userver/scripts/external_deps from userver/external-deps/{{ common_name or name }}.yaml -{% set libraries_variable = libraries.variable|default('%s_LIBRARIES'|format(name)) %} -{% set includes_variable = includes.variable|default('%s_INCLUDE_DIRS'|format(name)) %} -{% set programs_variable = programs.variable|default('%s_EXECUTABLE'|format(name)) %} -{% set programs_enabled = programs and programs.enabled|default(true) %} -{% set current_package_name = package_name or name %} - -{% if version %} -if (NOT {{ name }}_FIND_VERSION OR {{ name }}_FIND_VERSION VERSION_LESS {{ version }}) - set({{ name }}_FIND_VERSION {{ version }}) -endif() -{% endif %} - -if (NOT USERVER_CHECK_PACKAGE_VERSIONS) - unset({{ name }}_FIND_VERSION) -endif() - -if (TARGET {{ name }}) - if (NOT {{ name }}_FIND_VERSION) - set({{ name }}_FOUND ON) - return() - endif() - - if ({{ name }}_VERSION) - if ({{ name }}_FIND_VERSION VERSION_LESS_EQUAL {{ name }}_VERSION) - set({{ name }}_FOUND ON) - return() - else() - message(FATAL_ERROR - "Already using version {{ '${' }}{{ name }}_VERSION{{ '}' }} " - "of {{ name }} when version {{ '${' }}{{ name }}_FIND_VERSION{{ '}' }} " - "was requested." - ) - endif() - endif() -endif() - -{% for var_name, var_value in extra_cmake_vars.items() %} -set({{var_name}} {{var_value}}) -{% endfor %} - -{%- set full_error_message = '' -%} -{%- if fail_message -%} - {%- set full_error_message = fail_message|trim -%} -{%- else -%} - {%- set full_error_message = "Could not find `{}` package.".format(name) -%} - {%- if debian_names -%} - {%- set full_error_message = full_error_message + "\\n\\tDebian: sudo apt update && sudo apt install " + ' '.join(debian_names) -%} - {%- endif -%} - {%- if formula_name -%} - {%- set full_error_message = full_error_message + "\\n\\tMacOS: brew install {}".format(formula_name) -%} - {%- endif -%} - {%- if rpm_names -%} - {%- set full_error_message = full_error_message + "\\n\\tFedora: sudo dnf install " + ' '.join(rpm_names) -%} - {%- endif -%} - {%- if pacman_names -%} - {%- set full_error_message = full_error_message + "\\n\\tArchLinux: sudo pacman -S " + ' '.join(pacman_names) -%} - {%- endif -%} - {%- if pkg_names -%} - {%- set full_error_message = full_error_message + "\\n\\tFreeBSD: pkg install " + ' '.join(pkg_names) -%} - {%- endif -%} -{%- endif -%} - -set(FULL_ERROR_MESSAGE "{{ full_error_message }}\\n") - -{% macro cmake_find_impl(type, variable, value) %} -{%- set mangled_name = variable + '_' + value.names|join('_')|replace('/', '_')|replace('.', '_') -%} - -find_{{ type }}({{ mangled_name }} - NAMES {{ value.names|join(' ') }} -{% if value.get('path-suffixes') %} - PATH_SUFFIXES {{ value['path-suffixes']|join(' ') }} -{% endif %} -{% if value.get('paths') %} - PATHS {{ value['paths']|join(' ') }} -{% endif %} -) -list(APPEND {{ variable }} {{ '${' }}{{ mangled_name }}{{ '}'}}) -{% endmacro %} - -{%- macro cmake_find(type, variable, values) %} - {%- if values.find|default(false) -%} - {%- for value in values.find -%} - {{ cmake_find_impl(type, variable, value) }} - {%- endfor -%} - {%- endif %} -{% endmacro %} - -{%- for check in checks %} -if ({{ check.expression }}) - message(FATAL_ERROR "{{ check.error }}") -endif() -{% endfor -%} - -{%- if not virtual %} - -include(FindPackageHandleStandardArgs) - -{{ cmake_find('library', libraries_variable, libraries) }} -{{ cmake_find('path', includes_variable, includes) }} -{{ cmake_find('program', programs_variable, programs) }} - -if ({{ name }}_VERSION) - set({{ current_package_name }}_VERSION {{ '${' }}{{ name }}_VERSION{{ '}' }}) -endif() - -{# If it is possible detect the version as the dependent CMake files may rely on it #} -{% if debian_names or rpm_names or pacman_names or pkg_config_names or formula_name or pkg_names -%} -if (NOT {{ current_package_name }}_VERSION) - include(DetectVersion) - - {% if debian_names or rpm_names or pacman_names or pkg_config_names or pkg_names -%} - if (UNIX AND NOT APPLE) - {% if debian_names -%} - deb_version({{ current_package_name }}_VERSION {{ debian_names[0] }}) - {%- endif %} - - {% if rpm_names -%} - rpm_version({{ current_package_name }}_VERSION {{ rpm_names[0] }}) - {%- endif %} - - {% if pacman_names -%} - pacman_version({{ current_package_name }}_VERSION {{ pacman_names[0] }}) - {%- endif %} - - {% if pkg_names -%} - pkg_version({{ current_package_name }}_VERSION {{ pkg_names[0] }}) - {%- endif %} - - endif() - {%- endif %} - - {% if formula_name -%} - if (APPLE) - brew_version({{ current_package_name }}_VERSION {{ formula_name }}) - endif() - {%- endif %} - - {% if pkg_config_names -%} - pkg_config_version({{ current_package_name }}_VERSION {{ pkg_config_names[0] }}) - {%- endif %} - -endif() -{% endif %} - -{% if not libraries.find|default(false) and not includes.find|default(false) and not programs.find|default(false) %} -find_package({{ common_name or current_package_name }} {{ version or '' }} -{% if common_name %} - COMPONENTS {{ name }} -{% endif %} {# if common_name #} -) -set({{ name }}_FOUND {{ '${' }}{{ common_name or current_package_name }}_FOUND{{ '}'}}) -{% endif %} {# if not libraries.find.names|default(false) and not includes.names|default(false) #} - -{% if libraries.find|default(false) or includes.find|default(false) or programs.find|default(false) %} -find_package_handle_standard_args( - {{ current_package_name }} - REQUIRED_VARS - {{ libraries_variable if libraries.enabled|default(true) else '' }} - {{ includes_variable if includes.enabled|default(true) else '' }} - {{ programs_variable if programs_enabled else '' }} - FAIL_MESSAGE - "${FULL_ERROR_MESSAGE}" -) -mark_as_advanced( - {{ libraries_variable if libraries.enabled|default(true) else '' }} - {{ includes_variable if includes.enabled|default(true) else '' }} - {{ programs_variable if programs_enabled else '' }} -) -{% endif %} - -if (NOT {{ name }}_FOUND) - if ({{ name }}_FIND_REQUIRED) - message(FATAL_ERROR "${FULL_ERROR_MESSAGE}. Required version is at least {{ '${' }}{{ name }}_FIND_VERSION{{ '}' }}") - endif() - - return() -endif() - -if ({{ name }}_FIND_VERSION) - if ({{ current_package_name }}_VERSION VERSION_LESS {{ name }}_FIND_VERSION) - message(STATUS - "Version of {{ name }} is '{{ '${' }}{{ current_package_name }}_VERSION{{ '}'}}'. " - "Required version is at least '{{ '${' }}{{ name }}_FIND_VERSION{{ '}'}}'. " - "Ignoring found {{ name }}. " - "Note: Set -DUSERVER_CHECK_PACKAGE_VERSIONS=0 to skip package version checks if the package is fine." - ) - set({{ name }}_FOUND OFF) - return() - endif() -endif() - -{% endif %} {# if not virtual #} - -{% if common_name or libraries.enabled|default(true) or includes.enabled|default(true) or virtual %} -if (NOT TARGET {{ name }}) - add_library({{ name }} INTERFACE IMPORTED GLOBAL) - - {% if common_name -%} - if (TARGET {{ '{}::{}'.format(common_name, name) }}) - target_link_libraries({{ name }} INTERFACE {{ '{}::{}'.format(common_name, name) }}) - endif() - {% endif -%} - - {% if includes.enabled|default(true) -%} - target_include_directories({{ name }} INTERFACE {{ '${' }}{{ includes_variable }}{{ '}' }}) - message(STATUS "{{ name }} include directories: {{ '${' }}{{ includes_variable }}{{ '}' }}") - {% endif -%} - - {% if libraries.enabled|default(true) -%} - target_link_libraries({{ name }} INTERFACE {{ '${' }}{{ libraries_variable }}{{ '}' }}) - message(STATUS "{{ name }} libraries: {{ '${' }}{{ libraries_variable }}{{ '}' }}") - {% endif -%} - - {% if compile_definitions and compile_definitions.names -%} - target_compile_definitions({{ name }} INTERFACE - {%- for option in compile_definitions.names %} - {{ option }} - {% endfor %} - ) - {%- endif %} - - # Target {{ name }} is created -endif() - -if ({{ current_package_name }}_VERSION) - set({{ name }}_VERSION "{{ '${' }}{{ current_package_name }}_VERSION{{ '}' }}" CACHE STRING "Version of the {{ name }}") -endif() -{% endif %} diff --git a/universal/CMakeLists.txt b/universal/CMakeLists.txt index 1b61d921c099..5b6fbbe35dc7 100644 --- a/universal/CMakeLists.txt +++ b/universal/CMakeLists.txt @@ -364,12 +364,13 @@ if (USERVER_IS_THE_ROOT_PROJECT) endif() _userver_directory_install(COMPONENT universal FILES - "${CMAKE_BINARY_DIR}/cmake_generated/FindCryptoPP.cmake" - "${CMAKE_BINARY_DIR}/cmake_generated/Findlibyamlcpp.cmake" - "${CMAKE_BINARY_DIR}/cmake_generated/Findlibzstd.cmake" - "${CMAKE_BINARY_DIR}/cmake_generated/Findcctz.cmake" - "${CMAKE_BINARY_DIR}/cmake_generated/FindJemalloc.cmake" - "${CMAKE_BINARY_DIR}/cmake_generated/FindUserverGBench.cmake" + "${USERVER_ROOT_DIR}/cmake/modules/FindCryptoPP.cmake" + "${USERVER_ROOT_DIR}/cmake/modules/Findlibyamlcpp.cmake" + "${USERVER_ROOT_DIR}/cmake/modules/Findlibzstd.cmake" + "${USERVER_ROOT_DIR}/cmake/modules/Findcctz.cmake" + "${USERVER_ROOT_DIR}/cmake/modules/FindJemalloc.cmake" + "${USERVER_ROOT_DIR}/cmake/modules/FindUserverGBench.cmake" + "${USERVER_ROOT_DIR}/cmake/ModuleHelpers.cmake" "${USERVER_ROOT_DIR}/cmake/SetupGTest.cmake" "${USERVER_ROOT_DIR}/cmake/SetupGBench.cmake" "${USERVER_ROOT_DIR}/cmake/sanitize.blacklist.txt" From 8a4c22e6d5c004d4beafa9b46f8d440e79414105 Mon Sep 17 00:00:00 2001 From: antoshkka Date: Thu, 25 Jul 2024 15:01:09 +0300 Subject: [PATCH 083/629] feat core: remove unused virtual function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes: https://github.com/userver-framework/userver/issues/657 Tests: протестировано CI d82d04ecd6395bb37176c6e534eccb4233f31c77 --- core/include/userver/server/handlers/http_handler_base.hpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/core/include/userver/server/handlers/http_handler_base.hpp b/core/include/userver/server/handlers/http_handler_base.hpp index 890c81d33325..bde02f16375f 100644 --- a/core/include/userver/server/handlers/http_handler_base.hpp +++ b/core/include/userver/server/handlers/http_handler_base.hpp @@ -132,10 +132,6 @@ class HttpHandlerBase : public HandlerBase { virtual std::string HandleRequestThrow( const http::HttpRequest& request, request::RequestContext& context) const; - virtual void OnRequestCompleteThrow( - const http::HttpRequest& /*request*/, - request::RequestContext& /*context*/) const {} - /// The core method for HTTP request handling. /// `request` arg contains HTTP headers, full body, etc. /// The response body is passed in parts to `ResponseBodyStream`. From 0f7e992c56a35c15520a850558ad05476f0f3ed4 Mon Sep 17 00:00:00 2001 From: antonyzhilin Date: Thu, 25 Jul 2024 18:29:11 +0300 Subject: [PATCH 084/629] fix grpc: make userver::grpc-utest link to userver::utest b2c09c921e678562abd971ab082792ea7729e1ab --- .mapping.json | 5 +++-- grpc/CMakeLists.txt | 9 ++++----- grpc/benchmarks/ya.make | 1 - .../include/userver/ugrpc/tests/service.hpp | 2 +- grpc/{utest => }/src/ugrpc/tests/service.cpp | 0 .../userver/ugrpc/tests/service_fixtures.hpp | 8 ++------ grpc/utest/src/ugrpc/tests/service_fixtures.cpp | 16 ++++++++++++++++ scripts/docs/en/userver/tutorial/build.md | 1 + 8 files changed, 27 insertions(+), 15 deletions(-) rename grpc/{utest => }/include/userver/ugrpc/tests/service.hpp (98%) rename grpc/{utest => }/src/ugrpc/tests/service.cpp (100%) create mode 100644 grpc/utest/src/ugrpc/tests/service_fixtures.cpp diff --git a/.mapping.json b/.mapping.json index bd9a3f4d69e8..be3ee88b7fb2 100644 --- a/.mapping.json +++ b/.mapping.json @@ -1860,6 +1860,7 @@ "grpc/include/userver/ugrpc/server/service_component_base.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/server/service_component_base.hpp", "grpc/include/userver/ugrpc/server/storage_context.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/server/storage_context.hpp", "grpc/include/userver/ugrpc/status_codes.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/status_codes.hpp", + "grpc/include/userver/ugrpc/tests/service.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/tests/service.hpp", "grpc/library.yaml":"taxi/uservices/userver/grpc/library.yaml", "grpc/proto/tests/global_package.proto":"taxi/uservices/userver/grpc/proto/tests/global_package.proto", "grpc/proto/tests/messages.proto":"taxi/uservices/userver/grpc/proto/tests/messages.proto", @@ -1947,6 +1948,7 @@ "grpc/src/ugrpc/server/service_base.cpp":"taxi/uservices/userver/grpc/src/ugrpc/server/service_base.cpp", "grpc/src/ugrpc/server/service_component_base.cpp":"taxi/uservices/userver/grpc/src/ugrpc/server/service_component_base.cpp", "grpc/src/ugrpc/status_codes.cpp":"taxi/uservices/userver/grpc/src/ugrpc/status_codes.cpp", + "grpc/src/ugrpc/tests/service.cpp":"taxi/uservices/userver/grpc/src/ugrpc/tests/service.cpp", "grpc/tests/async_test.cpp":"taxi/uservices/userver/grpc/tests/async_test.cpp", "grpc/tests/baggage_test.cpp":"taxi/uservices/userver/grpc/tests/baggage_test.cpp", "grpc/tests/base_test.cpp":"taxi/uservices/userver/grpc/tests/base_test.cpp", @@ -1970,9 +1972,8 @@ "grpc/tests/tracing_test.cpp":"taxi/uservices/userver/grpc/tests/tracing_test.cpp", "grpc/tests/unimplemented_test.cpp":"taxi/uservices/userver/grpc/tests/unimplemented_test.cpp", "grpc/tests/wait_any_test.cpp":"taxi/uservices/userver/grpc/tests/wait_any_test.cpp", - "grpc/utest/include/userver/ugrpc/tests/service.hpp":"taxi/uservices/userver/grpc/utest/include/userver/ugrpc/tests/service.hpp", "grpc/utest/include/userver/ugrpc/tests/service_fixtures.hpp":"taxi/uservices/userver/grpc/utest/include/userver/ugrpc/tests/service_fixtures.hpp", - "grpc/utest/src/ugrpc/tests/service.cpp":"taxi/uservices/userver/grpc/utest/src/ugrpc/tests/service.cpp", + "grpc/utest/src/ugrpc/tests/service_fixtures.cpp":"taxi/uservices/userver/grpc/utest/src/ugrpc/tests/service_fixtures.cpp", "kafka/CMakeLists.txt":"taxi/uservices/userver/kafka/CMakeLists.txt", "kafka/README.md":"taxi/uservices/userver/kafka/README.md", "kafka/functional_tests/CMakeLists.txt":"taxi/uservices/userver/kafka/functional_tests/CMakeLists.txt", diff --git a/grpc/CMakeLists.txt b/grpc/CMakeLists.txt index bfc1ce3b8b6c..c9b36796c053 100644 --- a/grpc/CMakeLists.txt +++ b/grpc/CMakeLists.txt @@ -117,6 +117,7 @@ if (USERVER_FEATURE_UTEST) target_link_libraries(${PROJECT_NAME}-utest PUBLIC ${PROJECT_NAME}-internal + userver::utest ) target_include_directories(${PROJECT_NAME}-utest PUBLIC @@ -128,8 +129,8 @@ if (USERVER_FEATURE_UTEST) ) _userver_install_targets(COMPONENT grpc TARGETS ${PROJECT_NAME}-utest) _userver_directory_install(COMPONENT grpc - DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/utest/include - DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/.. + DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/utest/include + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/.. ) endif() @@ -154,7 +155,6 @@ if (USERVER_IS_THE_ROOT_PROJECT) target_link_libraries(${PROJECT_NAME}-unittest PUBLIC ${PROJECT_NAME}-internal - userver-utest ${PROJECT_NAME}-utest PRIVATE ${PROJECT_NAME}-unittest-proto @@ -165,11 +165,10 @@ if (USERVER_IS_THE_ROOT_PROJECT) target_link_libraries(${PROJECT_NAME}-benchmark ${PROJECT_NAME}-internal userver-ubench - ${PROJECT_NAME}-utest ${PROJECT_NAME}-unittest-proto ) target_include_directories(${PROJECT_NAME}-benchmark PRIVATE - $ + $ ) add_google_benchmark_tests(${PROJECT_NAME}-benchmark) diff --git a/grpc/benchmarks/ya.make b/grpc/benchmarks/ya.make index 4a43fdd96fff..dfaafc87f7f8 100644 --- a/grpc/benchmarks/ya.make +++ b/grpc/benchmarks/ya.make @@ -7,7 +7,6 @@ SUBSCRIBER(g:taxi-common) PEERDIR( taxi/uservices/userver-arc-utils/grpc/gen/grpc - taxi/uservices/userver/grpc/utest taxi/uservices/userver/grpc ) diff --git a/grpc/utest/include/userver/ugrpc/tests/service.hpp b/grpc/include/userver/ugrpc/tests/service.hpp similarity index 98% rename from grpc/utest/include/userver/ugrpc/tests/service.hpp rename to grpc/include/userver/ugrpc/tests/service.hpp index a7f8ee4412bb..4b0988442cf7 100644 --- a/grpc/utest/include/userver/ugrpc/tests/service.hpp +++ b/grpc/include/userver/ugrpc/tests/service.hpp @@ -2,7 +2,7 @@ /// @file userver/ugrpc/tests/service.hpp /// @brief Base classes for testing and benchmarking ugrpc service -/// implementations. Requires linking to `userver::grpc-utest`. +/// implementations in a simplified gRPC environment. #include #include diff --git a/grpc/utest/src/ugrpc/tests/service.cpp b/grpc/src/ugrpc/tests/service.cpp similarity index 100% rename from grpc/utest/src/ugrpc/tests/service.cpp rename to grpc/src/ugrpc/tests/service.cpp diff --git a/grpc/utest/include/userver/ugrpc/tests/service_fixtures.hpp b/grpc/utest/include/userver/ugrpc/tests/service_fixtures.hpp index 7d87c8c361e9..da83542bb62c 100644 --- a/grpc/utest/include/userver/ugrpc/tests/service_fixtures.hpp +++ b/grpc/utest/include/userver/ugrpc/tests/service_fixtures.hpp @@ -2,7 +2,7 @@ /// @file userver/ugrpc/tests/service_fixtures.hpp /// @brief gtest fixtures for testing ugrpc service implementations. Requires -/// linking to `userver::grpc-utest` and `userver::utest`. +/// linking to `userver::grpc-utest`. #include #include @@ -21,11 +21,7 @@ class ServiceFixtureBase : protected ServiceBase, public ::testing::Test { /// @returns the statistics of the server and clients. utils::statistics::Snapshot GetStatistics( std::string prefix, - std::vector require_labels = {}) { - return utils::statistics::Snapshot{this->GetStatisticsStorage(), - std::move(prefix), - std::move(require_labels)}; - } + std::vector require_labels = {}); }; /// @see @ref ugrpc::tests::Service diff --git a/grpc/utest/src/ugrpc/tests/service_fixtures.cpp b/grpc/utest/src/ugrpc/tests/service_fixtures.cpp new file mode 100644 index 000000000000..7b3a26a1eea7 --- /dev/null +++ b/grpc/utest/src/ugrpc/tests/service_fixtures.cpp @@ -0,0 +1,16 @@ +#include + +USERVER_NAMESPACE_BEGIN + +namespace ugrpc::tests { + +utils::statistics::Snapshot ServiceFixtureBase::GetStatistics( + std::string prefix, std::vector require_labels) { + return utils::statistics::Snapshot{this->GetStatisticsStorage(), + std::move(prefix), + std::move(require_labels)}; +} + +} // namespace ugrpc::tests + +USERVER_NAMESPACE_END diff --git a/scripts/docs/en/userver/tutorial/build.md b/scripts/docs/en/userver/tutorial/build.md index e3921e13dffd..63443fc8bd20 100644 --- a/scripts/docs/en/userver/tutorial/build.md +++ b/scripts/docs/en/userver/tutorial/build.md @@ -157,6 +157,7 @@ userver is split into multiple CMake libraries. | `userver::utest` | `USERVER_FEATURE_CORE` + `USERVER_FEATURE_UTEST` | `core` | @ref scripts/docs/en/userver/testing.md | | `userver::ubench` | `USERVER_FEATURE_CORE` + `USERVER_FEATURE_UTEST` | `core` | @ref scripts/docs/en/userver/testing.md | | `userver::grpc` | `USERVER_FEATURE_GRPC` | `grpc` | @ref scripts/docs/en/userver/grpc.md | +| `userver::grpc-utest` | `USERVER_FEATURE_GRPC` + `USERVER_FEATURE_UTEST` | `grpc` | @ref scripts/docs/en/userver/grpc.md | | `userver::mongo` | `USERVER_FEATURE_MONGODB` | `mongo` | @ref scripts/docs/en/userver/mongodb.md | | `userver::postgresql` | `USERVER_FEATURE_POSTGRESQL` | `postgresql` | @ref pg_driver | | `userver::redis` | `USERVER_FEATURE_REDIS` | `redis` | @ref scripts/docs/en/userver/redis.md | From 87c789894939f3401a4d4d63a7180f093203dc92 Mon Sep 17 00:00:00 2001 From: kosstin Date: Fri, 26 Jul 2024 06:03:03 +0300 Subject: [PATCH 085/629] feat core: grpc header propagation feat core: grpc header propagation https://st.yandex-team.ru/ 737ed3223a73cc653afb7f4ce653e0dcd27d3dd9 --- .mapping.json | 13 +++- core/include/userver/clients/http/client.hpp | 7 +- core/include/userver/clients/http/config.hpp | 7 +- .../plugins/headers_propagator/plugin.hpp} | 12 +--- core/include/userver/clients/http/request.hpp | 7 +- .../headers_propagator_component.hpp | 63 ------------------ .../server/middlewares/headers_propagator.hpp | 58 ++++++++++++++++ .../server/request/task_inherited_headers.hpp | 29 ++++++++ .../server/request/task_inherited_request.hpp | 6 +- core/src/clients/http/client.cpp | 2 - core/src/clients/http/component.cpp | 6 +- core/src/clients/http/request.cpp | 3 +- core/src/clients/http/request_state.cpp | 4 +- core/src/clients/http/request_state.hpp | 7 +- .../headers_propagator_component.cpp | 44 ------------- core/src/server/http/headers_propagator.cpp | 19 ++---- .../server/middlewares/headers_propagator.cpp | 66 +++++++++++++++++++ .../server/request/task_inherited_headers.cpp | 30 +++++++++ .../basic_server/grpc_service.cpp | 52 ++++++++++++++- .../basic_server/proto/samples/greeter.proto | 1 + .../basic_server/static_config.yaml | 16 ++++- .../tests-unix-socket/conftest.py | 13 +++- .../test_header_propagation.py | 39 +++++++++++ .../ugrpc/server/impl/service_worker_impl.hpp | 1 - .../headers_propagator/component.hpp | 37 +++++++++++ .../headers_propagator/component.cpp | 37 +++++++++++ .../headers_propagator/middleware.cpp | 35 ++++++++++ .../headers_propagator/middleware.hpp | 21 ++++++ 28 files changed, 478 insertions(+), 157 deletions(-) rename core/{src/server/http/headers_propagator.hpp => include/userver/clients/http/plugins/headers_propagator/plugin.hpp} (58%) delete mode 100644 core/include/userver/components/headers_propagator_component.hpp create mode 100644 core/include/userver/server/middlewares/headers_propagator.hpp create mode 100644 core/include/userver/server/request/task_inherited_headers.hpp delete mode 100644 core/src/components/headers_propagator_component.cpp create mode 100644 core/src/server/middlewares/headers_propagator.cpp create mode 100644 core/src/server/request/task_inherited_headers.cpp create mode 100644 grpc/functional_tests/basic_server/tests-unix-socket/test_header_propagation.py create mode 100644 grpc/include/userver/ugrpc/server/middlewares/headers_propagator/component.hpp create mode 100644 grpc/src/ugrpc/server/middlewares/headers_propagator/component.cpp create mode 100644 grpc/src/ugrpc/server/middlewares/headers_propagator/middleware.cpp create mode 100644 grpc/src/ugrpc/server/middlewares/headers_propagator/middleware.hpp diff --git a/.mapping.json b/.mapping.json index be3ee88b7fb2..667abc325be2 100644 --- a/.mapping.json +++ b/.mapping.json @@ -555,6 +555,7 @@ "core/include/userver/clients/http/local_stats.hpp":"taxi/uservices/userver/core/include/userver/clients/http/local_stats.hpp", "core/include/userver/clients/http/plugin.hpp":"taxi/uservices/userver/core/include/userver/clients/http/plugin.hpp", "core/include/userver/clients/http/plugin_component.hpp":"taxi/uservices/userver/core/include/userver/clients/http/plugin_component.hpp", + "core/include/userver/clients/http/plugins/headers_propagator/plugin.hpp":"taxi/uservices/userver/core/include/userver/clients/http/plugins/headers_propagator/plugin.hpp", "core/include/userver/clients/http/plugins/yandex_tracing/component.hpp":"taxi/uservices/userver/core/include/userver/clients/http/plugins/yandex_tracing/component.hpp", "core/include/userver/clients/http/request.hpp":"taxi/uservices/userver/core/include/userver/clients/http/request.hpp", "core/include/userver/clients/http/request_tracing_editor.hpp":"taxi/uservices/userver/core/include/userver/clients/http/request_tracing_editor.hpp", @@ -571,7 +572,6 @@ "core/include/userver/components/component_list.hpp":"taxi/uservices/userver/core/include/userver/components/component_list.hpp", "core/include/userver/components/dump_configurator.hpp":"taxi/uservices/userver/core/include/userver/components/dump_configurator.hpp", "core/include/userver/components/fs_cache.hpp":"taxi/uservices/userver/core/include/userver/components/fs_cache.hpp", - "core/include/userver/components/headers_propagator_component.hpp":"taxi/uservices/userver/core/include/userver/components/headers_propagator_component.hpp", "core/include/userver/components/loggable_component_base.hpp":"taxi/uservices/userver/core/include/userver/components/loggable_component_base.hpp", "core/include/userver/components/logging_configurator.hpp":"taxi/uservices/userver/core/include/userver/components/logging_configurator.hpp", "core/include/userver/components/manager_controller_component.hpp":"taxi/uservices/userver/core/include/userver/components/manager_controller_component.hpp", @@ -782,12 +782,14 @@ "core/include/userver/server/http/http_status.hpp":"taxi/uservices/userver/core/include/userver/server/http/http_status.hpp", "core/include/userver/server/middlewares/builtin.hpp":"taxi/uservices/userver/core/include/userver/server/middlewares/builtin.hpp", "core/include/userver/server/middlewares/configuration.hpp":"taxi/uservices/userver/core/include/userver/server/middlewares/configuration.hpp", + "core/include/userver/server/middlewares/headers_propagator.hpp":"taxi/uservices/userver/core/include/userver/server/middlewares/headers_propagator.hpp", "core/include/userver/server/middlewares/http_middleware_base.hpp":"taxi/uservices/userver/core/include/userver/server/middlewares/http_middleware_base.hpp", "core/include/userver/server/request/request_base.hpp":"taxi/uservices/userver/core/include/userver/server/request/request_base.hpp", "core/include/userver/server/request/request_config.hpp":"taxi/uservices/userver/core/include/userver/server/request/request_config.hpp", "core/include/userver/server/request/request_context.hpp":"taxi/uservices/userver/core/include/userver/server/request/request_context.hpp", "core/include/userver/server/request/response_base.hpp":"taxi/uservices/userver/core/include/userver/server/request/response_base.hpp", "core/include/userver/server/request/task_inherited_data.hpp":"taxi/uservices/userver/core/include/userver/server/request/task_inherited_data.hpp", + "core/include/userver/server/request/task_inherited_headers.hpp":"taxi/uservices/userver/core/include/userver/server/request/task_inherited_headers.hpp", "core/include/userver/server/request/task_inherited_request.hpp":"taxi/uservices/userver/core/include/userver/server/request/task_inherited_request.hpp", "core/include/userver/server/server.hpp":"taxi/uservices/userver/core/include/userver/server/server.hpp", "core/include/userver/server/websocket/server.hpp":"taxi/uservices/userver/core/include/userver/server/websocket/server.hpp", @@ -974,7 +976,6 @@ "core/src/components/config_not_required_test.cpp":"taxi/uservices/userver/core/src/components/config_not_required_test.cpp", "core/src/components/dump_configurator.cpp":"taxi/uservices/userver/core/src/components/dump_configurator.cpp", "core/src/components/fs_cache.cpp":"taxi/uservices/userver/core/src/components/fs_cache.cpp", - "core/src/components/headers_propagator_component.cpp":"taxi/uservices/userver/core/src/components/headers_propagator_component.cpp", "core/src/components/impl/component_base.cpp":"taxi/uservices/userver/core/src/components/impl/component_base.cpp", "core/src/components/impl/component_name_from_info.hpp":"taxi/uservices/userver/core/src/components/impl/component_name_from_info.hpp", "core/src/components/loggable_component_base.cpp":"taxi/uservices/userver/core/src/components/loggable_component_base.cpp", @@ -1437,7 +1438,6 @@ "core/src/server/http/handler_methods.cpp":"taxi/uservices/userver/core/src/server/http/handler_methods.cpp", "core/src/server/http/handler_methods.hpp":"taxi/uservices/userver/core/src/server/http/handler_methods.hpp", "core/src/server/http/headers_propagator.cpp":"taxi/uservices/userver/core/src/server/http/headers_propagator.cpp", - "core/src/server/http/headers_propagator.hpp":"taxi/uservices/userver/core/src/server/http/headers_propagator.hpp", "core/src/server/http/http_cached_date.cpp":"taxi/uservices/userver/core/src/server/http/http_cached_date.cpp", "core/src/server/http/http_cached_date.hpp":"taxi/uservices/userver/core/src/server/http/http_cached_date.hpp", "core/src/server/http/http_cached_date_benchmark.cpp":"taxi/uservices/userver/core/src/server/http/http_cached_date_benchmark.cpp", @@ -1491,6 +1491,7 @@ "core/src/server/middlewares/handler_adapter.hpp":"taxi/uservices/userver/core/src/server/middlewares/handler_adapter.hpp", "core/src/server/middlewares/handler_metrics.cpp":"taxi/uservices/userver/core/src/server/middlewares/handler_metrics.cpp", "core/src/server/middlewares/handler_metrics.hpp":"taxi/uservices/userver/core/src/server/middlewares/handler_metrics.hpp", + "core/src/server/middlewares/headers_propagator.cpp":"taxi/uservices/userver/core/src/server/middlewares/headers_propagator.cpp", "core/src/server/middlewares/http_middleware_base.cpp":"taxi/uservices/userver/core/src/server/middlewares/http_middleware_base.cpp", "core/src/server/middlewares/misc.cpp":"taxi/uservices/userver/core/src/server/middlewares/misc.cpp", "core/src/server/middlewares/misc.hpp":"taxi/uservices/userver/core/src/server/middlewares/misc.hpp", @@ -1525,6 +1526,7 @@ "core/src/server/request/request_parser.hpp":"taxi/uservices/userver/core/src/server/request/request_parser.hpp", "core/src/server/request/response_base.cpp":"taxi/uservices/userver/core/src/server/request/response_base.cpp", "core/src/server/request/task_inherited_data.cpp":"taxi/uservices/userver/core/src/server/request/task_inherited_data.cpp", + "core/src/server/request/task_inherited_headers.cpp":"taxi/uservices/userver/core/src/server/request/task_inherited_headers.cpp", "core/src/server/request/task_inherited_request.cpp":"taxi/uservices/userver/core/src/server/request/task_inherited_request.cpp", "core/src/server/request/task_inherited_request_impl.hpp":"taxi/uservices/userver/core/src/server/request/task_inherited_request_impl.hpp", "core/src/server/requests_view.cpp":"taxi/uservices/userver/core/src/server/requests_view.cpp", @@ -1763,6 +1765,7 @@ "grpc/functional_tests/basic_server/static_config.yaml":"taxi/uservices/userver/grpc/functional_tests/basic_server/static_config.yaml", "grpc/functional_tests/basic_server/tests-unix-socket/conftest.py":"taxi/uservices/userver/grpc/functional_tests/basic_server/tests-unix-socket/conftest.py", "grpc/functional_tests/basic_server/tests-unix-socket/test_grpc.py":"taxi/uservices/userver/grpc/functional_tests/basic_server/tests-unix-socket/test_grpc.py", + "grpc/functional_tests/basic_server/tests-unix-socket/test_header_propagation.py":"taxi/uservices/userver/grpc/functional_tests/basic_server/tests-unix-socket/test_header_propagation.py", "grpc/functional_tests/metrics/CMakeLists.txt":"taxi/uservices/userver/grpc/functional_tests/metrics/CMakeLists.txt", "grpc/functional_tests/metrics/config_vars.yaml":"taxi/uservices/userver/grpc/functional_tests/metrics/config_vars.yaml", "grpc/functional_tests/metrics/proto/samples/greeter.proto":"taxi/uservices/userver/grpc/functional_tests/metrics/proto/samples/greeter.proto", @@ -1852,6 +1855,7 @@ "grpc/include/userver/ugrpc/server/middlewares/congestion_control/component.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/server/middlewares/congestion_control/component.hpp", "grpc/include/userver/ugrpc/server/middlewares/deadline_propagation/component.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/server/middlewares/deadline_propagation/component.hpp", "grpc/include/userver/ugrpc/server/middlewares/fwd.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/server/middlewares/fwd.hpp", + "grpc/include/userver/ugrpc/server/middlewares/headers_propagator/component.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/server/middlewares/headers_propagator/component.hpp", "grpc/include/userver/ugrpc/server/middlewares/log/component.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/server/middlewares/log/component.hpp", "grpc/include/userver/ugrpc/server/rpc.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/server/rpc.hpp", "grpc/include/userver/ugrpc/server/server.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/server/server.hpp", @@ -1939,6 +1943,9 @@ "grpc/src/ugrpc/server/middlewares/deadline_propagation/component.cpp":"taxi/uservices/userver/grpc/src/ugrpc/server/middlewares/deadline_propagation/component.cpp", "grpc/src/ugrpc/server/middlewares/deadline_propagation/middleware.cpp":"taxi/uservices/userver/grpc/src/ugrpc/server/middlewares/deadline_propagation/middleware.cpp", "grpc/src/ugrpc/server/middlewares/deadline_propagation/middleware.hpp":"taxi/uservices/userver/grpc/src/ugrpc/server/middlewares/deadline_propagation/middleware.hpp", + "grpc/src/ugrpc/server/middlewares/headers_propagator/component.cpp":"taxi/uservices/userver/grpc/src/ugrpc/server/middlewares/headers_propagator/component.cpp", + "grpc/src/ugrpc/server/middlewares/headers_propagator/middleware.cpp":"taxi/uservices/userver/grpc/src/ugrpc/server/middlewares/headers_propagator/middleware.cpp", + "grpc/src/ugrpc/server/middlewares/headers_propagator/middleware.hpp":"taxi/uservices/userver/grpc/src/ugrpc/server/middlewares/headers_propagator/middleware.hpp", "grpc/src/ugrpc/server/middlewares/log/component.cpp":"taxi/uservices/userver/grpc/src/ugrpc/server/middlewares/log/component.cpp", "grpc/src/ugrpc/server/middlewares/log/middleware.cpp":"taxi/uservices/userver/grpc/src/ugrpc/server/middlewares/log/middleware.cpp", "grpc/src/ugrpc/server/middlewares/log/middleware.hpp":"taxi/uservices/userver/grpc/src/ugrpc/server/middlewares/log/middleware.hpp", diff --git a/core/include/userver/clients/http/client.hpp b/core/include/userver/clients/http/client.hpp index 6126dd8b4a09..694283f5d885 100644 --- a/core/include/userver/clients/http/client.hpp +++ b/core/include/userver/clients/http/client.hpp @@ -29,9 +29,9 @@ namespace tracing { class TracingManagerBase; } // namespace tracing -namespace server::http { +namespace clients::http::plugins::headers_propagator { class HeadersPropagator; -} // namespace server::http +} // namespace clients::http::plugins::headers_propagator namespace curl { class easy; @@ -178,7 +178,8 @@ class Client final { clients::dns::Resolver* resolver_{nullptr}; utils::NotNull tracing_manager_; - const server::http::HeadersPropagator* headers_propagator_{nullptr}; + const clients::http::plugins::headers_propagator::HeadersPropagator* + headers_propagator_{nullptr}; impl::PluginPipeline plugin_pipeline_; }; diff --git a/core/include/userver/clients/http/config.hpp b/core/include/userver/clients/http/config.hpp index a6a5c86d836d..0ab12b9f6e79 100644 --- a/core/include/userver/clients/http/config.hpp +++ b/core/include/userver/clients/http/config.hpp @@ -13,9 +13,9 @@ namespace tracing { class TracingManagerBase; } // namespace tracing -namespace server::http { +namespace clients::http::plugins::headers_propagator { class HeadersPropagator; -} // namespace server::http +} // namespace clients::http::plugins::headers_propagator namespace clients::http { @@ -38,7 +38,8 @@ struct ClientSettings final { bool defer_events{false}; DeadlinePropagationConfig deadline_propagation{}; const tracing::TracingManagerBase* tracing_manager{nullptr}; - const server::http::HeadersPropagator* headers_propagator{nullptr}; + const clients::http::plugins::headers_propagator::HeadersPropagator* + headers_propagator{nullptr}; CancellationPolicy cancellation_policy{CancellationPolicy::kCancel}; }; diff --git a/core/src/server/http/headers_propagator.hpp b/core/include/userver/clients/http/plugins/headers_propagator/plugin.hpp similarity index 58% rename from core/src/server/http/headers_propagator.hpp rename to core/include/userver/clients/http/plugins/headers_propagator/plugin.hpp index 1e246a0918f9..58cf31358688 100644 --- a/core/src/server/http/headers_propagator.hpp +++ b/core/include/userver/clients/http/plugins/headers_propagator/plugin.hpp @@ -1,25 +1,17 @@ #pragma once -#include -#include - #include #include USERVER_NAMESPACE_BEGIN -namespace server::http { +namespace clients::http::plugins::headers_propagator { class HeadersPropagator final { public: - explicit HeadersPropagator(std::vector&&); - void PropagateHeaders(clients::http::RequestTracingEditor request) const; - - private: - const std::vector headers_; }; -} // namespace server::http +} // namespace clients::http::plugins::headers_propagator USERVER_NAMESPACE_END diff --git a/core/include/userver/clients/http/request.hpp b/core/include/userver/clients/http/request.hpp index 5ea7c30aa542..295ed5dcec8e 100644 --- a/core/include/userver/clients/http/request.hpp +++ b/core/include/userver/clients/http/request.hpp @@ -23,9 +23,9 @@ namespace tracing { class TracingManagerBase; } // namespace tracing -namespace server::http { +namespace clients::http::plugins::headers_propagator { class HeadersPropagator; -} // namespace server::http +} // namespace clients::http::plugins::headers_propagator /// HTTP client helpers namespace clients::http { @@ -292,7 +292,8 @@ class Request final { void SetDeadlinePropagationConfig( const DeadlinePropagationConfig& deadline_propagation_config) &; - void SetHeadersPropagator(const server::http::HeadersPropagator*) &; + void SetHeadersPropagator( + const clients::http::plugins::headers_propagator::HeadersPropagator*) &; /// @endcond /// Disable auto-decoding of received replies. diff --git a/core/include/userver/components/headers_propagator_component.hpp b/core/include/userver/components/headers_propagator_component.hpp deleted file mode 100644 index 8c31231b1692..000000000000 --- a/core/include/userver/components/headers_propagator_component.hpp +++ /dev/null @@ -1,63 +0,0 @@ -#pragma once - -/// @file userver/components/headers_propagator_component.hpp -/// @brief HeadersPropagatorComponent and default components - -#include - -#include - -USERVER_NAMESPACE_BEGIN - -namespace server::http { -class HeadersPropagator; -} // namespace server::http - -namespace components { - -// clang-format on - -/// @ingroup userver_components -/// -/// @brief Headers Propagator Component can scrape configured headers -/// and then enrich HttpClient request with them. -/// -/// The component can be configured in service config. -/// -/// ## Static options: -/// Name | Description | Default value -/// ---- | ----------- | ------------- -/// headers | List of headers to propagate | [] -/// -// clang-format on -class HeadersPropagatorComponent final : public components::ComponentBase { - public: - /// @ingroup userver_component_names - /// @brief The default name of components::HeadersPropagatorComponent - /// component - static constexpr std::string_view kName = "headers-propagator"; - - HeadersPropagatorComponent(const components::ComponentConfig&, - const components::ComponentContext&); - ~HeadersPropagatorComponent() override; - - static yaml_config::Schema GetStaticConfigSchema(); - - server::http::HeadersPropagator& Get(); - - private: - std::unique_ptr propagator_; -}; - -} // namespace components - -template <> -inline constexpr bool - components::kHasValidate = true; - -template <> -inline constexpr auto - components::kConfigFileMode = - ConfigFileMode::kNotRequired; - -USERVER_NAMESPACE_END diff --git a/core/include/userver/server/middlewares/headers_propagator.hpp b/core/include/userver/server/middlewares/headers_propagator.hpp new file mode 100644 index 000000000000..44b1010fbf6b --- /dev/null +++ b/core/include/userver/server/middlewares/headers_propagator.hpp @@ -0,0 +1,58 @@ +#pragma once + +#include + +USERVER_NAMESPACE_BEGIN + +namespace clients::http::plugins::headers_propagator { +class HeadersPropagator; +} // namespace clients::http::plugins::headers_propagator + +namespace server::middlewares { + +class HeadersPropagator final : public HttpMiddlewareBase { + public: + explicit HeadersPropagator(const handlers::HttpHandlerBase&, + const std::vector& headers); + + private: + void HandleRequest(http::HttpRequest& request, + request::RequestContext& context) const override; + + const std::vector& headers_; +}; + +class HeadersPropagatorFactory final : public HttpMiddlewareFactoryBase { + public: + static constexpr std::string_view kName = "headers-propagator"; + + HeadersPropagatorFactory(const components::ComponentConfig&, + const components::ComponentContext&); + + clients::http::plugins::headers_propagator::HeadersPropagator& Get(); + + static yaml_config::Schema GetStaticConfigSchema(); + + private: + std::unique_ptr Create( + const handlers::HttpHandlerBase&, + yaml_config::YamlConfig middleware_config) const override; + + std::vector headers_; + std::unique_ptr + propagator_; +}; + +} // namespace server::middlewares + +template <> +inline constexpr bool + components::kHasValidate = + true; + +template <> +inline constexpr auto + components::kConfigFileMode = + ConfigFileMode::kNotRequired; + +USERVER_NAMESPACE_END diff --git a/core/include/userver/server/request/task_inherited_headers.hpp b/core/include/userver/server/request/task_inherited_headers.hpp new file mode 100644 index 000000000000..25ee56b1c903 --- /dev/null +++ b/core/include/userver/server/request/task_inherited_headers.hpp @@ -0,0 +1,29 @@ +#pragma once + +/// @file userver/server/request/task_inherited_headers.hpp +/// @brief Functions that provide access to incoming headers stored in +/// TaskInheritedVariable. + +#include + +#include + +#include + +USERVER_NAMESPACE_BEGIN + +namespace server::request { + +using HeadersToPropagate = + utils::impl::TransparentMap; + +/// @brief Get a headers that is handled by the current task hierarchy. +boost::iterator_range +GetTaskInheritedHeaders(); + +/// @brief Set a headers that is handled by the current task hierarchy. +void SetTaskInheritedHeaders(HeadersToPropagate headers); + +} // namespace server::request + +USERVER_NAMESPACE_END diff --git a/core/include/userver/server/request/task_inherited_request.hpp b/core/include/userver/server/request/task_inherited_request.hpp index 547ad751f5a6..1b3fcf706e32 100644 --- a/core/include/userver/server/request/task_inherited_request.hpp +++ b/core/include/userver/server/request/task_inherited_request.hpp @@ -11,7 +11,11 @@ USERVER_NAMESPACE_BEGIN namespace http::headers { class PredefinedHeader; -} +} // namespace http::headers + +namespace server::http { +class HttpRequestImpl; +} // namespace server::http namespace server::request { diff --git a/core/src/clients/http/client.cpp b/core/src/clients/http/client.cpp index 0b6ebc7d1ee5..59288f0851d8 100644 --- a/core/src/clients/http/client.cpp +++ b/core/src/clients/http/client.cpp @@ -6,7 +6,6 @@ #include -#include #include #include #include @@ -21,7 +20,6 @@ #include #include #include -#include USERVER_NAMESPACE_BEGIN diff --git a/core/src/clients/http/component.cpp b/core/src/clients/http/component.cpp index a400bf9133c3..37e2afcba5b4 100644 --- a/core/src/clients/http/component.cpp +++ b/core/src/clients/http/component.cpp @@ -4,9 +4,9 @@ #include #include -#include #include #include +#include #include #include @@ -34,8 +34,8 @@ clients::http::ClientSettings GetClientSettings( auto& tracing_locator = context.FindComponent(); settings.tracing_manager = &tracing_locator.GetTracingManager(); - auto* propagator_component = - context.FindComponentOptional(); + auto* propagator_component = context.FindComponentOptional< + server::middlewares::HeadersPropagatorFactory>(); if (propagator_component) { settings.headers_propagator = &propagator_component->Get(); } diff --git a/core/src/clients/http/request.cpp b/core/src/clients/http/request.cpp index 0b786d9e8660..3f55401c443f 100644 --- a/core/src/clients/http/request.cpp +++ b/core/src/clients/http/request.cpp @@ -647,7 +647,8 @@ Request Request::SetTracingManager( } void Request::SetHeadersPropagator( - const server::http::HeadersPropagator* headers_propagator) & { + const clients::http::plugins::headers_propagator::HeadersPropagator* + headers_propagator) & { pimpl_->SetHeadersPropagator(headers_propagator); } diff --git a/core/src/clients/http/request_state.cpp b/core/src/clients/http/request_state.cpp index ddf437dd512b..bc60592daf2c 100644 --- a/core/src/clients/http/request_state.cpp +++ b/core/src/clients/http/request_state.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -1084,7 +1085,8 @@ void RequestState::SetTracingManager(const tracing::TracingManagerBase& m) { } void RequestState::SetHeadersPropagator( - const server::http::HeadersPropagator* propagator) { + const clients::http::plugins::headers_propagator::HeadersPropagator* + propagator) { headers_propagator_ = propagator; } diff --git a/core/src/clients/http/request_state.hpp b/core/src/clients/http/request_state.hpp index 0b24085a098c..8b03ad562040 100644 --- a/core/src/clients/http/request_state.hpp +++ b/core/src/clients/http/request_state.hpp @@ -32,7 +32,6 @@ #include #include #include -#include USERVER_NAMESPACE_BEGIN @@ -133,7 +132,8 @@ class RequestState : public std::enable_shared_from_this { void SetEasyTimeout(std::chrono::milliseconds timeout); void SetTracingManager(const tracing::TracingManagerBase&); - void SetHeadersPropagator(const server::http::HeadersPropagator*); + void SetHeadersPropagator( + const clients::http::plugins::headers_propagator::HeadersPropagator*); RequestTracingEditor GetEditableTracingInstance(); @@ -216,7 +216,8 @@ class RequestState : public std::enable_shared_from_this { bool deadline_expired_{false}; utils::NotNull tracing_manager_; - const server::http::HeadersPropagator* headers_propagator_{nullptr}; + const clients::http::plugins::headers_propagator::HeadersPropagator* + headers_propagator_{nullptr}; /// struct for reties struct { /// maximum number of retries diff --git a/core/src/components/headers_propagator_component.cpp b/core/src/components/headers_propagator_component.cpp deleted file mode 100644 index af9239ce4ed1..000000000000 --- a/core/src/components/headers_propagator_component.cpp +++ /dev/null @@ -1,44 +0,0 @@ -#include - -#include - -#include -#include -#include -#include - -USERVER_NAMESPACE_BEGIN - -namespace components { - -HeadersPropagatorComponent::HeadersPropagatorComponent( - const components::ComponentConfig& config, - const components::ComponentContext& context) - : components::ComponentBase(config, context), - propagator_(std::make_unique( - config["headers"].As>({}))) {} - -HeadersPropagatorComponent::~HeadersPropagatorComponent() = default; - -yaml_config::Schema HeadersPropagatorComponent::GetStaticConfigSchema() { - return yaml_config::MergeSchemas(R"( -type: object -description: component for propagating headers as-is -additionalProperties: false -properties: - headers: - type: array - description: array of headers to propagate - items: - type: string - description: header -)"); -} - -server::http::HeadersPropagator& HeadersPropagatorComponent::Get() { - return *propagator_; -} - -} // namespace components - -USERVER_NAMESPACE_END diff --git a/core/src/server/http/headers_propagator.cpp b/core/src/server/http/headers_propagator.cpp index becf3d9f7bad..8ea01dd060bb 100644 --- a/core/src/server/http/headers_propagator.cpp +++ b/core/src/server/http/headers_propagator.cpp @@ -1,24 +1,19 @@ -#include +#include -#include +#include USERVER_NAMESPACE_BEGIN -namespace server::http { - -HeadersPropagator::HeadersPropagator(std::vector&& headers) - : headers_(std::move(headers)) {} +namespace clients::http::plugins::headers_propagator { void HeadersPropagator::PropagateHeaders( clients::http::RequestTracingEditor request) const { - for (const auto& header : headers_) { - if (server::request::HasTaskInheritedHeader(header)) { - request.SetHeader(header, - server::request::GetTaskInheritedHeader(header)); - } + for (const auto& [header, value] : + server::request::GetTaskInheritedHeaders()) { + request.SetHeader(header, value); } } -} // namespace server::http +} // namespace clients::http::plugins::headers_propagator USERVER_NAMESPACE_END diff --git a/core/src/server/middlewares/headers_propagator.cpp b/core/src/server/middlewares/headers_propagator.cpp new file mode 100644 index 000000000000..67f425789333 --- /dev/null +++ b/core/src/server/middlewares/headers_propagator.cpp @@ -0,0 +1,66 @@ +#include + +#include + +#include +#include +#include +#include +#include + +USERVER_NAMESPACE_BEGIN + +namespace server::middlewares { + +HeadersPropagator::HeadersPropagator(const handlers::HttpHandlerBase&, + const std::vector& headers) + : headers_(headers) {} + +void HeadersPropagator::HandleRequest(http::HttpRequest& request, + request::RequestContext& context) const { + USERVER_NAMESPACE::server::request::HeadersToPropagate headers_to_propagate; + for (const auto& header_name : headers_) { + if (!request.HasHeader(header_name)) { + continue; + } + headers_to_propagate.emplace(header_name, request.GetHeader(header_name)); + } + USERVER_NAMESPACE::server::request::SetTaskInheritedHeaders( + headers_to_propagate); + Next(request, context); +} +HeadersPropagatorFactory::HeadersPropagatorFactory( + const components::ComponentConfig& config, + const components::ComponentContext& context) + : HttpMiddlewareFactoryBase(config, context), + headers_(config["headers"].As>({})), + propagator_(std::make_unique()) {} +std::unique_ptr HeadersPropagatorFactory::Create( + const handlers::HttpHandlerBase& handler, yaml_config::YamlConfig) const { + return std::make_unique(handler, headers_); +} + +clients::http::plugins::headers_propagator::HeadersPropagator& +HeadersPropagatorFactory::Get() { + return *propagator_; +} + +yaml_config::Schema HeadersPropagatorFactory::GetStaticConfigSchema() { + return yaml_config::MergeSchemas(R"( +type: object +description: Http service headers propagator middleware +additionalProperties: false +properties: + headers: + type: array + description: array of headers to propagate + items: + type: string + description: header +)"); +} + +} // namespace server::middlewares + +USERVER_NAMESPACE_END diff --git a/core/src/server/request/task_inherited_headers.cpp b/core/src/server/request/task_inherited_headers.cpp new file mode 100644 index 000000000000..7f8e0f3f1d33 --- /dev/null +++ b/core/src/server/request/task_inherited_headers.cpp @@ -0,0 +1,30 @@ +#include + +#include + +USERVER_NAMESPACE_BEGIN + +namespace server::request { + +namespace { +inline engine::TaskInheritedVariable + kTaskInheritedHeaders; +const server::request::HeadersToPropagate kEmptyHeaders; +} // namespace + +boost::iterator_range +GetTaskInheritedHeaders() { + const auto* headers_ptr = kTaskInheritedHeaders.GetOptional(); + if (headers_ptr == nullptr) { + return kEmptyHeaders; + } + return *headers_ptr; +} + +void SetTaskInheritedHeaders(HeadersToPropagate headers) { + kTaskInheritedHeaders.Set(std::move(headers)); +} + +} // namespace server::request + +USERVER_NAMESPACE_END diff --git a/grpc/functional_tests/basic_server/grpc_service.cpp b/grpc/functional_tests/basic_server/grpc_service.cpp index 8a1aa7c21c88..0b2e38b733cf 100644 --- a/grpc/functional_tests/basic_server/grpc_service.cpp +++ b/grpc/functional_tests/basic_server/grpc_service.cpp @@ -4,10 +4,16 @@ #include +#include +#include +#include #include #include +#include #include +#include #include +#include #include #include @@ -25,9 +31,21 @@ class GreeterServiceComponent final GreeterServiceComponent(const components::ComponentConfig& config, const components::ComponentContext& context) - : api::GreeterServiceBase::Component(config, context) {} + : api::GreeterServiceBase::Component(config, context), + echo_url_{config["echo-url"].As()}, + http_client_( + context.FindComponent().GetHttpClient()) {} void SayHello(SayHelloCall& call, api::GreetingRequest&& request) override; + + void CallEchoNobody(CallEchoNobodyCall& call, + api::GreetingRequest&& request) override; + + static yaml_config::Schema GetStaticConfigSchema(); + + private: + const std::string echo_url_; + clients::http::Client& http_client_; }; void GreeterServiceComponent::SayHello( @@ -38,6 +56,32 @@ void GreeterServiceComponent::SayHello( call.Finish(response); } +void GreeterServiceComponent::CallEchoNobody( + api::GreeterServiceBase::CallEchoNobodyCall& call, + samples::api::GreetingRequest&&) { + api::GreetingResponse response; + response.set_greeting("Call Echo Nobody"); + auto http_response = http_client_.CreateRequest() + .get(echo_url_) + .retry(1) + .timeout(std::chrono::seconds{5}) + .perform(); + http_response->raise_for_status(); + call.Finish(response); +} + +yaml_config::Schema GreeterServiceComponent::GetStaticConfigSchema() { + return yaml_config::MergeSchemas(R"( + type: object + description: HTTP echo without body component + additionalProperties: false + properties: + echo-url: + type: string + description: some other microservice listens on this URL + )"); +} + } // namespace samples int main(int argc, char* argv[]) { @@ -46,6 +90,10 @@ int main(int argc, char* argv[]) { .Append() .Append() .Append() - .Append(); + .Append() + .Append() + .Append() + .Append() + .Append(); return utils::DaemonMain(argc, argv, component_list); } diff --git a/grpc/functional_tests/basic_server/proto/samples/greeter.proto b/grpc/functional_tests/basic_server/proto/samples/greeter.proto index b239fa823b85..ab4702ff61f1 100644 --- a/grpc/functional_tests/basic_server/proto/samples/greeter.proto +++ b/grpc/functional_tests/basic_server/proto/samples/greeter.proto @@ -4,6 +4,7 @@ package samples.api; service GreeterService { rpc SayHello(GreetingRequest) returns(GreetingResponse) {} + rpc CallEchoNobody(GreetingRequest) returns(GreetingResponse) {} } message GreetingRequest { diff --git a/grpc/functional_tests/basic_server/static_config.yaml b/grpc/functional_tests/basic_server/static_config.yaml index 6a10a570482f..828b9c20a898 100644 --- a/grpc/functional_tests/basic_server/static_config.yaml +++ b/grpc/functional_tests/basic_server/static_config.yaml @@ -27,8 +27,22 @@ components_manager: # Our GreeterService implementation greeter-service: + echo-url: 'mockserver/v1/translations' # Some other microservice listens on this URL task-processor: main-task-processor - middlewares: [] + middlewares: + - grpc-server-headers-propagator + + http-client: + fs-task-processor: main-task-processor + + dns-client: + fs-task-processor: fs-task-processor + + headers-propagator: + headers: ["custom-header-1", "custom-header-2"] + + grpc-server-headers-propagator: + headers: ["custom-header-1", "custom-header-2"] server: listener: diff --git a/grpc/functional_tests/basic_server/tests-unix-socket/conftest.py b/grpc/functional_tests/basic_server/tests-unix-socket/conftest.py index 38040af039ac..cc4bfa037c88 100644 --- a/grpc/functional_tests/basic_server/tests-unix-socket/conftest.py +++ b/grpc/functional_tests/basic_server/tests-unix-socket/conftest.py @@ -5,7 +5,18 @@ pytest_plugins = ['pytest_userver.plugins.grpc'] -USERVER_CONFIG_HOOKS = ['prepare_service_config'] +USERVER_CONFIG_HOOKS = ['prepare_service_config', 'config_echo_url'] + + +@pytest.fixture(scope='session') +def config_echo_url(mockserver_info): + def _do_patch(config_yaml, config_vars): + components = config_yaml['components_manager']['components'] + components['greeter-service']['echo-url'] = mockserver_info.url( + '/test-service/echo-no-body', + ) + + return _do_patch @pytest.fixture(scope='session') diff --git a/grpc/functional_tests/basic_server/tests-unix-socket/test_header_propagation.py b/grpc/functional_tests/basic_server/tests-unix-socket/test_header_propagation.py new file mode 100644 index 000000000000..98c3520826b9 --- /dev/null +++ b/grpc/functional_tests/basic_server/tests-unix-socket/test_header_propagation.py @@ -0,0 +1,39 @@ +from samples import greeter_pb2 + + +HEADERS_TO_PROPAGATE = ['custom-header-1', 'custom-header-2'] +HEADERS_NOT_TO_PROPAGATE = [ + 'custom-header-1-non-prop', + 'custom-header-2-non-prop', +] + + +async def test_propagation_headers_mixed_from_grpc(grpc_client, mockserver): + headers = { + HEADERS_NOT_TO_PROPAGATE[0]: '1', + HEADERS_NOT_TO_PROPAGATE[1]: '2', + HEADERS_TO_PROPAGATE[0]: '3', + HEADERS_TO_PROPAGATE[1]: '4', + } + + @mockserver.json_handler('/test-service/echo-no-body') + async def nobody_handler(request): + assert ( + request.headers[HEADERS_TO_PROPAGATE[0]] + == headers[HEADERS_TO_PROPAGATE[0]] + ) + assert ( + request.headers[HEADERS_TO_PROPAGATE[1]] + == headers[HEADERS_TO_PROPAGATE[1]] + ) + assert HEADERS_NOT_TO_PROPAGATE[0] not in request.headers + assert HEADERS_NOT_TO_PROPAGATE[1] not in request.headers + return mockserver.make_response() + + metadata = [] + for key, value in headers.items(): + metadata.append((key, value)) + request = greeter_pb2.GreetingRequest(name='Python') + response = await grpc_client.CallEchoNobody(request, metadata=metadata) + assert response.greeting == 'Call Echo Nobody' + assert nobody_handler.times_called == 1 diff --git a/grpc/include/userver/ugrpc/server/impl/service_worker_impl.hpp b/grpc/include/userver/ugrpc/server/impl/service_worker_impl.hpp index 64e2608f78b0..ebb484ed9276 100644 --- a/grpc/include/userver/ugrpc/server/impl/service_worker_impl.hpp +++ b/grpc/include/userver/ugrpc/server/impl/service_worker_impl.hpp @@ -16,7 +16,6 @@ #include #include #include -#include #include #include #include diff --git a/grpc/include/userver/ugrpc/server/middlewares/headers_propagator/component.hpp b/grpc/include/userver/ugrpc/server/middlewares/headers_propagator/component.hpp new file mode 100644 index 000000000000..fd5b25f58e5f --- /dev/null +++ b/grpc/include/userver/ugrpc/server/middlewares/headers_propagator/component.hpp @@ -0,0 +1,37 @@ +#pragma once + +/// @file userver/ugrpc/server/middlewares/headers_propagator/component.hpp +/// @brief @copybrief +/// ugrpc::server::middlewares::headers_propagator::Component + +#include + +USERVER_NAMESPACE_BEGIN + +/// Server baggae middleware +namespace ugrpc::server::middlewares::headers_propagator { + +/// @ingroup userver_components userver_base_classes +/// +/// @brief Component for gRPC server headers_propagator +class Component final : public MiddlewareComponentBase { + public: + /// @ingroup userver_component_names + /// @brief The default name of + /// ugrpc::server::middlewares::headers_propagator::Component + static constexpr std::string_view kName = "grpc-server-headers-propagator"; + + Component(const components::ComponentConfig& config, + const components::ComponentContext& context); + + std::shared_ptr GetMiddleware() override; + + static yaml_config::Schema GetStaticConfigSchema(); + + private: + const components::ComponentConfig& config_; +}; + +} // namespace ugrpc::server::middlewares::headers_propagator + +USERVER_NAMESPACE_END diff --git a/grpc/src/ugrpc/server/middlewares/headers_propagator/component.cpp b/grpc/src/ugrpc/server/middlewares/headers_propagator/component.cpp new file mode 100644 index 000000000000..470fd96721aa --- /dev/null +++ b/grpc/src/ugrpc/server/middlewares/headers_propagator/component.cpp @@ -0,0 +1,37 @@ +#include + +#include +#include +#include + +USERVER_NAMESPACE_BEGIN + +namespace ugrpc::server::middlewares::headers_propagator { + +Component::Component(const components::ComponentConfig& config, + const components::ComponentContext& context) + : MiddlewareComponentBase(config, context), config_(config) {} + +std::shared_ptr Component::GetMiddleware() { + return std::make_shared( + config_["headers"].As>({})); +} + +yaml_config::Schema Component::GetStaticConfigSchema() { + return yaml_config::MergeSchemas(R"( +type: object +description: gRPC service metadata fields (headers) propagator middleware component +additionalProperties: false +properties: + headers: + type: array + description: array of metadata fields(headers) to propagate + items: + type: string + description: header +)"); +} + +} // namespace ugrpc::server::middlewares::headers_propagator + +USERVER_NAMESPACE_END diff --git a/grpc/src/ugrpc/server/middlewares/headers_propagator/middleware.cpp b/grpc/src/ugrpc/server/middlewares/headers_propagator/middleware.cpp new file mode 100644 index 000000000000..7f94246fd0ff --- /dev/null +++ b/grpc/src/ugrpc/server/middlewares/headers_propagator/middleware.cpp @@ -0,0 +1,35 @@ +#include "middleware.hpp" + +#include +#include +#include + +USERVER_NAMESPACE_BEGIN + +namespace ugrpc::server::middlewares::headers_propagator { + +Middleware::Middleware(const std::vector& headers) + : headers_(headers) {} + +void Middleware::Handle(MiddlewareCallContext& context) const { + auto& call = context.GetCall(); + const auto& server_context = call.GetContext(); + USERVER_NAMESPACE::server::request::HeadersToPropagate headers_to_propagate; + for (const auto& header_name : headers_) { + const auto* header_value = utils::FindOrNullptr( + server_context.client_metadata(), + grpc::string_ref{header_name.data(), header_name.size()}); + if (!header_value) { + continue; + } + headers_to_propagate.emplace(header_name, + ugrpc::impl::ToString(*header_value)); + } + USERVER_NAMESPACE::server::request::SetTaskInheritedHeaders( + headers_to_propagate); + context.Next(); +} + +} // namespace ugrpc::server::middlewares::headers_propagator + +USERVER_NAMESPACE_END diff --git a/grpc/src/ugrpc/server/middlewares/headers_propagator/middleware.hpp b/grpc/src/ugrpc/server/middlewares/headers_propagator/middleware.hpp new file mode 100644 index 000000000000..bb625cda7198 --- /dev/null +++ b/grpc/src/ugrpc/server/middlewares/headers_propagator/middleware.hpp @@ -0,0 +1,21 @@ +#pragma once + +#include + +USERVER_NAMESPACE_BEGIN + +namespace ugrpc::server::middlewares::headers_propagator { + +class Middleware final : public MiddlewareBase { + public: + explicit Middleware(const std::vector& headers); + + void Handle(MiddlewareCallContext& context) const override; + + private: + const std::vector headers_; +}; + +} // namespace ugrpc::server::middlewares::headers_propagator + +USERVER_NAMESPACE_END From 9a4e3af68d69df00a2f4ac0551ae2a21d3c1b95c Mon Sep 17 00:00:00 2001 From: kosstin Date: Sun, 28 Jul 2024 11:42:58 +0300 Subject: [PATCH 086/629] fix grpc: don't store ComponentConfig& in headers middleware 6933b5143bb2e789c941eef3307f1b1259850c2c --- .../userver/server/middlewares/headers_propagator.hpp | 4 ++-- core/src/server/middlewares/headers_propagator.cpp | 4 ++-- .../server/middlewares/headers_propagator/component.hpp | 2 +- .../server/middlewares/headers_propagator/component.cpp | 6 +++--- .../server/middlewares/headers_propagator/middleware.cpp | 4 ++-- .../server/middlewares/headers_propagator/middleware.hpp | 2 +- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/core/include/userver/server/middlewares/headers_propagator.hpp b/core/include/userver/server/middlewares/headers_propagator.hpp index 44b1010fbf6b..97a43f2a7285 100644 --- a/core/include/userver/server/middlewares/headers_propagator.hpp +++ b/core/include/userver/server/middlewares/headers_propagator.hpp @@ -13,13 +13,13 @@ namespace server::middlewares { class HeadersPropagator final : public HttpMiddlewareBase { public: explicit HeadersPropagator(const handlers::HttpHandlerBase&, - const std::vector& headers); + std::vector headers); private: void HandleRequest(http::HttpRequest& request, request::RequestContext& context) const override; - const std::vector& headers_; + std::vector headers_; }; class HeadersPropagatorFactory final : public HttpMiddlewareFactoryBase { diff --git a/core/src/server/middlewares/headers_propagator.cpp b/core/src/server/middlewares/headers_propagator.cpp index 67f425789333..2a8ccd8cda86 100644 --- a/core/src/server/middlewares/headers_propagator.cpp +++ b/core/src/server/middlewares/headers_propagator.cpp @@ -13,8 +13,8 @@ USERVER_NAMESPACE_BEGIN namespace server::middlewares { HeadersPropagator::HeadersPropagator(const handlers::HttpHandlerBase&, - const std::vector& headers) - : headers_(headers) {} + std::vector headers) + : headers_(std::move(headers)) {} void HeadersPropagator::HandleRequest(http::HttpRequest& request, request::RequestContext& context) const { diff --git a/grpc/include/userver/ugrpc/server/middlewares/headers_propagator/component.hpp b/grpc/include/userver/ugrpc/server/middlewares/headers_propagator/component.hpp index fd5b25f58e5f..56088b53b637 100644 --- a/grpc/include/userver/ugrpc/server/middlewares/headers_propagator/component.hpp +++ b/grpc/include/userver/ugrpc/server/middlewares/headers_propagator/component.hpp @@ -29,7 +29,7 @@ class Component final : public MiddlewareComponentBase { static yaml_config::Schema GetStaticConfigSchema(); private: - const components::ComponentConfig& config_; + const std::vector headers_; }; } // namespace ugrpc::server::middlewares::headers_propagator diff --git a/grpc/src/ugrpc/server/middlewares/headers_propagator/component.cpp b/grpc/src/ugrpc/server/middlewares/headers_propagator/component.cpp index 470fd96721aa..dcded42d7a04 100644 --- a/grpc/src/ugrpc/server/middlewares/headers_propagator/component.cpp +++ b/grpc/src/ugrpc/server/middlewares/headers_propagator/component.cpp @@ -10,11 +10,11 @@ namespace ugrpc::server::middlewares::headers_propagator { Component::Component(const components::ComponentConfig& config, const components::ComponentContext& context) - : MiddlewareComponentBase(config, context), config_(config) {} + : MiddlewareComponentBase(config, context), + headers_(config["headers"].As>({})) {} std::shared_ptr Component::GetMiddleware() { - return std::make_shared( - config_["headers"].As>({})); + return std::make_shared(headers_); } yaml_config::Schema Component::GetStaticConfigSchema() { diff --git a/grpc/src/ugrpc/server/middlewares/headers_propagator/middleware.cpp b/grpc/src/ugrpc/server/middlewares/headers_propagator/middleware.cpp index 7f94246fd0ff..775474366c8a 100644 --- a/grpc/src/ugrpc/server/middlewares/headers_propagator/middleware.cpp +++ b/grpc/src/ugrpc/server/middlewares/headers_propagator/middleware.cpp @@ -8,8 +8,8 @@ USERVER_NAMESPACE_BEGIN namespace ugrpc::server::middlewares::headers_propagator { -Middleware::Middleware(const std::vector& headers) - : headers_(headers) {} +Middleware::Middleware(std::vector headers) + : headers_(std::move(headers)) {} void Middleware::Handle(MiddlewareCallContext& context) const { auto& call = context.GetCall(); diff --git a/grpc/src/ugrpc/server/middlewares/headers_propagator/middleware.hpp b/grpc/src/ugrpc/server/middlewares/headers_propagator/middleware.hpp index bb625cda7198..80ab483e64f3 100644 --- a/grpc/src/ugrpc/server/middlewares/headers_propagator/middleware.hpp +++ b/grpc/src/ugrpc/server/middlewares/headers_propagator/middleware.hpp @@ -8,7 +8,7 @@ namespace ugrpc::server::middlewares::headers_propagator { class Middleware final : public MiddlewareBase { public: - explicit Middleware(const std::vector& headers); + explicit Middleware(std::vector headers); void Handle(MiddlewareCallContext& context) const override; From 7822b8463a98ecc5b354215b67fb9f39656c280d Mon Sep 17 00:00:00 2001 From: antonyzhilin Date: Mon, 29 Jul 2024 12:25:18 +0300 Subject: [PATCH 087/629] fix grpc: don't cause OOM in generic API by default 3b070279809da6b11ee97ebb5fcc08ce2c827d09 --- .../userver/utils/statistics/testing.hpp | 4 + core/utest/src/utils/statistics/testing.cpp | 28 ++++-- grpc/include/userver/ugrpc/client/generic.hpp | 9 +- .../include/userver/ugrpc/impl/statistics.hpp | 3 + .../userver/ugrpc/impl/statistics_scope.hpp | 10 ++- .../ugrpc/server/generic_service_base.hpp | 19 ++-- .../userver/ugrpc/server/impl/call_params.hpp | 5 ++ .../ugrpc/server/impl/service_worker_impl.hpp | 41 ++++----- grpc/include/userver/ugrpc/server/rpc.hpp | 59 ++++++++----- grpc/src/ugrpc/impl/statistics.cpp | 14 ++- grpc/src/ugrpc/impl/statistics_scope.cpp | 31 +++++-- .../server/impl/generic_service_worker.cpp | 7 +- .../deadline_propagation/middleware.cpp | 2 +- .../server/middlewares/log/middleware.cpp | 2 +- grpc/src/ugrpc/server/rpc.cpp | 19 +++- grpc/tests/generic_client_test.cpp | 41 +++++---- grpc/tests/generic_server_test.cpp | 87 ++++++++++++++----- .../grpc-generic-proxy/src/proxy_service.cpp | 7 ++ 18 files changed, 262 insertions(+), 126 deletions(-) diff --git a/core/utest/include/userver/utils/statistics/testing.hpp b/core/utest/include/userver/utils/statistics/testing.hpp index 9ab341fc30ea..6e9c7247ffe1 100644 --- a/core/utest/include/userver/utils/statistics/testing.hpp +++ b/core/utest/include/userver/utils/statistics/testing.hpp @@ -49,6 +49,10 @@ class Snapshot final { MetricValue SingleMetric(std::string path, std::vector