【腾讯犀牛鸟开源课题实战】prometheus插件专项建设（PUSH模式支持等）

.bazelrc

@@ -3,3 +3,5 @@ build --copt=-O2
 #build --copt=-g --strip=never
 build --jobs 16
 #test --cache_test_results=no --test_output=errors
+
+build --define trpc_include_prometheus=true

LICENSE

@@ -453,6 +453,19 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 
 As an exception, if, as a result of your compiling your source code, portions of this Software are embedded into a machine-executable object form of such source code, you may redistribute such embedded portions in such object form without including the above copyright and permission notices.
 
+Open Source Software Licensed under the MIT with exceptions:
+--------------------------------------------------------------------
+1. jwt-cpp
+Copyright (c) 2018 Dominik Thalhammer
+
+Terms of the MIT with exceptions:
+--------------------------------------------------------------------
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
 
 Open Source Software Licensed under the OpenSSL License and the original SSLeay License:
 --------------------------------------------------------------------

docs/zh/prometheus_metrics.md

@@ -99,6 +99,7 @@ client:
 | aApp | 主调app名 |
 | aServer | 主调server名 |
 | aService | 主调service名 |
+| aIp | 主调ip地址 |
 | pApp | 被调app名 |
 | pServer | 被调server名 |
 | pService | 被调service名 |
@@ -109,6 +110,7 @@ client:
 | frame_ret_code | 调用的框架错误码 |
 | interface_ret_code | 调用的接口错误码 |
 
+
 ### 被调监控上报
 
 只需在框架配置文件的 `server` 中加上 `prometheus` 拦截器，即可开启被调监控：
@@ -124,7 +126,6 @@ server:
 
 统计数据：
 
-```mermaid
 | 监控名 | 监控类型 | 说明 |
 | ------ | ------ | ------ |
 | rpc_server_counter_metric | Counter | 服务端收到的请求总次数 |
@@ -149,7 +150,7 @@ server:
 | pConSetId | 被调所属set |
 | frame_ret_code | 调用的框架错误码 |
 | interface_ret_code | 调用的接口错误码 |
-```
+
 
 ## 属性监控上报
 
@@ -314,7 +315,7 @@ single_metrics_info.single_attr_info.value = 1;
 
 #### 通用多维属性上报
 
-Prometheus 监控插件支持框架通用的多维属性上报方式，即通过构造 `::trpc::TrpcMultiAttrMetricsInfo` 然后使用`::trpc::metrics::MultiAttrReport`接口来上报。**Prometheus 的单维属性上报是指上报统计标签包含多个键值对的数据。**。
+Prometheus 监控插件支持框架通用的多维属性上报方式，即通过构造 `::trpc::TrpcMultiAttrMetricsInfo` 然后使用`::trpc::metrics::MultiAttrReport`接口来上报。**Prometheus 的多维属性上报是指上报统计标签包含多个键值对的数据。**。
 
 设置 `::trpc::TrpcMultiAttrMetricsInfo` 值需要注意：
 
@@ -398,3 +399,78 @@ std::vector<::prometheus::MetricFamily> Collect();
 ## 通过 admin 获取
 
 如果服务开启了 [admin 功能](./admin_service.md)，则可以通过访问 `http://admin_ip:admin_port/metrics` 获取序列化为字符串后的 Prometheus 数据。
+
+# 鉴权
+
+Prometheus插件鉴权分为两种模式：pull模式 和 push模式，不同模式下的配置方式有所区别。
+
+## pull模式
+
+在pull模式下，使用Json Web Token（JWT）方式来鉴权。需要同时配置**trpc的Prometheus插件**和**Prometheus服务器**。
+
+### 插件配置
+
+插件配置样例如下：
+
+```yaml
+plugins:
+  metrics:
+    prometheus:
+      auth_cfg:
+        iss: admin # issuer 签发人
+        sub: prometheus-pull # subject 主题 
+        aud: trpc-server # audience 受众
+        secret: test # 密钥
+```
+
+### Prometheus服务器配置
+
+```yaml
+global:
+  scrape_interval:     15s
+  evaluation_interval: 15s
+
+scrape_configs:
+    - job_name: trpc-cpp 
+      static_configs:
+        - targets: ['127.0.0.1:8889']
+          labels:
+            instance: trpc-cpp 
+      bearer_token: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJwcm9tZXRoZXVzLXB1bGwiLCJhdWQiOiJ0cnBjLXNlcnZlciIsImlzcyI6ImFkbWluIiwiaWF0IjoxNTE2MjM5MDIyfQ.WWYY3jgxelzAXzX0IJSmZUeQeqb5YLV4oBAO7FTUI5o 
+```
+
+需要配置**bearer_token**字段，该token可以通过[JWT官方工具](https://jwt.io/)生成。在payload中填写相应的iss，sub和aud字段，verify signature中填写secret字段，加密算法使用默认的 HS256。
+
+## push模式
+
+在push模式下，为了和pushgateway兼容，鉴权使用**username**和**password**的形式。
+
+### 插件配置
+
+插件配置样例如下：
+
+```yaml
+plugins:
+  metrics:
+    prometheus:
+      auth_cfg:
+        username: admin
+        password: test
+```
+
+### Pushgateway服务器配置
+
+需要在Pushgateway服务器启动时，通过带有通过**bcrypt**加密的密文的配置文件启动。Pushgateway启动的配置文件如下：
+
+```yaml
+basic_auth_users:                                                                                                                                          
+  admin: $2y$05$5uq4H5p8JyfQm.e16o3xduW6tkI2bTRpArTK4MF4dEuvncpz/bqy.
+```
+
+密码的密文可以通过htpasswd工具生成：
+```shell
+> htpasswd -nbB admin test
+admin:$2y$05$5uq4H5p8JyfQm.e16o3xduW6tkI2bTRpArTK4MF4dEuvncpz/bqy.
+```
+
+

examples/features/prometheus/proxy/BUILD

@@ -44,3 +44,13 @@ cc_library(
         "@trpc_cpp//trpc/metrics/prometheus:prometheus_metrics_api",
     ],
 )
+
+cc_binary(
+    name = "push",
+    srcs = ["push.cc"],
+    deps = [
+        "@trpc_cpp//trpc/metrics/prometheus:prometheus_metrics_api",
+        "@trpc_cpp//trpc/log:trpc_log",
+
+    ],
+)

examples/features/prometheus/proxy/forward_service.cc

@@ -77,6 +77,12 @@ ::trpc::Status ForwardServiceImpl::Route(::trpc::ServerContextPtr context,
       "counter_name", "counter_desc", {{"const_counter_key", "const_counter_value"}});
   ::prometheus::Counter& counter = counter_family->Add({{"counter_key", "counter_value"}});
   counter.Increment(random_num);
+
+  if (::trpc::prometheus::PushMetricsInfo()) {
+        TRPC_FMT_INFO("Successfully pushed metrics to Pushgateway");
+    } else {
+        TRPC_FMT_ERROR("Failed to push metrics to Pushgateway");
+    }
 #endif
 
   auto client_context = ::trpc::MakeClientContext(context, greeter_proxy_);

examples/features/prometheus/proxy/push.cc

@@ -0,0 +1,22 @@
+#include <chrono>
+#include <thread>
+#include "trpc/metrics/prometheus/prometheus_metrics_api.h"
+#include "trpc/log/trpc_log.h"
+
+
+
+int main(int argc, char** argv) {
+
+   while (true) {
+        if (::trpc::prometheus::PushMetricsInfo())
+            {
+                std::cout << "Successfully pushed metrics to Pushgateway" << std::endl;
+            } else {
+                std::cerr << "Failed to push metrics to Pushgateway" << std::endl;
+            }
+
+        std::this_thread::sleep_for(std::chrono::seconds(5)); // 每60秒推送一次
+    }
+
+    return 0;
+}

examples/features/prometheus/proxy/trpc_cpp_fiber.yaml

@@ -44,6 +44,16 @@ plugins:
       const_labels:
         const_key1: const_value1
         const_key2: const_value2
+      push_mode:
+        enabled: true
+        gateway_url: "http://pushgateway:9091"
+        job_name: "test_job"
+        push_interval_seconds: 2
+      auth_cfg:
+        iss: admin 
+        sub: prometheus-pull
+        aud: trpc-server
+        secret: test
   log:
     default:
       - name: default

examples/features/prometheus/run.sh

@@ -6,7 +6,7 @@ echo "begin"
 sleep 1
 ./bazel-bin/examples/features/prometheus/proxy/forward_server --config=examples/features/prometheus/proxy/trpc_cpp_fiber.yaml &
 sleep 1
-./bazel-bin/examples/features/prometheus/client/client_config --config=examples/features/prometheus/client/trpc_cpp_fiber.yaml
+./bazel-bin/examples/features/prometheus/client/client --client_config=examples/features/prometheus/client/trpc_cpp_fiber.yaml
 
 killall helloworld_svr
 if [ $? -ne 0 ]; then

third_party/com_github_thalhammer_jwt_cpp/jwt_cpp.BUILD

@@ -0,0 +1,12 @@
+package(
+    default_visibility = ["//visibility:public"],
+)
+
+cc_library(
+    name = "jwt-cpp",
+    hdrs = glob(["**/*.h"]),
+    deps = [
+      "@com_github_openssl_openssl//:libcrypto",
+      "@com_github_openssl_openssl//:libssl",
+    ],
+)

trpc/admin/BUILD

@@ -354,13 +354,20 @@ cc_library(
         ":admin_handler",
         ":base_funcs",
         "//trpc/util:prometheus",
+        "//trpc/common/config:trpc_config",
+        "//trpc/log:trpc_log",
+        "//trpc/util/http:base64",
+        "//trpc/util/string:string_helper",
+        "//trpc/util:jwt",
     ] + select({
         "//conditions:default": [],
         "//trpc:trpc_include_prometheus": [
             "@com_github_jupp0r_prometheus_cpp//pull",
+            "//trpc/metrics/prometheus:prometheus_metrics",
         ],
         "//trpc:include_metrics_prometheus": [
             "@com_github_jupp0r_prometheus_cpp//pull",
+            "//trpc/metrics/prometheus:prometheus_metrics",
         ],
     }),
 )

trpc/admin/admin_service.cc

@@ -117,7 +117,9 @@ AdminService::AdminService() {
 
 #ifdef TRPC_BUILD_INCLUDE_PROMETHEUS
   // Prometheus metrics.
-  RegisterCmd(http::OperationType::GET, "/metrics", std::make_shared<admin::PrometheusHandler>());
+  auto prometheus_handle_ptr = std::make_shared<admin::PrometheusHandler>();
+  prometheus_handle_ptr->Init();
+  RegisterCmd(http::OperationType::GET, "/metrics", prometheus_handle_ptr);
 #endif
 
   RegisterCmd(http::OperationType::POST, "/client_detach", std::make_shared<admin::ClientDetachHandler>());

trpc/admin/prometheus_handler.cc

@@ -20,10 +20,87 @@ namespace trpc::admin {
 
 PrometheusHandler::PrometheusHandler() { description_ = "[GET /metrics] get prometheus metrics"; }
 
+void PrometheusHandler::Init() {
+  PrometheusConfig prometheus_conf;
+  bool ret = TrpcConfig::GetInstance()->GetPluginConfig<PrometheusConfig>(
+      "metrics", trpc::prometheus::kPrometheusMetricsName, prometheus_conf);
+  if (!ret) {
+    TRPC_LOG_WARN(
+        "Failed to obtain Prometheus plugin configuration from the framework configuration file. Default configuration "
+        "will be used.");
+  }
+  auth_cfg_ = prometheus_conf.auth_cfg;
+}
+
+bool PrometheusHandler::CheckTokenAuth(std::string bearer_token) {
+  auto splited = Split(bearer_token, ' ');
+  if (splited.size() != 2) {
+    TRPC_FMT_ERROR("error token: {}", bearer_token);
+    return false;
+  }
+  auto method = splited[0];
+  if (method != "Bearer") {
+    TRPC_FMT_ERROR("error auth method: {}", method);
+    return false;
+  }
+  std::string token = std::string(splited[1]);
+  if (!Jwt::isValid(token, auth_cfg_)) {
+    TRPC_FMT_ERROR("error token: {}", token);
+    return false;
+  }
+  return true;
+}
+
+bool PrometheusHandler::CheckBasicAuth(std::string token) {
+  auto splited = Split(token, ' ');
+  if (splited.size() != 2) {
+    TRPC_FMT_ERROR("error token: {}", token);
+    return false;
+  }
+  if (splited[0] != "Basic") {
+    TRPC_FMT_ERROR("error token: {}", token);
+    return false;
+  }
+
+  std::string username_pwd = http::Base64Decode(std::begin(splited[1]), std::end(splited[1]));
+  auto sp = Split(username_pwd, ':');
+  if (sp.size() != 2) {
+    TRPC_FMT_ERROR("error token: {}", token);
+    return false;
+  }
+
+  auto username = sp[0], pwd = sp[1];
+  if (username != auth_cfg_["username"] || pwd != auth_cfg_["password"]) {
+    TRPC_FMT_ERROR("error username or password: username: {}, password: {}", username, pwd);
+    return false;
+  }
+  return true;
+}
+
 void PrometheusHandler::CommandHandle(http::HttpRequestPtr req, rapidjson::Value& result,
                                       rapidjson::Document::AllocatorType& alloc) {
   static std::unique_ptr<::prometheus::Serializer> serializer = std::make_unique<::prometheus::TextSerializer>();
 
+  std::string token = req->GetHeader("authorization");
+
+  if (!auth_cfg_.empty()) {
+    if (auth_cfg_.count("username") && auth_cfg_.count("password")) {
+      // push mode
+      // use the basic auth if already config the username and password.
+      if (!CheckBasicAuth(token)) {
+        result.AddMember("message", "wrong request without right username or password", alloc);
+        return;
+      }
+    } else {
+      // pull mode
+      // use the json web token auth.
+      if (!CheckTokenAuth(token)) {
+        result.AddMember("message", "wrong request without right token", alloc);
+        return;
+      }
+    }
+  }
+
   std::string prometheus_str = serializer->Serialize(trpc::prometheus::Collect());
   result.AddMember(rapidjson::StringRef("trpc-html"), rapidjson::Value(prometheus_str, alloc).Move(), alloc);
 }

trpc/admin/prometheus_handler.h

@@ -15,7 +15,14 @@
 #pragma once
 
 #include "trpc/admin/admin_handler.h"
+#include "trpc/common/config/trpc_config.h"
+#include "trpc/log/trpc_log.h"
+#include "trpc/metrics/prometheus/prometheus_metrics.h"
+#include "trpc/util/http/base64.h"
+#include "trpc/util/jwt.h"
 #include "trpc/util/prometheus.h"
+#include "trpc/util/string/string_helper.h"
+#include "trpc/util/time.h"
 
 namespace trpc::admin {
 
@@ -24,8 +31,17 @@ class PrometheusHandler : public AdminHandlerBase {
  public:
   PrometheusHandler();
 
+  void Init();
+
   void CommandHandle(http::HttpRequestPtr req, rapidjson::Value& result,
                      rapidjson::Document::AllocatorType& alloc) override;
+
+ private:
+  bool CheckTokenAuth(std::string token);
+
+  bool CheckBasicAuth(std::string token);
+
+  std::map<std::string, std::string> auth_cfg_;
 };
 
 }  // namespace trpc::admin

trpc/common/plugin.h

@@ -71,7 +71,7 @@ class Plugin : public RefCounted<Plugin> {
 
   /// @brief Stop the runtime environment of the plugin
   virtual void Stop() noexcept {}
-
+  
   /// @brief destroy plugin internal resources
   virtual void Destroy() noexcept {}