Merge #2870

2870: refactor: replace metrics-rs with opentelemetry-rust r=doitian,driftluo a=yangby-cryptape ### Purpose - Remove all `tokio<1.0` dependencies. Reduce `502 - 483 = 19` dependencies. - Fix the metrics feature, it was broken since v0.43.0. ### Changes - **(BREAK CHANGES)** Due to the refactor of [`metrics-rs`] `>0.12.1`, we couldn't keep the same configurations as before. Since the metrics feature is an experimental and development-only feature, I just made some break changes. - According to the suggestion from `@doitian,` replaces [`metrics-rs`] with [`opentelemetry-rust`]. As [`opentelemetry-rust`] said, [it is not stable, too](https://docs.rs/opentelemetry/0.15.0/opentelemetry/#metrics), so I didn't change the APIs. I just wrote a series of compatible macros. Due to the difference of those two crates, the presentation of data couldn't be totally the same as before. We could tweak them later according to actual requirements. ### In the Further I just let it works again, it only supports export data over a Prometheus endpoint with a preset settings, now. Any more changes should be added later, according to actual requirements. [`opentelemetry-rust`]: https://github.com/open-telemetry/opentelemetry-rust [`metrics-rs`]: https://github.com/metrics-rs/metrics  --- This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/nervosnetwork/ckb/2870)  Co-authored-by: Boyu Yang <[email protected]>
nervosnetwork · Jul 26, 2021 · b8726f5 · b8726f5
2 parents ca4474d + 19f5362
commit b8726f5
Show file tree

Hide file tree

Showing 8 changed files with 196 additions and 495 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/freezer/src/freezer_files.rs b/freezer/src/freezer_files.rs
@@ -167,7 +167,7 @@ impl FreezerFiles {
         metrics!(
             gauge,
             "ckb-freezer.size",
-            (data_size as i64 + INDEX_ENTRY_SIZE as i64)
+            data_size as i64 + INDEX_ENTRY_SIZE as i64
         );
         Ok(())
     }
@@ -220,7 +220,7 @@ impl FreezerFiles {
             metrics!(
                 counter,
                 "ckb-freezer.read",
-                (size as u64 + 2 * INDEX_ENTRY_SIZE)
+                size as u64 + 2 * INDEX_ENTRY_SIZE
             );
             Ok(Some(data))
         } else {

diff --git a/util/metrics-config/Cargo.toml b/util/metrics-config/Cargo.toml
@@ -9,5 +9,4 @@ homepage = "https://github.com/nervosnetwork/ckb"
 repository = "https://github.com/nervosnetwork/ckb"
 
 [dependencies]
-log = { version = "0.4", features = ["serde"] }
 serde = { version = "1.0", features = ["derive"] }
diff --git a/util/metrics-config/src/lib.rs b/util/metrics-config/src/lib.rs
@@ -8,105 +8,40 @@ use std::collections::HashMap;
 
 use serde::{Deserialize, Serialize};
 
-pub use log::Level as LogLevel;
-
 /// The whole CKB metrics configuration.
 ///
-/// This struct is used to configure [CKB metrics service]:
-/// builds one [`metrics_runtime::Receiver`] and any number of [exporters]
+/// This struct is used to configure [CKB metrics service].
 ///
 /// # An example which is used in `ckb.toml`:
 /// ```toml
-/// [metrics]
-/// threads = 3
-/// histogram_window = 60
-/// histogram_granularity = 1
-/// upkeep_interval = 500
 /// [metrics.exporter.prometheus]
-/// target = { type = "http", listen_address = "127.0.0.1:8100" }
-/// format = { type = "prometheus" }
-/// [metrics.exporter.log_yaml]
-/// target = { type = "log", level = "warn", interval = 600 }
-/// format = { type = "yaml" }
-/// [metrics.exporter.log_json]
-/// target = { type = "log", level = "error", interval = 900 }
-/// format = { type = "json" }
+/// target = { type = "prometheus", listen_address = "127.0.0.1:8100" }
 /// ```
 ///
 /// [CKB metrics service]: ../ckb_metrics_service/index.html
-/// [`metrics_runtime::Receiver`]: https://docs.rs/metrics-runtime/0.13.1/metrics_runtime/struct.Receiver.html
-/// [exporters]: https://docs.rs/metrics-runtime/0.13.1/metrics_runtime/exporters/index.html
 #[derive(Default, Clone, Debug, Serialize, Deserialize)]
 #[serde(deny_unknown_fields)]
 pub struct Config {
-    /// How many threads are required for metrics service.
-    #[serde(default)]
-    pub threads: usize,
-
-    /// Sets the [histogram] window configuration in seconds.
-    ///
-    /// [histogram]: https://docs.rs/metrics-runtime/0.13.1/metrics_runtime/struct.Builder.html#method.histogram
-    #[serde(default)]
-    pub histogram_window: u64,
-    /// Sets the [histogram] granularity configuration in seconds.
-    ///
-    /// [histogram]: https://docs.rs/metrics-runtime/0.13.1/metrics_runtime/struct.Builder.html#method.histogram
-    #[serde(default)]
-    pub histogram_granularity: u64,
-    /// Sets the [upkeep interval] configuration in milliseconds.
-    ///
-    /// [upkeep interval]: https://docs.rs/metrics-runtime/0.13.1/metrics_runtime/struct.Builder.html#method.upkeep_interval
-    #[serde(default)]
-    pub upkeep_interval: u64,
-
     /// Stores all exporters configurations.
     #[serde(default)]
     pub exporter: HashMap<String, Exporter>,
 }
 
-/// The configuration of an [exporter].
-///
-/// [exporter]: https://docs.rs/metrics-runtime/0.13.1/metrics_runtime/exporters/index.html
+/// The configuration of an exporter.
 #[derive(Clone, Debug, Serialize, Deserialize)]
 pub struct Exporter {
     /// How to output the metrics data.
     pub target: Target,
-    /// The metrics output data in which format.
-    pub format: Format,
 }
 
 /// The target to output the metrics data.
 #[derive(Clone, Debug, Serialize, Deserialize)]
 #[serde(rename_all = "snake_case")]
 #[serde(tag = "type")]
 pub enum Target {
-    /// Outputs the metrics data into logs.
-    Log {
-        /// The log records will be output at which level.
-        level: LogLevel,
-        /// Outputs each log record after how many seconds.
-        interval: u64,
-    },
-    /// Outputs the metrics data through HTTP Protocol.
-    Http {
+    /// Outputs the metrics data through Prometheus.
+    Prometheus {
         /// The HTTP listen address.
         listen_address: String,
     },
 }
-
-/// Records the metrics data in which format.
-#[derive(Clone, Debug, Serialize, Deserialize)]
-#[serde(rename_all = "snake_case")]
-#[serde(tag = "type")]
-pub enum Format {
-    /// JSON format.
-    Json {
-        /// Sets whether or not to render the JSON as "pretty."
-        #[serde(default)]
-        pretty: bool,
-    },
-    /// YAML format.
-    Yaml,
-    /// Prometheus exposition format.
-    Prometheus,
-}
diff --git a/util/metrics-service/Cargo.toml b/util/metrics-service/Cargo.toml
@@ -10,7 +10,9 @@ repository = "https://github.com/nervosnetwork/ckb"
 
 [dependencies]
 ckb-metrics-config = { path = "../metrics-config", version = "= 0.100.0-pre" }
+ckb-logger = { path = "../logger", version = "= 0.100.0-pre" }
 ckb-async-runtime = { path = "../runtime", version = "= 0.100.0-pre" }
 ckb-util = { path = "..", version = "= 0.100.0-pre" }
-metrics-runtime = { package = "ckb-metrics-runtime", version = "0.13.1" }
-metrics-core = "0.5.2"
+opentelemetry-prometheus = "0.8"
+prometheus = "0.12"
+hyper = "0.14"
diff --git a/util/metrics-service/src/lib.rs b/util/metrics-service/src/lib.rs
@@ -1,16 +1,17 @@
 //! The service which handles the metrics data in CKB.
 
-use std::{net::SocketAddr, time::Duration};
+use std::{convert::Infallible, net::SocketAddr, sync::Arc};
 
-use metrics_core::Observe;
-use metrics_runtime::{
-    exporters::{HttpExporter, LogExporter},
-    observers::{JsonBuilder, PrometheusBuilder, YamlBuilder},
-    Receiver,
+use hyper::{
+    header::CONTENT_TYPE,
+    service::{make_service_fn, service_fn},
+    Body, Error as HyperError, Method, Request, Response, Server,
 };
+use opentelemetry_prometheus::PrometheusExporter;
+use prometheus::Encoder as _;
 
 use ckb_async_runtime::Handle;
-use ckb_metrics_config::{Config, Exporter, Format, Target};
+use ckb_metrics_config::{Config, Exporter, Target};
 use ckb_util::strings;
 
 /// Ensures the metrics service can shutdown gracefully.
@@ -30,98 +31,72 @@ pub fn init(config: Config, handle: Handle) -> Result<Guard, String> {
         return Ok(Guard::Off);
     }
 
-    let receiver = {
-        let histogram_window_secs = if config.histogram_window > 0 {
-            config.histogram_window
-        } else {
-            10
-        };
-        let histogram_granularity_secs = if config.histogram_granularity > 0 {
-            config.histogram_granularity
-        } else {
-            1
-        };
-        let upkeep_interval_millis = if config.upkeep_interval > 0 {
-            config.upkeep_interval
-        } else {
-            50
-        };
-        let histogram_window = Duration::from_secs(histogram_window_secs);
-        let histogram_granularity = Duration::from_secs(histogram_granularity_secs);
-        let upkeep_interval = Duration::from_millis(upkeep_interval_millis);
-        Receiver::builder()
-            .histogram(histogram_window, histogram_granularity)
-            .upkeep_interval(upkeep_interval)
-    }
-    .build()
-    .unwrap();
-    let controller = receiver.controller();
-
     for (name, exporter) in config.exporter {
         check_exporter_name(&name)?;
-        run_exporter(exporter, &handle, controller.clone())?;
+        run_exporter(exporter, &handle)?;
     }
 
-    receiver.install();
-
     Ok(Guard::On)
 }
 
 fn check_exporter_name(name: &str) -> Result<(), String> {
     strings::check_if_identifier_is_valid(name)
 }
 
-fn run_exporter<C>(exporter: Exporter, handle: &Handle, c: C) -> Result<(), String>
-where
-    C: Observe + Sync + Send + 'static,
-{
-    let Exporter { target, format } = exporter;
+fn run_exporter(exporter: Exporter, handle: &Handle) -> Result<(), String> {
+    let Exporter { target } = exporter;
     match target {
-        Target::Log {
-            level: lv,
-            interval,
-        } => {
-            let dur = Duration::from_secs(interval);
-            match format {
-                Format::Json { pretty } => {
-                    let b = JsonBuilder::new().set_pretty_json(pretty);
-                    let exporter = LogExporter::new(c, b, lv, dur);
-                    handle.spawn(exporter.async_run());
-                }
-                Format::Yaml => {
-                    let b = YamlBuilder::new();
-                    let exporter = LogExporter::new(c, b, lv, dur);
-                    handle.spawn(exporter.async_run());
-                }
-                Format::Prometheus => {
-                    let b = PrometheusBuilder::new();
-                    let exporter = LogExporter::new(c, b, lv, dur);
-                    handle.spawn(exporter.async_run());
-                }
-            };
-        }
-        Target::Http { listen_address } => {
+        Target::Prometheus { listen_address } => {
             let addr = listen_address
                 .parse::<SocketAddr>()
                 .map_err(|err| format!("failed to parse listen_address because {}", err))?;
-            match format {
-                Format::Json { pretty } => {
-                    let b = JsonBuilder::new().set_pretty_json(pretty);
-                    let exporter = HttpExporter::new(c, b, addr);
-                    handle.spawn(exporter.async_run());
-                }
-                Format::Yaml => {
-                    let b = YamlBuilder::new();
-                    let exporter = HttpExporter::new(c, b, addr);
-                    handle.spawn(exporter.async_run());
+            // TODO Not allow to configure the prometheus exporter, since the API is not stable.
+            // If anyone who want to customize the configurations, update the follow code.
+            // Ref: https://docs.rs/opentelemetry-prometheus/*/opentelemetry_prometheus/struct.ExporterBuilder.html
+            let exporter = {
+                let exporter = opentelemetry_prometheus::exporter()
+                    .try_init()
+                    .map_err(|err| format!("failed to init prometheus exporter because {}", err))?;
+                Arc::new(exporter)
+            };
+            let make_svc = make_service_fn(move |_conn| {
+                let exporter = Arc::clone(&exporter);
+                async move {
+                    Ok::<_, Infallible>(service_fn(move |req| {
+                        start_prometheus_service(req, Arc::clone(&exporter))
+                    }))
                 }
-                Format::Prometheus => {
-                    let b = PrometheusBuilder::new();
-                    let exporter = HttpExporter::new(c, b, addr);
-                    handle.spawn(exporter.async_run());
+            });
+            ckb_logger::info!("start prometheus exporter at {}", addr);
+            handle.spawn(async move {
+                let server = Server::bind(&addr).serve(make_svc);
+                if let Err(err) = server.await {
+                    ckb_logger::error!("prometheus server error: {}", err);
                 }
-            };
+            });
         }
     }
     Ok(())
 }
+
+async fn start_prometheus_service(
+    req: Request<Body>,
+    exporter: Arc<PrometheusExporter>,
+) -> Result<Response<Body>, HyperError> {
+    Ok(match (req.method(), req.uri().path()) {
+        (&Method::GET, "/") => {
+            let mut buffer = vec![];
+            let encoder = prometheus::TextEncoder::new();
+            let metric_families = exporter.registry().gather();
+            encoder.encode(&metric_families, &mut buffer).unwrap();
+            Response::builder()
+                .status(200)
+                .header(CONTENT_TYPE, encoder.format_type())
+                .body(Body::from(buffer))
+        }
+        _ => Response::builder()
+            .status(404)
+            .body(Body::from("Page Not Found")),
+    }
+    .unwrap())
+}
diff --git a/util/metrics/Cargo.toml b/util/metrics/Cargo.toml
@@ -9,4 +9,4 @@ homepage = "https://github.com/nervosnetwork/ckb"
 repository = "https://github.com/nervosnetwork/ckb"
 
 [dependencies]
-metrics = "0.12.1"
+opentelemetry = { version = "0.15", default-features=false, features = ["metrics"] }