From 4180b178a094e61f9baaa81ca4c54593bfdec5d3 Mon Sep 17 00:00:00 2001 From: danielle-tfh Date: Thu, 17 Oct 2024 15:09:12 +0200 Subject: [PATCH] Add metrics for serial id and event time --- Cargo.lock | 4 ++++ deploy/stage/common-values-iris-mpc.yaml | 2 +- .../stage/smpcv2-0-stage/values-iris-mpc.yaml | 17 +++++++++++++++++ .../stage/smpcv2-1-stage/values-iris-mpc.yaml | 17 +++++++++++++++++ .../stage/smpcv2-2-stage/values-iris-mpc.yaml | 17 +++++++++++++++++ iris-mpc-gpu/Cargo.toml | 2 ++ iris-mpc-gpu/src/server/actor.rs | 19 +++++++++++++------ iris-mpc/Cargo.toml | 2 ++ iris-mpc/src/bin/server.rs | 5 +++++ 9 files changed, 78 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 39c5731c4..753f488b6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2505,6 +2505,8 @@ dependencies = [ "iris-mpc-common", "iris-mpc-gpu", "iris-mpc-store", + "metrics 0.22.3", + "metrics-exporter-statsd 0.7.0", "ndarray", "rand", "serde_json", @@ -2605,6 +2607,8 @@ dependencies = [ "hex", "iris-mpc-common", "itertools 0.13.0", + "metrics 0.22.3", + "metrics-exporter-statsd 0.7.0", "ndarray", "num-traits", "rand", diff --git a/deploy/stage/common-values-iris-mpc.yaml b/deploy/stage/common-values-iris-mpc.yaml index 4e2401872..f3d5da972 100644 --- a/deploy/stage/common-values-iris-mpc.yaml +++ b/deploy/stage/common-values-iris-mpc.yaml @@ -1,4 +1,4 @@ -image: "ghcr.io/worldcoin/iris-mpc:v0.8.25" +image: "ghcr.io/worldcoin/iris-mpc:v0.8.26" environment: stage replicaCount: 1 diff --git a/deploy/stage/smpcv2-0-stage/values-iris-mpc.yaml b/deploy/stage/smpcv2-0-stage/values-iris-mpc.yaml index 503c0f400..768718cc2 100644 --- a/deploy/stage/smpcv2-0-stage/values-iris-mpc.yaml +++ b/deploy/stage/smpcv2-0-stage/values-iris-mpc.yaml @@ -77,6 +77,23 @@ env: - name: SMPC__MAX_BATCH_SIZE value: "64" + - name: SMPC__SERVICE__METRICS__HOST + valueFrom: + fieldRef: + fieldPath: status.hostIP + + - name: SMPC__SERVICE__METRICS__PORT + value: "8125" + + - name: SMPC__SERVICE__METRICS__QUEUE_SIZE + value: "5000" + + - name: SMPC__SERVICE__METRICS__BUFFER_SIZE + value: "1024" + + - name: SMPC__SERVICE__METRICS__PREFIX + value: "smpcv2-0" + initContainer: enabled: true image: "amazon/aws-cli:2.17.62" diff --git a/deploy/stage/smpcv2-1-stage/values-iris-mpc.yaml b/deploy/stage/smpcv2-1-stage/values-iris-mpc.yaml index c873dc1d9..200b9c405 100644 --- a/deploy/stage/smpcv2-1-stage/values-iris-mpc.yaml +++ b/deploy/stage/smpcv2-1-stage/values-iris-mpc.yaml @@ -77,6 +77,23 @@ env: - name: SMPC__MAX_BATCH_SIZE value: "64" + - name: SMPC__SERVICE__METRICS__HOST + valueFrom: + fieldRef: + fieldPath: status.hostIP + + - name: SMPC__SERVICE__METRICS__PORT + value: "8125" + + - name: SMPC__SERVICE__METRICS__QUEUE_SIZE + value: "5000" + + - name: SMPC__SERVICE__METRICS__BUFFER_SIZE + value: "1024" + + - name: SMPC__SERVICE__METRICS__PREFIX + value: "smpcv2-1" + initContainer: enabled: true image: "amazon/aws-cli:2.17.62" diff --git a/deploy/stage/smpcv2-2-stage/values-iris-mpc.yaml b/deploy/stage/smpcv2-2-stage/values-iris-mpc.yaml index 3aaa24a04..275d0934a 100644 --- a/deploy/stage/smpcv2-2-stage/values-iris-mpc.yaml +++ b/deploy/stage/smpcv2-2-stage/values-iris-mpc.yaml @@ -77,6 +77,23 @@ env: - name: SMPC__MAX_BATCH_SIZE value: "64" + - name: SMPC__SERVICE__METRICS__HOST + valueFrom: + fieldRef: + fieldPath: status.hostIP + + - name: SMPC__SERVICE__METRICS__PORT + value: "8125" + + - name: SMPC__SERVICE__METRICS__QUEUE_SIZE + value: "5000" + + - name: SMPC__SERVICE__METRICS__BUFFER_SIZE + value: "1024" + + - name: SMPC__SERVICE__METRICS__PREFIX + value: "smpcv2-2" + initContainer: enabled: true image: "amazon/aws-cli:2.17.62" diff --git a/iris-mpc-gpu/Cargo.toml b/iris-mpc-gpu/Cargo.toml index 813b3da87..9ee89c19e 100644 --- a/iris-mpc-gpu/Cargo.toml +++ b/iris-mpc-gpu/Cargo.toml @@ -29,6 +29,8 @@ serde_json.workspace = true sodiumoxide = "0.2.7" iris-mpc-common = { path = "../iris-mpc-common" } base64 = "0.22.1" +metrics = "0.22.1" +metrics-exporter-statsd = "0.7" [dev-dependencies] criterion = "0.5" diff --git a/iris-mpc-gpu/src/server/actor.rs b/iris-mpc-gpu/src/server/actor.rs index 5456142ab..1a678597c 100644 --- a/iris-mpc-gpu/src/server/actor.rs +++ b/iris-mpc-gpu/src/server/actor.rs @@ -33,6 +33,7 @@ use rand::{rngs::StdRng, SeedableRng}; use ring::hkdf::{Algorithm, Okm, Salt, HKDF_SHA256}; use std::{collections::HashMap, mem, sync::Arc, time::Instant}; use tokio::sync::{mpsc, oneshot}; +use tracing::Value; macro_rules! record_stream_time { ($manager:expr, $streams:expr, $map:expr, $label:expr, $block:block) => { @@ -883,20 +884,26 @@ impl ServerActor { // ---- END RESULT PROCESSING ---- log_timers(events); - + let processed_mil_elements_per_second = (self.max_batch_size * previous_total_db_size) + as f64 + / now.elapsed().as_secs_f64() + / 1e6; tracing::info!( "Batch took {:?} [{:.2} Melems/s]", now.elapsed(), - (self.max_batch_size * previous_total_db_size) as f64 - / now.elapsed().as_secs_f64() - / 1e6 + processed_mil_elements_per_second ); + metrics::histogram!("processed_melems_per_second") + .record(processed_mil_elements_per_second); + + let new_db_size = self.current_db_sizes.iter().sum::(); tracing::info!( "Old DB size: {}, New DB size: {}", previous_total_db_size, - self.current_db_sizes.iter().sum::() + new_db_size ); + metrics::gauge!("db_size").set(new_db_size as f64); Ok(()) } @@ -1277,7 +1284,7 @@ fn log_timers(events: HashMap<&str, Vec>>) { .sum(); tracing::info!("Event {}: {:?} ms", name, duration); - // TODO: send to metrics + metrics::histogram!("duration", "event_name" => name.to_string()).record(duration); } } diff --git a/iris-mpc/Cargo.toml b/iris-mpc/Cargo.toml index 8a39d5add..f91407733 100644 --- a/iris-mpc/Cargo.toml +++ b/iris-mpc/Cargo.toml @@ -32,6 +32,8 @@ iris-mpc-gpu = { path = "../iris-mpc-gpu" } iris-mpc-common = { path = "../iris-mpc-common" } iris-mpc-store = { path = "../iris-mpc-store" } sha2 = "0.10.8" +metrics = "0.22.1" +metrics-exporter-statsd = "0.7" [dev-dependencies] criterion = "0.5" diff --git a/iris-mpc/src/bin/server.rs b/iris-mpc/src/bin/server.rs index c347bf5d0..1b4dd66b4 100644 --- a/iris-mpc/src/bin/server.rs +++ b/iris-mpc/src/bin/server.rs @@ -841,6 +841,11 @@ async fn server_main(config: Config) -> eyre::Result<()> { tx.commit().await?; + for memory_serial_id in memory_serial_ids { + tracing::info!("Inserted serial_id: {}", memory_serial_id); + metrics::gauge!("results_inserted.latest_serial_id").set(memory_serial_id as f64); + } + tracing::info!("Sending {} uniqueness results", uniqueness_results.len()); send_results_to_sns( uniqueness_results,