feat: GFE Latency Metrics

google-cloud-spanner/src/main/java/com/google/cloud/spanner/MetricRegistryConstants.java

@@ -96,5 +96,16 @@ class MetricRegistryConstants {
       "The number of sessions released by the user and pool maintainer.";
   static final String NUM_SESSIONS_IN_POOL_DESCRIPTION = "The number of sessions in the pool.";
 
+  public static final String SPANNER_GFE_LATENCY_NAME = "gfe_latency";
+
+  public static final String SPANNER_GFE_LATENCY_DESCRIPTION =
+      "Latency between Google's network receiving an RPC and reading back the first byte of the response";
+
+  public static final String SPANNER_GFE_HEADER_MISSING_COUNT_NAME = "gfe_header_missing_count";
+
+  public static final String SPANNER_GFE_HEADER_MISSING_COUNT_DESCRIPTION =
+      "Number of RPC responses received without the server-timing header, most likely means that the RPC never reached Google's network";
+
+  public static final String MILLISECOND = "ms";
   static final String Scope = "cloud.google.com/java/spanner";
 }

google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java

@@ -1216,6 +1216,7 @@ public SpannerOptions build() {
             this.grpcGcpExtensionEnabled ? GRPC_GCP_ENABLED_DEFAULT_CHANNELS : DEFAULT_CHANNELS;
       }
 
+      SpannerRpcMetrics.initializeRPCMetrics(this.openTelemetry);
       return new SpannerOptions(this);
     }
   }

google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerRpcMetrics.java

@@ -0,0 +1,55 @@
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.cloud.spanner;
+
+import io.opentelemetry.api.OpenTelemetry;
+import io.opentelemetry.api.common.Attributes;
+import io.opentelemetry.api.metrics.LongCounter;
+import io.opentelemetry.api.metrics.LongHistogram;
+import io.opentelemetry.api.metrics.Meter;
+
+public class SpannerRpcMetrics {
+  private static LongHistogram gfeLatencies = null;
+  private static LongCounter gfeHeaderMissingCount = null;
+
+  static void initializeRPCMetrics(OpenTelemetry openTelemetry) {
+
+    if (openTelemetry != null) {
+      Meter meter = openTelemetry.getMeter(MetricRegistryConstants.Scope);
+      gfeLatencies =
+          meter
+              .histogramBuilder(MetricRegistryConstants.SPANNER_GFE_LATENCY_NAME)
+              .ofLongs()
+              .setDescription(MetricRegistryConstants.SPANNER_GFE_LATENCY_DESCRIPTION)
+              .setUnit(MetricRegistryConstants.MILLISECOND)
+              .build();
+      gfeHeaderMissingCount =
+          meter
+              .counterBuilder(MetricRegistryConstants.SPANNER_GFE_HEADER_MISSING_COUNT_NAME)
+              .setDescription(MetricRegistryConstants.SPANNER_GFE_HEADER_MISSING_COUNT_DESCRIPTION)
+              .setUnit(MetricRegistryConstants.COUNT)
+              .build();
+    }
+  }
+
+  public static void gfeLatencyRecorder(long value, Attributes attributes) {
+    if (gfeLatencies != null) gfeLatencies.record(value, attributes);
+  }
+
+  public static void gfeHeaderMissingCountRecorder(long value, Attributes attributes) {
+    if (gfeHeaderMissingCount != null) gfeHeaderMissingCount.add(value, attributes);
+  }
+}

google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/HeaderInterceptor.java

@@ -22,6 +22,7 @@
 import static com.google.cloud.spanner.spi.v1.SpannerRpcViews.SPANNER_GFE_HEADER_MISSING_COUNT;
 import static com.google.cloud.spanner.spi.v1.SpannerRpcViews.SPANNER_GFE_LATENCY;
 
+import com.google.cloud.spanner.SpannerRpcMetrics;
 import io.grpc.CallOptions;
 import io.grpc.Channel;
 import io.grpc.ClientCall;
@@ -37,6 +38,8 @@
 import io.opencensus.tags.TagValue;
 import io.opencensus.tags.Tagger;
 import io.opencensus.tags.Tags;
+import io.opentelemetry.api.common.Attributes;
+import io.opentelemetry.api.common.AttributesBuilder;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import java.util.regex.Matcher;
@@ -73,12 +76,13 @@ public <ReqT, RespT> ClientCall<ReqT, RespT> interceptCall(
       @Override
       public void start(Listener<RespT> responseListener, Metadata headers) {
         TagContext tagContext = getTagContext(headers, method.getFullMethodName());
+        Attributes attributes = getMetricAttributes(headers, method.getFullMethodName());
         super.start(
             new SimpleForwardingClientCallListener<RespT>(responseListener) {
               @Override
               public void onHeaders(Metadata metadata) {
 
-                processHeader(metadata, tagContext);
+                processHeader(metadata, tagContext, attributes);
                 super.onHeaders(metadata);
               }
             },
@@ -87,7 +91,7 @@ public void onHeaders(Metadata metadata) {
     };
   }
 
-  private void processHeader(Metadata metadata, TagContext tagContext) {
+  private void processHeader(Metadata metadata, TagContext tagContext, Attributes attributes) {
     MeasureMap measureMap = STATS_RECORDER.newMeasureMap();
     if (metadata.get(SERVER_TIMING_HEADER_KEY) != null) {
       String serverTiming = metadata.get(SERVER_TIMING_HEADER_KEY);
@@ -98,11 +102,15 @@ private void processHeader(Metadata metadata, TagContext tagContext) {
           measureMap.put(SPANNER_GFE_LATENCY, latency);
           measureMap.put(SPANNER_GFE_HEADER_MISSING_COUNT, 0L);
           measureMap.record(tagContext);
+
+          SpannerRpcMetrics.gfeLatencyRecorder(latency, attributes);
+          SpannerRpcMetrics.gfeHeaderMissingCountRecorder(0L, attributes);
         } catch (NumberFormatException e) {
           LOGGER.log(LEVEL, "Invalid server-timing object in header", matcher.group("dur"));
         }
       }
     } else {
+      SpannerRpcMetrics.gfeHeaderMissingCountRecorder(1L, attributes);
       measureMap.put(SPANNER_GFE_HEADER_MISSING_COUNT, 1L).record(tagContext);
     }
   }
@@ -139,4 +147,32 @@ private TagContext getTagContext(Metadata headers, String method) {
     }
     return getTagContext(method, projectId, instanceId, databaseId);
   }
+
+  private Attributes getMetricAttributes(Metadata headers, String method) {
+    String projectId = "undefined-project";
+    String instanceId = "undefined-database";
+    String databaseId = "undefined-database";
+    if (headers.get(GOOGLE_CLOUD_RESOURCE_PREFIX_KEY) != null) {
+      String googleResourcePrefix = headers.get(GOOGLE_CLOUD_RESOURCE_PREFIX_KEY);
+      Matcher matcher = GOOGLE_CLOUD_RESOURCE_PREFIX_PATTERN.matcher(googleResourcePrefix);
+      if (matcher.find()) {
+        projectId = matcher.group("project");
+        if (matcher.group("instance") != null) {
+          instanceId = matcher.group("instance");
+        }
+        if (matcher.group("database") != null) {
+          databaseId = matcher.group("database");
+        }
+      } else {
+        LOGGER.log(LEVEL, "Error parsing google cloud resource header: " + googleResourcePrefix);
+      }
+    }
+    AttributesBuilder attributesBuilder = Attributes.builder();
+    attributesBuilder.put("database", databaseId);
+    attributesBuilder.put("instance_id", instanceId);
+    attributesBuilder.put("project_id", projectId);
+    attributesBuilder.put("method", method);
+
+    return attributesBuilder.build();
+  }
 }