Add log and metric for overlord segment cache (apache#17728)

Add metric `segment/metadataCache/transactions`
monetate · Feb 14, 2025 · 91cfdde · 91cfdde
1 parent d40cf14
commit 91cfdde
Show file tree

Hide file tree

Showing 4 changed files with 28 additions and 1 deletion.
diff --git a/docs/operations/metrics.md b/docs/operations/metrics.md
@@ -322,6 +322,7 @@ The following metrics are emitted only when [segment metadata caching](../config
 |`segment/used/count`|Number of used segments currently present in the metadata store.|`dataSource`|
 |`segment/unused/count`|Number of unused segments currently present in the metadata store.|`dataSource`|
 |`segment/pending/count`|Number of pending segments currently present in the metadata store.|`dataSource`|
+|`segment/metadataCache/transactions`|Number of read or write transactions performed on the cache for a single datasource.|`dataSource`|
 |`segment/metadataCache/sync/time`|Number of milliseconds taken for the cache to sync with the metadata store.||
 |`segment/metadataCache/deleted`|Total number of segments deleted from the cache during the latest sync.||
 |`segment/metadataCache/skipped`|Total number of unparseable segment records that were skipped in the latest sync.||

diff --git a/...src/main/java/org/apache/druid/metadata/segment/cache/HeapMemorySegmentMetadataCache.java b/...src/main/java/org/apache/druid/metadata/segment/cache/HeapMemorySegmentMetadataCache.java
@@ -138,8 +138,13 @@ public HeapMemorySegmentMetadataCache(
   @LifecycleStart
   public void start()
   {
+    if (!isCacheEnabled) {
+      log.info("Segment metadata cache is not enabled.");
+      return;
+    }
+
     synchronized (cacheStateLock) {
-      if (isCacheEnabled && currentCacheState == CacheState.STOPPED) {
+      if (currentCacheState == CacheState.STOPPED) {
         updateCacheState(CacheState.FOLLOWER, "Scheduling sync with metadata store");
         scheduleSyncWithMetadataStore(pollDuration.getMillis());
       }
@@ -200,6 +205,7 @@ public boolean isEnabled()
   public DatasourceSegmentCache getDatasource(String dataSource)
   {
     verifyCacheIsUsableAndAwaitSync();
+    emitMetric(dataSource, Metric.TRANSACTION_COUNT, 1);
     return getCacheForDatasource(dataSource);
   }
 

diff --git a/server/src/main/java/org/apache/druid/metadata/segment/cache/Metric.java b/server/src/main/java/org/apache/druid/metadata/segment/cache/Metric.java
@@ -47,6 +47,11 @@ private Metric()
   // CACHE METRICS
   private static final String METRIC_NAME_PREFIX = "segment/metadataCache/";
 
+  /**
+   * Number of transactions performed on the cache for a datasource.
+   */
+  public static final String TRANSACTION_COUNT = "transactions";
+
   /**
    * Time taken in milliseconds for the latest sync with metadata store.
    */

diff --git a/...test/java/org/apache/druid/metadata/segment/cache/HeapMemorySegmentMetadataCacheTest.java b/...test/java/org/apache/druid/metadata/segment/cache/HeapMemorySegmentMetadataCacheTest.java
@@ -32,6 +32,7 @@
 import org.apache.druid.metadata.PendingSegmentRecord;
 import org.apache.druid.metadata.SegmentsMetadataManagerConfig;
 import org.apache.druid.metadata.TestDerbyConnector;
+import org.apache.druid.query.DruidMetrics;
 import org.apache.druid.segment.TestDataSource;
 import org.apache.druid.segment.TestHelper;
 import org.apache.druid.segment.realtime.appenderator.SegmentIdWithShardSpec;
@@ -51,6 +52,7 @@
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 
 public class HeapMemorySegmentMetadataCacheTest
@@ -629,6 +631,19 @@ public void testSync_removesPendingSegment_ifNotPresentInMetadataStore()
     );
   }
 
+  @Test
+  public void testGetDatasource_increasesTransactionCount()
+  {
+    setupAndSyncCache();
+    cache.getDatasource(TestDataSource.WIKI);
+    cache.getDatasource(TestDataSource.WIKI);
+    serviceEmitter.verifyEmitted(
+        Metric.TRANSACTION_COUNT,
+        Map.of(DruidMetrics.DATASOURCE, TestDataSource.WIKI),
+        2
+    );
+  }
+
   private void insertSegmentsInMetadataStore(Set<DataSegmentPlus> segments)
   {
     final String table = derbyConnectorRule.metadataTablesConfigSupplier().get().getSegmentsTable();