Merge remote-tracking branch 'upstream/master'

317brian · Sep 18, 2024 · 044718b · 044718b
2 parents dd587d4 + d84d53c
commit 044718b
Show file tree

Hide file tree

Showing 992 changed files with 29,690 additions and 16,073 deletions.
diff --git a/.github/workflows/reusable-unit-tests.yml b/.github/workflows/reusable-unit-tests.yml
@@ -117,6 +117,28 @@ jobs:
           MAVEN_PROJECTS: ${{ inputs.maven_projects }}
         run: ./.github/scripts/unit_tests_script.sh
 
+      - name: Check for .hprof files on failure
+        if: ${{ failure() }}
+        id: check_for_heap_dump
+        run: |
+          if ls ${GITHUB_WORKSPACE}/target/*.hprof 1> /dev/null 2>&1; then
+            echo "found_hprof=true" >> "$GITHUB_ENV"
+          else
+            echo "found_hprof=false" >> "$GITHUB_ENV"
+          fi
+
+      - name: Collect tarball hprof dumps if they exist on failure
+        if: ${{ failure() && env.found_hprof == 'true' }}
+        run: |
+          tar cvzf ${RUNNER_TEMP}/hprof-dumps.tgz ${GITHUB_WORKSPACE}/target/*.hprof
+
+      - name: Upload hprof dumps to GitHub if they exist on failure
+        if: ${{ failure() && env.found_hprof == 'true' }}
+        uses: actions/upload-artifact@master
+        with:
+          name: Hprof-${{ inputs.group }} hprof dumps (Compile=jdk${{ inputs.build_jdk }}, Run=jdk${{ inputs.runtime_jdk }})
+          path: ${{ runner.temp }}/hprof-dumps.tgz
+
       - name: set outputs on failure
         id: set_outputs
         if: ${{ failure() }}

diff --git a/.idea/inspectionProfiles/Druid.xml b/.idea/inspectionProfiles/Druid.xml
diff --git a/benchmarks/pom.xml b/benchmarks/pom.xml
@@ -27,7 +27,7 @@
   <parent>
     <groupId>org.apache.druid</groupId>
     <artifactId>druid</artifactId>
-    <version>31.0.0-SNAPSHOT</version>
+    <version>32.0.0-SNAPSHOT</version>
   </parent>
 
   <dependencies>

diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/ExpressionAggregationBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/ExpressionAggregationBenchmark.java
@@ -36,7 +36,7 @@
 import org.apache.druid.segment.CursorBuildSpec;
 import org.apache.druid.segment.CursorHolder;
 import org.apache.druid.segment.QueryableIndex;
-import org.apache.druid.segment.QueryableIndexStorageAdapter;
+import org.apache.druid.segment.QueryableIndexCursorFactory;
 import org.apache.druid.segment.column.ValueType;
 import org.apache.druid.segment.generator.GeneratorColumnSchema;
 import org.apache.druid.segment.generator.GeneratorSchemaInfo;
@@ -161,9 +161,8 @@ public void queryUsingNative(Blackhole blackhole)
 
   private double compute(final Function<ColumnSelectorFactory, BufferAggregator> aggregatorFactory)
   {
-    final QueryableIndexStorageAdapter adapter = new QueryableIndexStorageAdapter(index);
-
-    try (final CursorHolder cursorHolder = adapter.makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
+    final QueryableIndexCursorFactory cursorFactory = new QueryableIndexCursorFactory(index);
+    try (final CursorHolder cursorHolder = cursorFactory.makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
       final Cursor cursor = cursorHolder.asCursor();
 
       final BufferAggregator bufferAggregator = aggregatorFactory.apply(cursor.getColumnSelectorFactory());

diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/ExpressionFilterBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/ExpressionFilterBenchmark.java
@@ -35,7 +35,7 @@
 import org.apache.druid.segment.CursorBuildSpec;
 import org.apache.druid.segment.CursorHolder;
 import org.apache.druid.segment.QueryableIndex;
-import org.apache.druid.segment.QueryableIndexStorageAdapter;
+import org.apache.druid.segment.QueryableIndexCursorFactory;
 import org.apache.druid.segment.column.ValueType;
 import org.apache.druid.segment.generator.GeneratorColumnSchema;
 import org.apache.druid.segment.generator.GeneratorSchemaInfo;
@@ -148,7 +148,9 @@ public void expressionFilter(Blackhole blackhole)
     final CursorBuildSpec buildSpec = CursorBuildSpec.builder()
                                                      .setFilter(expressionFilter.toFilter())
                                                      .build();
-    try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(buildSpec)) {
+    final QueryableIndexCursorFactory cursorFactory = new QueryableIndexCursorFactory(index);
+
+    try (final CursorHolder cursorHolder = cursorFactory.makeCursorHolder(buildSpec)) {
       final Cursor cursor = cursorHolder.asCursor();
 
 
@@ -166,7 +168,9 @@ public void nativeFilter(Blackhole blackhole)
     final CursorBuildSpec buildSpec = CursorBuildSpec.builder()
                                                      .setFilter(nativeFilter.toFilter())
                                                      .build();
-    try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(buildSpec)) {
+    final QueryableIndexCursorFactory cursorFactory = new QueryableIndexCursorFactory(index);
+
+    try (final CursorHolder cursorHolder = cursorFactory.makeCursorHolder(buildSpec)) {
       final Cursor cursor = cursorHolder.asCursor();
       final ColumnValueSelector selector = cursor.getColumnSelectorFactory().makeColumnValueSelector("x");
       while (!cursor.isDone()) {

diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/ExpressionSelectorBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/ExpressionSelectorBenchmark.java
@@ -42,9 +42,8 @@
 import org.apache.druid.segment.Cursors;
 import org.apache.druid.segment.DimensionSelector;
 import org.apache.druid.segment.QueryableIndex;
-import org.apache.druid.segment.QueryableIndexStorageAdapter;
+import org.apache.druid.segment.QueryableIndexCursorFactory;
 import org.apache.druid.segment.QueryableIndexTimeBoundaryInspector;
-import org.apache.druid.segment.StorageAdapter;
 import org.apache.druid.segment.VirtualColumns;
 import org.apache.druid.segment.column.ColumnHolder;
 import org.apache.druid.segment.column.ColumnType;
@@ -160,7 +159,8 @@ public void timeFloorUsingExpression(Blackhole blackhole)
                                                          )
                                                      )
                                                      .build();
-    try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(buildSpec)) {
+    final QueryableIndexCursorFactory cursorFactory = new QueryableIndexCursorFactory(index);
+    try (final CursorHolder cursorHolder = cursorFactory.makeCursorHolder(buildSpec)) {
       final Cursor cursor = cursorHolder.asCursor();
 
       final ColumnValueSelector selector = cursor.getColumnSelectorFactory().makeColumnValueSelector("v");
@@ -171,7 +171,8 @@ public void timeFloorUsingExpression(Blackhole blackhole)
   @Benchmark
   public void timeFloorUsingExtractionFn(Blackhole blackhole)
   {
-    try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
+    final QueryableIndexCursorFactory cursorFactory = new QueryableIndexCursorFactory(index);
+    try (final CursorHolder cursorHolder = cursorFactory.makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
       final Cursor cursor = cursorHolder.asCursor();
 
       final DimensionSelector selector = cursor
@@ -190,15 +191,15 @@ public void timeFloorUsingExtractionFn(Blackhole blackhole)
   @Benchmark
   public void timeFloorUsingCursor(Blackhole blackhole)
   {
-    final StorageAdapter adapter = new QueryableIndexStorageAdapter(index);
-    try (final CursorHolder cursorHolder = adapter.makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
+    final QueryableIndexCursorFactory cursorFactory = new QueryableIndexCursorFactory(index);
+    try (final CursorHolder cursorHolder = cursorFactory.makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
       final Cursor cursor = cursorHolder.asCursor();
       final CursorGranularizer granularizer = CursorGranularizer.create(
           cursor,
           QueryableIndexTimeBoundaryInspector.create(index),
           Cursors.getTimeOrdering(index.getOrdering()),
           Granularities.HOUR,
-          adapter.getInterval()
+          index.getDataInterval()
       );
       final Sequence<Long> results =
           Sequences.simple(granularizer.getBucketIterable())
@@ -241,7 +242,8 @@ public void timeFormatUsingExpression(Blackhole blackhole)
                                                      )
                                                      .build();
 
-    try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(buildSpec)) {
+    final QueryableIndexCursorFactory cursorFactory = new QueryableIndexCursorFactory(index);
+    try (final CursorHolder cursorHolder = cursorFactory.makeCursorHolder(buildSpec)) {
       final Cursor cursor = cursorHolder.asCursor();
       final DimensionSelector selector = cursor.getColumnSelectorFactory().makeDimensionSelector(
           DefaultDimensionSpec.of("v")
@@ -253,7 +255,8 @@ public void timeFormatUsingExpression(Blackhole blackhole)
   @Benchmark
   public void timeFormatUsingExtractionFn(Blackhole blackhole)
   {
-    try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
+    final QueryableIndexCursorFactory cursorFactory = new QueryableIndexCursorFactory(index);
+    try (final CursorHolder cursorHolder = cursorFactory.makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
       final Cursor cursor = cursorHolder.asCursor();
       final DimensionSelector selector = cursor
           .getColumnSelectorFactory()
@@ -284,7 +287,8 @@ public void strlenUsingExpressionAsLong(Blackhole blackhole)
                                                      )
                                                      .build();
 
-    try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(buildSpec)) {
+    final QueryableIndexCursorFactory cursorFactory = new QueryableIndexCursorFactory(index);
+    try (final CursorHolder cursorHolder = cursorFactory.makeCursorHolder(buildSpec)) {
       final Cursor cursor = cursorHolder.asCursor();
       final ColumnValueSelector selector = cursor.getColumnSelectorFactory().makeColumnValueSelector("v");
       consumeLong(cursor, selector, blackhole);
@@ -307,7 +311,8 @@ public void strlenUsingExpressionAsString(Blackhole blackhole)
                                                      )
                                                      .build();
 
-    try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(buildSpec)) {
+    final QueryableIndexCursorFactory cursorFactory = new QueryableIndexCursorFactory(index);
+    try (final CursorHolder cursorHolder = cursorFactory.makeCursorHolder(buildSpec)) {
       final Cursor cursor = cursorHolder.asCursor();
       final DimensionSelector selector = cursor
           .getColumnSelectorFactory()
@@ -320,7 +325,8 @@ public void strlenUsingExpressionAsString(Blackhole blackhole)
   @Benchmark
   public void strlenUsingExtractionFn(Blackhole blackhole)
   {
-    try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
+    final QueryableIndexCursorFactory cursorFactory = new QueryableIndexCursorFactory(index);
+    try (final CursorHolder cursorHolder = cursorFactory.makeCursorHolder(CursorBuildSpec.FULL_SCAN)) {
       final Cursor cursor = cursorHolder.asCursor();
       final DimensionSelector selector = cursor
           .getColumnSelectorFactory()
@@ -346,7 +352,8 @@ public void arithmeticOnLong(Blackhole blackhole)
                                                      )
                                                      .build();
 
-    try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(buildSpec)) {
+    final QueryableIndexCursorFactory cursorFactory = new QueryableIndexCursorFactory(index);
+    try (final CursorHolder cursorHolder = cursorFactory.makeCursorHolder(buildSpec)) {
       final Cursor cursor = cursorHolder.asCursor();
       final ColumnValueSelector selector = cursor.getColumnSelectorFactory().makeColumnValueSelector("v");
       consumeLong(cursor, selector, blackhole);
@@ -368,7 +375,8 @@ public void stringConcatAndCompareOnLong(Blackhole blackhole)
                                                          )
                                                      )
                                                      .build();
-    try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(buildSpec)) {
+    final QueryableIndexCursorFactory cursorFactory = new QueryableIndexCursorFactory(index);
+    try (final CursorHolder cursorHolder = cursorFactory.makeCursorHolder(buildSpec)) {
       final Cursor cursor = cursorHolder.asCursor();
       final ColumnValueSelector selector = cursor.getColumnSelectorFactory().makeColumnValueSelector("v");
       consumeLong(cursor, selector, blackhole);
@@ -390,7 +398,8 @@ public void caseSearched1(Blackhole blackhole)
                                                          )
                                                      )
                                                      .build();
-    try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(buildSpec)) {
+    final QueryableIndexCursorFactory cursorFactory = new QueryableIndexCursorFactory(index);
+    try (final CursorHolder cursorHolder = cursorFactory.makeCursorHolder(buildSpec)) {
       final Cursor cursor = cursorHolder.asCursor();
       final ColumnValueSelector selector = cursor.getColumnSelectorFactory().makeColumnValueSelector("v");
       consumeLong(cursor, selector, blackhole);
@@ -412,7 +421,8 @@ public void caseSearched2(Blackhole blackhole)
                                                          )
                                                      )
                                                      .build();
-    try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(buildSpec)) {
+    final QueryableIndexCursorFactory cursorFactory = new QueryableIndexCursorFactory(index);
+    try (final CursorHolder cursorHolder = cursorFactory.makeCursorHolder(buildSpec)) {
       final Cursor cursor = cursorHolder.asCursor();
       final ColumnValueSelector selector = cursor.getColumnSelectorFactory().makeColumnValueSelector("v");
       consumeLong(cursor, selector, blackhole);
@@ -447,7 +457,8 @@ public void caseSearched100(Blackhole blackhole)
                                                      )
                                                      .build();
 
-    try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(buildSpec)) {
+    final QueryableIndexCursorFactory cursorFactory = new QueryableIndexCursorFactory(index);
+    try (final CursorHolder cursorHolder = cursorFactory.makeCursorHolder(buildSpec)) {
       final Cursor cursor = cursorHolder.asCursor();
       final ColumnValueSelector selector = cursor.getColumnSelectorFactory().makeColumnValueSelector("v");
       consumeLong(cursor, selector, blackhole);
@@ -476,7 +487,8 @@ public void caseSearchedWithLookup(Blackhole blackhole)
                                                      )
                                                      .build();
 
-    try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(buildSpec)) {
+    final QueryableIndexCursorFactory cursorFactory = new QueryableIndexCursorFactory(index);
+    try (final CursorHolder cursorHolder = cursorFactory.makeCursorHolder(buildSpec)) {
       final Cursor cursor = cursorHolder.asCursor();
       final ColumnValueSelector selector = cursor.getColumnSelectorFactory().makeColumnValueSelector("v");
       consumeLong(cursor, selector, blackhole);
@@ -513,7 +525,8 @@ public void caseSearchedWithLookup2(Blackhole blackhole)
                                                      )
                                                      .build();
 
-    try (final CursorHolder cursorHolder = new QueryableIndexStorageAdapter(index).makeCursorHolder(buildSpec)) {
+    final QueryableIndexCursorFactory cursorFactory = new QueryableIndexCursorFactory(index);
+    try (final CursorHolder cursorHolder = cursorFactory.makeCursorHolder(buildSpec)) {
       final Cursor cursor = cursorHolder.asCursor();
       final ColumnValueSelector selector = cursor.getColumnSelectorFactory().makeColumnValueSelector("v");
       consumeLong(cursor, selector, blackhole);

diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/ExpressionVectorSelectorBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/ExpressionVectorSelectorBenchmark.java
@@ -35,7 +35,7 @@
 import org.apache.druid.segment.CursorBuildSpec;
 import org.apache.druid.segment.CursorHolder;
 import org.apache.druid.segment.QueryableIndex;
-import org.apache.druid.segment.QueryableIndexStorageAdapter;
+import org.apache.druid.segment.QueryableIndexCursorFactory;
 import org.apache.druid.segment.VirtualColumns;
 import org.apache.druid.segment.generator.GeneratorBasicSchemas;
 import org.apache.druid.segment.generator.GeneratorSchemaInfo;
@@ -155,7 +155,7 @@ public void scan(Blackhole blackhole)
                                                      .setVirtualColumns(virtualColumns)
                                                      .build();
     final CursorHolder cursorHolder = closer.register(
-        new QueryableIndexStorageAdapter(index).makeCursorHolder(buildSpec)
+        new QueryableIndexCursorFactory(index).makeCursorHolder(buildSpec)
     );
     if (vectorize) {
       VectorCursor cursor = cursorHolder.asVectorCursor();