Merge remote-tracking branch 'upstream/master' into feature-fix-msq-w…

…ith-hllsketch-in-realtime

# Conflicts:
#	extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestBase.java

.github/scripts/setup_generate_license.sh

@@ -18,6 +18,9 @@
 set -e
 
 sudo apt-get update && sudo apt-get install python3 -y
-curl https://bootstrap.pypa.io/pip/3.5/get-pip.py | sudo -H python3
+# creating python virtual env
+python3 -m venv ~/.python3venv
+source ~/.python3venv/bin/activate
+sudo apt install python3-pip
 pip3 install wheel  # install wheel first explicitly
 pip3 install --upgrade pyyaml

.github/scripts/unit_tests_script.sh

@@ -21,8 +21,8 @@ unset _JAVA_OPTIONS
 
 # Set MAVEN_OPTS for Surefire launcher.
 MAVEN_OPTS='-Xmx2500m' ${MVN} test -pl ${MAVEN_PROJECTS} \
-${MAVEN_SKIP} -Ddruid.generic.useDefaultValueForNull=${DRUID_USE_DEFAULT_VALUE_FOR_NULL} \
--DjfrProfilerArgLine="${JFR_PROFILER_ARG_LINE}"
+${MAVEN_SKIP} \
+-DjfrProfilerArgLine="${JFR_PROFILER_ARG_LINE}" -Pci
 sh -c "dmesg | egrep -i '(oom|out of memory|kill process|killed).*' -C 1 || exit 0"
 free -m
 ${MVN} -pl ${MAVEN_PROJECTS} jacoco:report || { echo "coverage_failure=false" >> "$GITHUB_ENV" && false; }

.github/workflows/codeql.yml

@@ -40,7 +40,7 @@ jobs:
     - uses: actions/setup-java@v4
       with:
         distribution: 'zulu'
-        java-version: '8'
+        java-version: '17'
         cache: 'maven'
 
     # Initializes the CodeQL tools for scanning.

.github/workflows/cron-job-its.yml

@@ -28,7 +28,7 @@ on:
 jobs:
   build:
     if: (github.event_name == 'schedule' && github.repository == 'apache/druid')
-    name: build (jdk8)
+    name: build (jdk17)
     runs-on: ubuntu-latest
     steps:
       - name: Checkout branch
@@ -37,7 +37,7 @@ jobs:
       - name: setup java
         uses: actions/setup-java@v4
         with:
-          java-version: '8'
+          java-version: '17'
           distribution: 'zulu'
 
       - name: Cache Maven m2 repository
@@ -60,8 +60,8 @@ jobs:
     uses: ./.github/workflows/reusable-standard-its.yml
     needs: build
     with:
-      build_jdk: 8
-      runtime_jdk: 11
+      build_jdk: 17
+      runtime_jdk: 17
       testing_groups: -Dgroups=${{ matrix.testing_group }}
       use_indexer: middleManager
       group: ${{ matrix.testing_group }}
@@ -74,8 +74,8 @@ jobs:
     uses: ./.github/workflows/reusable-standard-its.yml
     needs: build
     with:
-      build_jdk: 8
-      runtime_jdk: 11
+      build_jdk: 17
+      runtime_jdk: 17
       testing_groups: -Dgroups=${{ matrix.testing_group }}
       use_indexer: indexer
       group: ${{ matrix.testing_group }}
@@ -88,8 +88,8 @@ jobs:
     uses: ./.github/workflows/reusable-standard-its.yml
     needs: build
     with:
-      build_jdk: 8
-      runtime_jdk: 11
+      build_jdk: 17
+      runtime_jdk: 17
       testing_groups: -Dgroups=${{ matrix.testing_group }}
       use_indexer: middleManager
       override_config_path: ./environment-configs/test-groups/prepopulated-data
@@ -103,8 +103,8 @@ jobs:
     uses: ./.github/workflows/reusable-standard-its.yml
     needs: build
     with:
-      build_jdk: 8
-      runtime_jdk: 11
+      build_jdk: 17
+      runtime_jdk: 17
       testing_groups: -DexcludedGroups=batch-index,input-format,input-source,perfect-rollup-parallel-batch-index,kafka-index,query,query-retry,query-error,realtime-index,security,ldap-security,s3-deep-storage,gcs-deep-storage,azure-deep-storage,hdfs-deep-storage,s3-ingestion,kinesis-index,kinesis-data-format,kafka-transactional-index,kafka-index-slow,kafka-transactional-index-slow,kafka-data-format,hadoop-s3-to-s3-deep-storage,hadoop-s3-to-hdfs-deep-storage,hadoop-azure-to-azure-deep-storage,hadoop-azure-to-hdfs-deep-storage,hadoop-gcs-to-gcs-deep-storage,hadoop-gcs-to-hdfs-deep-storage,aliyun-oss-deep-storage,append-ingestion,compaction,high-availability,upgrade,shuffle-deep-store,custom-coordinator-duties
       use_indexer: ${{ matrix.indexer }}
       group: other
@@ -122,7 +122,7 @@ jobs:
       - name: setup java
         uses: actions/setup-java@v4
         with:
-          java-version: '8'
+          java-version: '17'
           distribution: 'zulu'
           cache: maven
 

.github/workflows/reusable-revised-its.yml

@@ -57,6 +57,19 @@ on:
       AWS_SECRET_ACCESS_KEY:
         required: false
         type: string
+      BACKWARD_COMPATIBILITY_IT_ENABLED:
+        required: false
+        type: string
+        default: false
+      DRUID_PREVIOUS_VERSION:
+        required: false
+        type: string
+      DRUID_PREVIOUS_VERSION_DOWNLOAD_URL:
+        required: false
+        type: string
+      DRUID_PREVIOUS_IT_IMAGE_NAME:
+        required: false
+        type: string
 
 env:
   MYSQL_DRIVER_CLASSNAME: ${{ inputs.mysql_driver }} # Used by tests to connect to metadata store directly.
@@ -106,6 +119,15 @@ jobs:
             ./druid-container-jdk${{ inputs.build_jdk }}.tar.gz
             ./integration-tests-ex/image/target/env.sh
 
+      - name: Retrieve previous version cached docker image
+        id: docker-restore-previous-version
+        if: ${{ inputs.BACKWARD_COMPATIBILITY_IT_ENABLED == 'true' }}
+        uses: actions/cache/restore@v4
+        with:
+          key: druid-container-jdk${{ inputs.build_jdk }}-version${{ inputs.DRUID_PREVIOUS_VERSION }}.tar.gz-${{ github.sha }}
+          path: |
+            ./druid-container-jdk${{ inputs.build_jdk }}-version${{ inputs.DRUID_PREVIOUS_VERSION }}.tar.gz           
+
       - name: Maven build
         if: steps.maven-restore.outputs.cache-hit != 'true' || ( steps.docker-restore.outputs.cache-hit != 'true' && steps.targets-restore.outputs.cache-hit != 'true' )
         run: |
@@ -115,13 +137,26 @@ jobs:
         if: steps.docker-restore.outputs.cache-hit != 'true' || steps.maven-restore.outputs.cache-hit != 'true'
         env:
           docker-restore: ${{ toJson(steps.docker-restore.outputs) }}
+          BACKWARD_COMPATIBILITY_IT_ENABLED: ${{ inputs.BACKWARD_COMPATIBILITY_IT_ENABLED }}
+          DRUID_PREVIOUS_VERSION: ${{ inputs.DRUID_PREVIOUS_VERSION }}
+          DRUID_PREVIOUS_VERSION_DOWNLOAD_URL: ${{ inputs.DRUID_PREVIOUS_VERSION_DOWNLOAD_URL }}
+          DRUID_PREVIOUS_IT_IMAGE_NAME: ${{ inputs.DRUID_PREVIOUS_IT_IMAGE_NAME }}
         run: |
           ./it.sh image
           source ./integration-tests-ex/image/target/env.sh
           docker tag $DRUID_IT_IMAGE_NAME $DRUID_IT_IMAGE_NAME-jdk${{ inputs.build_jdk }}
           echo $DRUID_IT_IMAGE_NAME
           docker save "$DRUID_IT_IMAGE_NAME" | gzip > druid-container-jdk${{ inputs.build_jdk }}.tar.gz
 
+      - name: Save previous version docker image
+        if: ${{ inputs.BACKWARD_COMPATIBILITY_IT_ENABLED == 'true' && (steps.docker-restore.outputs.cache-hit != 'true' || steps.maven-restore.outputs.cache-hit != 'true') }}
+        env:
+          docker-restore: ${{ toJson(steps.docker-restore.outputs) }}
+        run: |
+          docker tag ${{ inputs.DRUID_PREVIOUS_IT_IMAGE_NAME }} ${{ inputs.DRUID_PREVIOUS_IT_IMAGE_NAME }}-jdk${{ inputs.build_jdk }}-version${{ inputs.DRUID_PREVIOUS_VERSION }}
+          echo ${DRUID_PREVIOUS_IT_IMAGE_NAME}
+          docker save "${{ inputs.DRUID_PREVIOUS_IT_IMAGE_NAME }}" | gzip > druid-container-jdk${{ inputs.build_jdk }}-version${{ inputs.DRUID_PREVIOUS_VERSION }}.tar.gz
+
       - name: Stop and remove docker containers
         run: |
           echo "Force stopping all containers and pruning"
@@ -133,9 +168,21 @@ jobs:
           docker load --input druid-container-jdk${{ inputs.build_jdk }}.tar.gz
           docker images
 
+      - name: Load previous version docker image
+        if: ${{ inputs.BACKWARD_COMPATIBILITY_IT_ENABLED == 'true' }}
+        run: |
+          docker load --input druid-container-jdk${{ inputs.build_jdk }}-version${{ inputs.DRUID_PREVIOUS_VERSION }}.tar.gz
+          docker images
+
       - name: Run IT
         id: run-it
-        run: ${{ inputs.script }}
+        env:
+          BACKWARD_COMPATIBILITY_IT_ENABLED: ${{ inputs.BACKWARD_COMPATIBILITY_IT_ENABLED }}
+          DRUID_PREVIOUS_VERSION: ${{ inputs.DRUID_PREVIOUS_VERSION }}
+          DRUID_PREVIOUS_VERSION_DOWNLOAD_URL: ${{ inputs.DRUID_PREVIOUS_VERSION_DOWNLOAD_URL }}
+          DRUID_PREVIOUS_IT_IMAGE_NAME: ${{ inputs.DRUID_PREVIOUS_IT_IMAGE_NAME }}
+        run: |
+          ${{ inputs.script }}
 
       - name: Collect docker logs on failure
         if: ${{ failure() && steps.run-it.conclusion == 'failure' }}

.github/workflows/reusable-unit-tests.yml

@@ -21,11 +21,6 @@ on:
         required: true
         type: string
         description: 'JDK version used to test Druid'
-      sql_compatibility:
-        required: false
-        type: boolean
-        default: true
-        description: 'For SQL compatibility'
       module:
         required: true
         type: string
@@ -84,13 +79,6 @@ jobs:
         run: |
           export base_ref=${{ github.base_ref }}
           echo "GITHUB_BASE_REF=${base_ref}" >> $GITHUB_ENV
-          # If sql_compatibilty is true, we want to set default_value_for_null
-          # which enables compatibility mode
-          if (${{ inputs.sql_compatibility }} == true); then
-            echo "DRUID_USE_DEFAULT_VALUE_FOR_NULL=false" >> $GITHUB_ENV
-          else
-            echo "DRUID_USE_DEFAULT_VALUE_FOR_NULL=true" >> $GITHUB_ENV
-          fi
 
       - name: test profiling
         run: | 
@@ -117,6 +105,31 @@ jobs:
           MAVEN_PROJECTS: ${{ inputs.maven_projects }}
         run: ./.github/scripts/unit_tests_script.sh
 
+      - name: Check for dumps on failure
+        if: ${{ failure() }}
+        id: check_for_dumps
+        run: |
+          if test -n "$(find "${GITHUB_WORKSPACE}" \( -name '*.hprof' -or -name 'hs_err_pid*' -or -name 'replay_pid*' -or -regex '.*/core\.[0-9]*' \))"
+          then
+            echo "found_dumps=true" >> "$GITHUB_ENV"
+          else
+            echo "found_dumps=false" >> "$GITHUB_ENV"
+          fi
+
+      - name: Collect dumps if they exist on failure
+        if: ${{ failure() && env.found_dumps == 'true' }}
+        run: |
+          find "${GITHUB_WORKSPACE}" \
+            \( -name '*.hprof' -or -name 'hs_err_pid*' -or -name 'replay_pid*' -or -regex '.*/core\.[0-9]*' \) \
+            -exec tar -cvzf ${RUNNER_TEMP}/failure-dumps.tar.gz {} +
+
+      - name: Upload dumps to GitHub if they exist on failure
+        if: ${{ failure() && env.found_dumps == 'true' }}
+        uses: actions/upload-artifact@master
+        with:
+          name: Failure-${{ inputs.group }} failure dumps (Compile=jdk${{ inputs.build_jdk }}, Run=jdk${{ inputs.runtime_jdk }})
+          path: ${{ runner.temp }}/failure-dumps.tar.gz
+
       - name: set outputs on failure
         id: set_outputs
         if: ${{ failure() }}

.github/workflows/revised-its.yml

@@ -18,6 +18,24 @@
 name: "Revised ITs workflow"
 on:
   workflow_call:
+    inputs:
+      BACKWARD_COMPATIBILITY_IT_ENABLED:
+        description: "Flag for backward compatibility IT"
+        required: false
+        default: false
+        type: string
+      DRUID_PREVIOUS_VERSION:
+        description: "Previous druid versions to run the test against."
+        required: false
+        type: string
+      DRUID_PREVIOUS_VERSION_DOWNLOAD_URL:
+        description: "URL to download the previous druid version."
+        required: false
+        type: string
+      DRUID_PREVIOUS_IT_IMAGE_NAME:
+        description: "Druid previous version image name."
+        required: false
+        type: string
   workflow_dispatch:
 
 jobs:
@@ -48,8 +66,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        #jdk: [8, 11, 17]
-        jdk: [8]
+        # jdk: [11, 17]
+        jdk: [17]
         it: [HighAvailability, MultiStageQuery, Catalog, BatchIndex, MultiStageQueryWithMM, InputSource, InputFormat, Security, Query]
         #indexer: [indexer, middleManager]
         indexer: [middleManager]
@@ -68,8 +86,8 @@ jobs:
     uses: ./.github/workflows/reusable-revised-its.yml
     if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }}
     with:
-      build_jdk: 8
-      runtime_jdk: 11
+      build_jdk: 17
+      runtime_jdk: 17
       use_indexer: middleManager
       script: ./it.sh github S3DeepStorage
       it: S3DeepStorage
@@ -79,3 +97,24 @@ jobs:
       AWS_REGION: us-east-1
       AWS_ACCESS_KEY_ID: admin
       AWS_SECRET_ACCESS_KEY: miniopassword
+
+  backward-compatibility-it:
+    needs: changes
+    uses: ./.github/workflows/reusable-revised-its.yml
+    if: ${{ inputs.BACKWARD_COMPATIBILITY_IT_ENABLED == 'true' && (needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true') }}
+    with:
+      build_jdk: 17
+      runtime_jdk: 17
+      use_indexer: middleManager
+      script: ./it.sh github BackwardCompatibilityMain
+      it: BackwardCompatibilityMain
+      mysql_driver: com.mysql.jdbc.Driver
+      BACKWARD_COMPATIBILITY_IT_ENABLED: ${{ inputs.BACKWARD_COMPATIBILITY_IT_ENABLED }}
+      DRUID_PREVIOUS_VERSION: ${{ inputs.DRUID_PREVIOUS_VERSION }}
+      DRUID_PREVIOUS_VERSION_DOWNLOAD_URL: ${{ inputs.DRUID_PREVIOUS_VERSION_DOWNLOAD_URL }}
+      DRUID_PREVIOUS_IT_IMAGE_NAME: ${{ inputs.DRUID_PREVIOUS_IT_IMAGE_NAME }}
+      DRUID_CLOUD_BUCKET: druid-qa
+      DRUID_CLOUD_PATH: aws-${{ github.run_id }}-${{ github.run_attempt }}
+      AWS_REGION: us-east-1
+      AWS_ACCESS_KEY_ID: admin
+      AWS_SECRET_ACCESS_KEY: miniopassword

.github/workflows/stale.yml

@@ -52,7 +52,7 @@ jobs:
           stale-issue-label: stale
           stale-pr-label: stale
           exempt-issue-labels: 'Evergreen,Security,Bug,Proposal,Design Review,Improvement,Performance,Refactoring,Apache,Area - Automation/Static Analysis,Area - Batch Indexing,Area - Cache,Area - Deep Storage,Area - Dependencies,Area - Dependency Injection,Area - Dev,Area - Documentation,Area - Extension,Area - Kafka/Kinesis Indexing,Area - Lookups,Area - Metadata,Area - Metrics/Event Emitting,Area - Null Handling,Area - Operations,Area - Query UI,Area - Querying,Area - Router,Area - Segment Balancing/Coordination,Area - Segment Format and Ser/De,Area - SQL,Area - Testing,Area - Web Console,Area - Zookeeper/Curator,Compatibility,Contributions Welcome,Development Blocker,Ease of Use,Error handling,HTTP,Incompatible,Stable API'
-          exempt-pr-labels: 'Evergreen'
+          exempt-pr-labels: 'Evergreen,Area - Dependencies'
           exempt-milestones: true
           exempt-assignees: true
           ascending: true