OBSDATA-8616 Apply Confluent Patches on top of Druid 30.0.1

.github/CODEOWNERS

@@ -0,0 +1 @@
+*	@confluentinc/obs-data

.gitignore

@@ -8,7 +8,7 @@ target
 *.swo
 *.pyc
 .classpath
-.idea
+.idea/
 .project
 .PVS-Studio
 .settings/

.semaphore/semaphore.yml

@@ -0,0 +1,149 @@
+version: v1.0
+name: Apache Druid
+agent:
+  machine:
+    type: s1-prod-ubuntu20-04-amd64-1
+execution_time_limit:
+  hours: 3
+blocks:
+  - name: "Install"
+    task:
+      env_vars: &env_vars
+        - name: MVN
+          value: "mvn -B"
+        - name: MAVEN_OPTS
+          value: "-Dmaven.repo.local=.m2"
+        - name: MAVEN_SKIP
+          value: >
+            -Danimal.sniffer.skip=true
+            -Dcheckstyle.skip=true
+            -Ddruid.console.skip=true
+            -Denforcer.skip=true
+            -Dforbiddenapis.skip=true
+            -Dmaven.javadoc.skip=true
+            -Dpmd.skip=true
+            -Dspotbugs.skip=true
+
+        - name: MAVEN_SKIP_TESTS
+          value: "-DskipTests -Djacoco.skip=true"
+      prologue:
+        commands:
+          - echo $SEMAPHORE_WORKFLOW_ID
+          - sem-version java 17
+          - checkout
+      jobs:
+        - name: "Install"
+          commands:
+            # This is a change meant to validate semaphore public builds
+            # so thus removing configurations for Confluent's internal CodeArtifact
+            - rm ~/.m2/settings.xml
+            - >
+              MAVEN_OPTS="${MAVEN_OPTS} -Xmx3000m" ${MVN} clean install
+              -q -ff ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} -T1C
+            # downstream tests depend on artifacts installed by mvn install into .m2
+            # also cache target to avoid the cost of recompiling tests
+            - tar zcf cache-post-install.tgz .m2 target
+            - artifact push workflow cache-post-install.tgz
+
+  - name: "Tests"
+    task:
+      env_vars: *env_vars
+      prologue:
+        commands:
+          - echo $SEMAPHORE_WORKFLOW_ID
+          - sem-version java 17
+          - checkout
+          - artifact pull workflow cache-post-install.tgz
+          - tar zxf cache-post-install.tgz
+          # This is a change meant to validate semaphore public builds
+          # so thus removing configurations for Confluent's internal CodeArtifact
+          - rm ~/.m2/settings.xml
+      jobs:
+        - name: "animal sniffer checks"
+          commands:
+            - ${MVN} test-compile ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS}
+            - ${MVN} animal-sniffer:check --fail-at-end
+
+        - name: "checkstyle"
+          commands:
+            - ${MVN} checkstyle:checkstyle --fail-at-end
+
+        - name: "enforcer checks"
+          commands:
+            - ${MVN} enforcer:enforce --fail-at-end
+
+        - name: "forbidden api checks"
+          commands:
+            - ${MVN} test-compile ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS}
+            - ${MVN} forbiddenapis:check forbiddenapis:testCheck --fail-at-end
+
+        - name: "pmd checks"
+          commands:
+            - ${MVN} pmd:check --fail-at-end  # TODO: consider adding pmd:cpd-check
+
+        - name: "spotbugs checks"
+          commands:
+            - ${MVN} spotbugs:check --fail-at-end -pl '!benchmarks'
+
+        - name: "analyze dependencies"
+          commands:
+            - >
+              ${MVN} ${MAVEN_SKIP} dependency:analyze -DoutputXML=true -DignoreNonCompile=true -DfailOnWarning=true --fail-at-end || { echo "
+
+              The dependency analysis has found a dependency that is either:
+              1) Used and undeclared: These are available as a transitive dependency but should be explicitly
+              added to the POM to ensure the dependency version. The XML to add the dependencies to the POM is
+              shown above.
+              2) Unused and declared: These are not needed and removing them from the POM will speed up the build
+              and reduce the artifact size. The dependencies to remove are shown above.
+              If there are false positive dependency analysis warnings, they can be suppressed:
+              https://maven.apache.org/plugins/maven-dependency-plugin/analyze-mojo.html#usedDependencies
+              https://maven.apache.org/plugins/maven-dependency-plugin/examples/exclude-dependencies-from-dependency-analysis.html
+              For more information, refer to:
+              https://maven.apache.org/plugins/maven-dependency-plugin/analyze-mojo.html
+              " && false; }
+
+        - name: "Confluent Extensions"
+          env_vars:
+            - name: MAVEN_PROJECTS
+              value: extensions-contrib/confluent-extensions
+          commands: &run_tests
+            - >
+              MAVEN_OPTS="${MAVEN_OPTS} -Xmx1g" ${MVN} test -pl ${MAVEN_PROJECTS}
+              ${MAVEN_SKIP} -Dremoteresources.skip=true
+
+        - name: "Server"
+          env_vars:
+            - name: MAVEN_PROJECTS
+              value: server
+          commands: *run_tests
+
+        - name: "Processing"
+          env_vars:
+            - name: MAVEN_PROJECTS
+              value: processing
+          commands: *run_tests
+
+        - name: "Indexing Service"
+          env_vars:
+            - name: MAVEN_PROJECTS
+              value: indexing-service
+          commands: *run_tests
+
+        - name: "Kafka Indexing Service"
+          env_vars:
+            - name: MAVEN_PROJECTS
+              value: extensions-core/kafka-indexing-service
+          commands: *run_tests
+
+        - name: "Web Console"
+          env_vars:
+            - name: MAVEN_PROJECTS
+              value: web-console
+          commands: *run_tests
+
+        - name: "Other Tests"
+          env_vars:
+            - name: MAVEN_PROJECTS
+              value: '!server,!processing,!indexing-service,!extensions-core/kafka-indexing-service,!extensions-contrib/confluent-extensions,!integration-tests-ex/cases,!web-console'
+          commands: *run_tests

codestyle/checkstyle-suppressions.xml

@@ -53,4 +53,8 @@
   <suppress checks="ImportOrder" message="^'java\..*'.*" />
 
   <suppress checks="AvoidStaticImport" files="[\\/]src[\\/](test)[\\/]"/>
+
+  <!-- Confluent Extensions -->
+  <suppress checks="Header" files="[\\/]io[\\/]confluent[\\/]" />
+  <suppress checks="PackageName" files="[\\/]io[\\/]confluent[\\/]" />
 </suppressions>

distribution/docker/Dockerfile

@@ -18,6 +18,7 @@
 #
 
 ARG JDK_VERSION=17
+ARG BASE_IMAGE=gcr.io/distroless/java$JDK_VERSION-debian12
 
 # The platform is explicitly specified as x64 to build the Druid distribution.
 # This is because it's not able to build the distribution on arm64 due to dependency problem of web-console. See: https://github.com/apache/druid/issues/13012
@@ -36,7 +37,7 @@ RUN export DEBIAN_FRONTEND=noninteractive \
 COPY . /src
 WORKDIR /src
 RUN --mount=type=cache,target=/root/.m2 if [ "$BUILD_FROM_SOURCE" = "true" ]; then \
-      mvn -B -ff -q \
+      mvn -B -ff -q dependency:go-offline \
       install \
       -Pdist,bundle-contrib-exts \
       -Pskip-static-checks,skip-tests \
@@ -49,54 +50,78 @@ RUN --mount=type=cache,target=/root/.m2 VERSION=$(mvn -B -q org.apache.maven.plu
  && tar -zxf ./distribution/target/apache-druid-${VERSION}-bin.tar.gz -C /opt \
  && mv /opt/apache-druid-${VERSION} /opt/druid
 
-FROM alpine:3 as bash-static
+FROM busybox:1.34.1-glibc as busybox
+
+FROM $BASE_IMAGE
+
+LABEL maintainer="Apache Druid Developers <[email protected]>"
+
+USER root
+
+COPY --from=busybox /bin/busybox /busybox/busybox
+RUN ["/busybox/busybox", "sh", "-c", "if [ ! -x \"$(command -v bash)\" ]; then \
+                                        /busybox/busybox --install /bin; \
+                                      else \
+                                        rm /busybox/busybox; \
+                                      fi;"]
+# Predefined builtin arg, see: https://docs.docker.com/engine/reference/builder/#automatic-platform-args-in-the-global-scope
 ARG TARGETARCH
+
 #
 # Download bash-static binary to execute scripts that require bash.
 # Although bash-static supports multiple platforms, but there's no need for us to support all those platform, amd64 and arm64 are enough.
 #
 ARG BASH_URL_BASE="https://github.com/robxu9/bash-static/releases/download/5.1.016-1.2.3"
-RUN if [ "$TARGETARCH" = "arm64" ]; then \
-      BASH_URL="${BASH_URL_BASE}/bash-linux-aarch64" ; \
-    elif [ "$TARGETARCH" = "amd64" ]; then \
-      BASH_URL="${BASH_URL_BASE}/bash-linux-x86_64" ; \
+RUN if [ ! -x "$(command -v bash)" ]; then \
+      if [ "$TARGETARCH" = "arm64" ]; then \
+        BASH_URL="${BASH_URL_BASE}/bash-linux-aarch64" ; \
+      elif [ "$TARGETARCH" = "amd64" ]; then \
+        BASH_URL="${BASH_URL_BASE}/bash-linux-x86_64" ; \
+      else \
+        echo "Unsupported architecture ($TARGETARCH)" && exit 1; \
+      fi; \
+      echo "Downloading bash-static from ${BASH_URL}" \
+      && wget ${BASH_URL} -O /bin/bash \
+      && chmod 755 /bin/bash; \
+    fi;
+
+RUN if [ ! -x "$(command -v useradd)" ]; then \
+      addgroup -S -g 1000 druid \
+      && adduser -S -u 1000 -D -H -h /opt/druid -s /bin/sh -g '' -G druid druid; \
     else \
-      echo "Unsupported architecture ($TARGETARCH)" && exit 1; \
-    fi; \
-    echo "Downloading bash-static from ${BASH_URL}" \
-    && wget ${BASH_URL} -O /bin/bash
-
-FROM busybox:1.35.0-glibc as busybox
-
-FROM gcr.io/distroless/java$JDK_VERSION-debian12
-LABEL maintainer="Apache Druid Developers <[email protected]>"
-
-COPY --from=busybox /bin/busybox /busybox/busybox
-RUN ["/busybox/busybox", "--install", "/bin"]
-
-
-RUN addgroup -S -g 1000 druid \
- && adduser -S -u 1000 -D -H -h /opt/druid -s /bin/sh -g '' -G druid druid
-
-
-COPY --from=bash-static /bin/bash /bin/bash
-RUN chmod 755 /bin/bash
+      groupadd --system --gid 1000 druid \
+      && useradd --system --uid 1000 -M --home /opt/druid --shell /bin/sh -c '' --gid 1000 druid; \
+    fi;
 
+COPY --chown=druid:druid --from=builder /opt /opt
 COPY distribution/docker/druid.sh /druid.sh
 COPY distribution/docker/peon.sh /peon.sh
-COPY distribution/docker/deduplicate_jars.sh /deduplicate_jars.sh
 
 # create necessary directories which could be mounted as volume
-# copy and de-duplicate jars from builder in same layer to reduce image size
 #   /opt/druid/var is used to keep individual files(e.g. log) of each Druid service
 #   /opt/shared is used to keep segments and task logs shared among Druid services
-RUN --mount=type=bind,from=builder,source=/opt,target=/builder/opt \
- mkdir -p /opt/druid/var /opt/shared \
- && cp -r /builder/opt/druid /opt/ \
- && /deduplicate_jars.sh /opt/druid \
- && chown -R druid:druid /opt/druid /opt/shared \
+RUN mkdir /opt/druid/var /opt/shared \
+ && chown druid:druid /opt/druid/var /opt/shared \
  && chmod 775 /opt/druid/var /opt/shared
 
+# Install iproute2 to get the ip command needed to set config of druid.host IP address
+# Command needed in druid.sh Line 140;
+RUN if [ ! -x "$(command -v ip)" ]; then \
+      if [ -x "$(command -v apt)" ]; then \
+        apt update \
+        && apt install -y iproute2; \
+      else \
+        apk add iproute2; \
+      fi; \
+    fi;
+
+RUN if [ -x "$(command -v apt)" ]; then \
+    apt update \
+      && apt install -y curl htop strace bind-tools netcat ; \
+    else \
+      apk add --no-cache curl htop strace bind-tools netcat-openbsd  ; \
+    fi;
+
 USER druid
 VOLUME /opt/druid/var
 WORKDIR /opt/druid