AVRO-4022: Add docker CI (#3037)

* Second try * Add OpenSSL 1.1 * Fix the toolchain * Move to pre-compiled OpenSSL and switch to MaybeXS for perl * Check if there are more issues * We need distutils * Add cargo to PATH * Cleanup * Bump to Java11 Co-authored-by: Martin Grigorov <[email protected]> --------- Co-authored-by: Martin Grigorov <[email protected]>
apache · Jul 24, 2024 · fc9380b · fc9380b
1 parent 85f99b0
commit fc9380b
Show file tree

Hide file tree

Showing 4 changed files with 63 additions and 26 deletions.
diff --git a/.github/workflows/test-docker.yml b/.github/workflows/test-docker.yml
@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: 'Docker tests'
+on:
+  workflow_dispatch:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+    paths:
+      - 'share/docker/*'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Run Docker tests
+        shell: bash
+        run: ./build.sh docker-test
diff --git a/build.sh b/build.sh
@@ -353,9 +353,15 @@ do
       ;;
 
     docker-test)
+      if [ -z "$BUILDPLATFORM" ]; then
+        export BUILDPLATFORM=$(docker info --format "{{.OSType}}/{{.Architecture}}")
+      fi
       tar -cf- share/docker/Dockerfile $DOCKER_EXTRA_CONTEXT |
-        DOCKER_BUILDKIT=1 docker build -t avro-test -f share/docker/Dockerfile -
-      docker run --rm -v "${PWD}:/avro${DOCKER_MOUNT_FLAG}" --env "JAVA=${JAVA:-8}" avro-test /avro/share/docker/run-tests.sh
+        DOCKER_BUILDKIT=1 docker build -t avro-test --build-arg BUILDPLATFORM="${BUILDPLATFORM}" -f share/docker/Dockerfile -
+      docker run --rm \
+        --volume "${PWD}:/avro${DOCKER_MOUNT_FLAG}" \
+        --volume "${PWD}/share/docker/m2/:/root/.m2/" \
+        --env "JAVA=${JAVA:-11}" avro-test /avro/share/docker/run-tests.sh
       ;;
 
     *)

diff --git a/share/docker/Dockerfile b/share/docker/Dockerfile
@@ -57,13 +57,13 @@ RUN apt-get -qqy update \
                                                  libssl-dev \
                                                  make \
                                                  mypy \
-                                                 openjdk-8-jdk \
                                                  openjdk-11-jdk \
                                                  openjdk-17-jdk \
                                                  openjdk-21-jdk \
                                                  perl \
                                                  python3 \
                                                  python3-pip \
+                                                 python3-distutils \
                                                  python3-setuptools \
                                                  python3-snappy \
                                                  python3-venv \
@@ -74,7 +74,9 @@ RUN apt-get -qqy update \
                                                  python3.7 \
                                                  python3.7-distutils \
                                                  python3.8 \
+                                                 python3.8-distutils \
                                                  python3.9 \
+                                                 python3.9-distutils \
                                                  source-highlight \
                                                  subversion \
                                                  valgrind \
@@ -92,6 +94,14 @@ RUN apt-get -qqy install --no-install-recommends libzstd-dev \
                                                  php-mbstring \
                                                  php-dev
 
+# Required for Dotnet
+RUN export ARCH="$(dpkg --print-architecture)"; \
+  export LIBSSL_DEB="libssl1.1_1.1.1f-1ubuntu2.22_$ARCH.deb"; \
+  wget -q "http://security.ubuntu.com/ubuntu/pool/main/o/openssl/$LIBSSL_DEB" || \
+    wget -q "http://ports.ubuntu.com/ubuntu-ports/pool/main/o/openssl/$LIBSSL_DEB"; \
+  dpkg -i "$LIBSSL_DEB"; \
+  rm "$LIBSSL_DEB"
+
 # Install a maven release  -------------------------------------------
 # Inspired from https://github.com/apache/accumulo-docker/blob/bbb9892e165d40fb35fa19f38929effc5d0c709b/Dockerfile#L30
 ENV MAVEN_VERSION 3.9.6
@@ -125,7 +135,7 @@ ENV PATH="/opt/maven/bin:${PATH}"
 # The node deprecation warnings cause a 20 second sleep.
 # But mom, I'm not even tired!
 RUN curl -sSL https://deb.nodesource.com/setup_14.x \
-  | sed 's/sleep 20/echo "But mom!"/' \
+  | sed 's/sleep/echo "But mom!"/' \
   | bash \
  && apt-get -qqy install nodejs \
  && apt-get -qqy clean \
@@ -163,7 +173,7 @@ RUN apt-get -qqy install --no-install-recommends libcompress-raw-zlib-perl \
                                                  libcpan-uploader-perl \
                                                  libencode-perl \
                                                  libio-string-perl \
-                                                 libjson-xs-perl \
+                                                 libjson-maybexs-perl \
                                                  libmodule-install-perl \
                                                  libmodule-install-readmefrompod-perl \
                                                  libobject-tiny-perl \
@@ -214,15 +224,7 @@ RUN gem install bundler --no-document && \
 
 # Install Rust
 RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain 1.73.0
-
-# Note: This "ubertool" container has 4 JDK versions:
-# - OpenJDK 8
-# - OpenJDK 11
-# - OpenJDK 17
-# - OpenJDK 21
-# - The root build.sh script switches between the versions according to
-#   the JAVA environment variable.
-
+ENV PATH $PATH:/root/.cargo/bin/
 
 # Install .NET SDK
 RUN cd /opt ; \

diff --git a/share/docker/m2/toolchains.xml b/share/docker/m2/toolchains.xml
@@ -17,22 +17,13 @@
   ~ limitations under the License.
 -->
 <toolchains>
-  <toolchain>
-    <type>jdk</type>
-    <provides>
-      <version>8</version>
-    </provides>
-    <configuration>
-      <jdkHome>/usr/lib/jvm/java-8-openjdk-amd64</jdkHome>
-    </configuration>
-  </toolchain>
   <toolchain>
     <type>jdk</type>
     <provides>
       <version>11</version>
     </provides>
     <configuration>
-      <jdkHome>/usr/lib/jvm/java-11-openjdk-amd64</jdkHome>
+      <jdkHome>/usr/lib/jvm/openjdk-11</jdkHome>
     </configuration>
   </toolchain>
   <toolchain>
@@ -41,7 +32,7 @@
       <version>17</version>
     </provides>
     <configuration>
-      <jdkHome>/usr/lib/jvm/java-17-openjdk-amd64</jdkHome>
+      <jdkHome>/usr/lib/jvm/openjdk-17</jdkHome>
     </configuration>
   </toolchain>
   <toolchain>
@@ -50,7 +41,7 @@
       <version>21</version>
     </provides>
     <configuration>
-      <jdkHome>/usr/lib/jvm/java-21-openjdk-amd64</jdkHome>
+      <jdkHome>/usr/lib/jvm/openjdk-21</jdkHome>
     </configuration>
   </toolchain>
 </toolchains>