Merge pull request DSpace#9973 from tdonohue/add_docker_deploy_job

[GitHub Actions] Add a job to test Docker deployment with newly built images & test Handle Server

.github/workflows/docker.yml

@@ -147,4 +147,89 @@ jobs:
       tags_flavor: suffix=-loadsql
     secrets:
       DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
-      DOCKER_ACCESS_TOKEN: ${{ secrets.DOCKER_ACCESS_TOKEN }}
+      DOCKER_ACCESS_TOKEN: ${{ secrets.DOCKER_ACCESS_TOKEN }}
+
+  ########################################################################
+  # Test Deployment via Docker to ensure images are working properly
+  ########################################################################
+  docker-deploy:
+    # Ensure this job never runs on forked repos. It's only executed for 'dspace/dspace'
+    if: github.repository == 'dspace/dspace'
+    runs-on: ubuntu-latest
+    # Must run after all major images are built
+    needs: [dspace, dspace-test, dspace-cli, dspace-postgres-pgcrypto, dspace-solr]
+    env:
+      # Override defaults dspace.server.url because backend starts at http://127.0.0.1:8080
+      dspace__P__server__P__url: http://127.0.0.1:8080/server
+      # Force using "pr-testing" version of all Docker images. The "pr-testing" tag is a temporary tag that we
+      # assign to all PR-built docker images in reusabe-docker-build.yml
+      DSPACE_VER: pr-testing
+    steps:
+      # Checkout our codebase (to get access to Docker Compose scripts)
+      - name: Checkout codebase
+        uses: actions/checkout@v4
+      # For PRs, download Docker image artifacts (built by reusable-docker-build.yml for all PRs)
+      - name: Download Docker image artifacts (for PRs)
+        if: github.event_name == 'pull_request'
+        uses: actions/download-artifact@v4
+        with:
+          # Download all Docker images (TAR files) into the /tmp/docker directory
+          pattern: docker-image-*
+          path: /tmp/docker
+          merge-multiple: true
+      # For PRs, load each of the images into Docker by calling "docker image load" for each.
+      # This ensures we are using the images built from this PR & not the prior versions on DockerHub
+      - name: Load all downloaded Docker images (for PRs)
+        if: github.event_name == 'pull_request'
+        run: |
+          find /tmp/docker -type f -name "*.tar" -exec docker image load --input "{}" \;
+          docker image ls -a
+      # Start backend using our compose script in the codebase.
+      - name: Start backend in Docker
+        run: |
+          docker compose -f docker-compose.yml up -d
+          sleep 10
+          docker container ls
+      # Create a test admin account. Load test data from a simple set of AIPs as defined in cli.ingest.yml
+      - name: Load test data into Backend
+        run: |
+          docker compose -f docker-compose-cli.yml run --rm dspace-cli create-administrator -e [email protected] -f admin -l user -p admin -c en
+          docker compose -f docker-compose-cli.yml -f dspace/src/main/docker-compose/cli.ingest.yml run --rm dspace-cli
+      # Verify backend started successfully.
+      # 1. Make sure root endpoint is responding (check for dspace.name defined in docker-compose.yml)
+      # 2. Also check /collections endpoint to ensure the test data loaded properly (check for a collection name in AIPs)
+      - name: Verify backend is responding properly
+        run: |
+          result=$(wget -O- -q http://127.0.0.1:8080/server/api)
+          echo "$result"
+          echo "$result" |  grep -oE "\"DSpace Started with Docker Compose\","
+          result=$(wget -O- -q http://127.0.0.1:8080/server/api/core/collections)
+          echo "$result"
+          echo "$result" |  grep -oE "\"Dog in Yard\","
+      # Verify Handle Server can be stared and is working properly
+      # 1. First generate the "[dspace]/handle-server" folder with the sitebndl.zip
+      # 2. Start the Handle Server (and wait 20 seconds to let it start up)
+      # 3. Verify logs do NOT include "Exception" in the text (as that means an error occurred)
+      # 4. Check that Handle Proxy HTML page is responding on default port (8000)
+      - name: Verify Handle Server is working properly
+        run: |
+          docker exec -i dspace /dspace/bin/make-handle-config
+          echo "Starting Handle Server..."
+          docker exec -i dspace /dspace/bin/start-handle-server
+          sleep 20
+          echo "Checking for errors in error.log"
+          result=$(docker exec -i dspace sh -c "cat /dspace/handle-server/logs/error.log* || echo ''")
+          echo "$result"
+          echo "$result" | grep -vqz "Exception"
+          echo "Checking for errors in handle-server.log..."
+          result=$(docker exec -i dspace cat /dspace/log/handle-server.log)
+          echo "$result"
+          echo "$result" | grep -vqz "Exception"
+          echo "Checking to see if Handle Proxy webpage is available..."
+          result=$(wget -O- -q http://127.0.0.1:8000/)
+          echo "$result"
+          echo "$result" | grep -oE "Handle Proxy"
+      # Shutdown our containers
+      - name: Shutdown Docker containers
+        run: |
+          docker compose -f docker-compose.yml down

.github/workflows/reusable-docker-build.yml

@@ -113,6 +113,12 @@ jobs:
       - name: Set up QEMU emulation to build for multiple architectures
         uses: docker/setup-qemu-action@v3
 
+      #------------------------------------------------------------
+      # Build & deploy steps for new commits to a branch (non-PRs)
+      #
+      # These steps build the images, push to DockerHub, and
+      # (if necessary) redeploy demo/sandbox sites.
+      #------------------------------------------------------------
       # https://github.com/docker/login-action
       - name: Login to DockerHub
         # Only login if not a PR, as PRs only trigger a Docker build and not a push
@@ -125,6 +131,7 @@ jobs:
       # https://github.com/docker/metadata-action
       # Get Metadata for docker_build_deps step below
       - name: Sync metadata (tags, labels) from GitHub to Docker for image
+        if: ${{ ! matrix.isPr }}
         id: meta_build
         uses: docker/metadata-action@v5
         with:
@@ -133,7 +140,9 @@ jobs:
           flavor: ${{ env.TAGS_FLAVOR }}
 
       # https://github.com/docker/build-push-action
-      - name: Build and push image
+      - name: Build and push image to DockerHub
+        # Only build & push if not a PR
+        if: ${{ ! matrix.isPr }}
         id: docker_build
         uses: docker/build-push-action@v5
         with:
@@ -142,9 +151,7 @@ jobs:
           context: ${{ inputs.dockerfile_context }}
           file: ${{ inputs.dockerfile_path }}
           platforms: ${{ matrix.arch }}
-          # For pull requests, we run the Docker build (to ensure no PR changes break the build),
-          # but we ONLY do an image push to DockerHub if it's NOT a PR
-          push: ${{ ! matrix.isPr }}
+          push: true
           # Use tags / labels provided by 'docker/metadata-action' above
           tags: ${{ steps.meta_build.outputs.tags }}
           labels: ${{ steps.meta_build.outputs.labels }}
@@ -189,11 +196,54 @@ jobs:
         run: |
           curl -X POST $REDEPLOY_DEMO_URL
 
+      #-------------------------------------------------------------
+      # Build steps for PRs only
+      #
+      # These steps build the images and store as a build artifact.
+      # These artifacts can then be used by later jobs to run the
+      # brand-new images for automated testing.
+      #--------------------------------------------------------------
+      # Get Metadata for docker_build_deps step below
+      - name: Create metadata (tags, labels) for local Docker image
+        if: matrix.isPr
+        id: meta_build_pr
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.IMAGE_NAME }}
+          # Hardcode to use custom "pr-testing" tag because that will allow us to spin up this PR
+          # for testing in docker.yml
+          tags: pr-testing
+          flavor: ${{ env.TAGS_FLAVOR }}
+      # Build local image and stores in a TAR file in /tmp directory
+      - name: Build and push image to local image
+        if: matrix.isPr
+        uses: docker/build-push-action@v5
+        with:
+          build-contexts: |
+            ${{ inputs.dockerfile_additional_contexts }}
+          context: ${{ inputs.dockerfile_context }}
+          file: ${{ inputs.dockerfile_path }}
+          platforms: ${{ matrix.arch }}
+          tags: ${{ steps.meta_build_pr.outputs.tags }}
+          labels: ${{ steps.meta_build_pr.outputs.labels }}
+          # Export image to a local TAR file
+          outputs: type=docker,dest=/tmp/${{ inputs.build_id }}.tar
+      # Upload the local docker image (in TAR file) to a build Artifact
+      - name: Upload local image to artifact
+        if: matrix.isPr
+        uses: actions/upload-artifact@v4
+        with:
+          name: docker-image-${{ inputs.build_id }}
+          path: /tmp/${{ inputs.build_id }}.tar
+          if-no-files-found: error
+          retention-days: 1
+
   # Merge Docker digests (from various architectures) into a manifest.
   # This runs after all Docker builds complete above, and it tells hub.docker.com
   # that these builds should be all included in the manifest for this tag.
   # (e.g. AMD64 and ARM64 should be listed as options under the same tagged Docker image)
   docker-build_manifest:
+    # Only run if this is NOT a PR
     if: ${{ github.event_name != 'pull_request' }}
     runs-on: ubuntu-latest
     needs:

Dockerfile

@@ -58,8 +58,13 @@ ENV DSPACE_INSTALL=/dspace
 # Copy the /dspace directory from 'ant_build' container to /dspace in this container
 COPY --from=ant_build /dspace $DSPACE_INSTALL
 WORKDIR $DSPACE_INSTALL
-# Expose Tomcat port
-EXPOSE 8080
+# Need host command for "[dspace]/bin/make-handle-config"
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends host \
+    && apt-get purge -y --auto-remove \
+    && rm -rf /var/lib/apt/lists/*
+# Expose Tomcat port (8080) & Handle Server HTTP port (8000)
+EXPOSE 8080 8000
 # Give java extra memory (2GB)
 ENV JAVA_OPTS=-Xmx2000m
 # On startup, run DSpace Runnable JAR

Dockerfile.test

@@ -57,6 +57,11 @@ ENV DSPACE_INSTALL=/dspace
 # Copy the /dspace directory from 'ant_build' container to /dspace in this container
 COPY --from=ant_build /dspace $DSPACE_INSTALL
 WORKDIR $DSPACE_INSTALL
+# Need host command for "[dspace]/bin/make-handle-config"
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends host \
+    && apt-get purge -y --auto-remove \
+    && rm -rf /var/lib/apt/lists/*
 # Expose Tomcat port and debugging port
 EXPOSE 8080 8000
 # Give java extra memory (2GB)

dspace-api/pom.xml

@@ -388,6 +388,13 @@
         <dependency>
             <groupId>org.springframework</groupId>
             <artifactId>spring-orm</artifactId>
+            <exclusions>
+                <!-- Spring JCL is unnecessary and conflicts with commons-logging when both are on classpath -->
+                <exclusion>
+                    <groupId>org.springframework</groupId>
+                    <artifactId>spring-jcl</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
 
         <dependency>
@@ -406,6 +413,16 @@
                     <groupId>org.mortbay.jasper</groupId>
                     <artifactId>apache-jsp</artifactId>
                 </exclusion>
+                <!-- Excluded BouncyCastle dependencies because we use a later version of BouncyCastle.
+                     Having two versions of BouncyCastle in the classpath can cause Handle Server to throw errors. -->
+                <exclusion>
+                    <groupId>org.bouncycastle</groupId>
+                    <artifactId>bcpkix-jdk15on</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.bouncycastle</groupId>
+                    <artifactId>bcprov-jdk15on</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
 

dspace-iiif/pom.xml

@@ -44,6 +44,11 @@
                     <groupId>org.springframework.boot</groupId>
                     <artifactId>spring-boot-starter-logging</artifactId>
                 </exclusion>
+                <!-- Spring JCL is unnecessary and conflicts with commons-logging when both are on classpath -->
+                <exclusion>
+                    <groupId>org.springframework</groupId>
+                    <artifactId>spring-jcl</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
 

dspace-oai/pom.xml

@@ -80,6 +80,11 @@
                     <groupId>org.springframework.boot</groupId>
                     <artifactId>spring-boot-starter-logging</artifactId>
                 </exclusion>
+                <!-- Spring JCL is unnecessary and conflicts with commons-logging when both are on classpath -->
+                <exclusion>
+                    <groupId>org.springframework</groupId>
+                    <artifactId>spring-jcl</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
 

dspace-rdf/pom.xml

@@ -67,6 +67,11 @@
                     <groupId>org.springframework.boot</groupId>
                     <artifactId>spring-boot-starter-logging</artifactId>
                 </exclusion>
+                <!-- Spring JCL is unnecessary and conflicts with commons-logging when both are on classpath -->
+                <exclusion>
+                    <groupId>org.springframework</groupId>
+                    <artifactId>spring-jcl</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
 

dspace-server-webapp/pom.xml

@@ -468,6 +468,11 @@
                     <groupId>org.springframework.boot</groupId>
                     <artifactId>spring-boot-starter-logging</artifactId>
                 </exclusion>
+                <!-- Spring JCL is unnecessary and conflicts with commons-logging when both are on classpath -->
+                <exclusion>
+                    <groupId>org.springframework</groupId>
+                    <artifactId>spring-jcl</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
 

dspace-services/pom.xml

@@ -90,6 +90,13 @@
             <artifactId>spring-context-support</artifactId>
             <version>${spring.version}</version>
             <scope>compile</scope>
+            <exclusions>
+                <!-- Spring JCL is unnecessary and conflicts with commons-logging when both are on classpath -->
+                <exclusion>
+                    <groupId>org.springframework</groupId>
+                    <artifactId>spring-jcl</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
         <dependency>
             <groupId>org.apache.commons</groupId>

dspace-sword/pom.xml

@@ -55,6 +55,11 @@
                     <groupId>org.springframework.boot</groupId>
                     <artifactId>spring-boot-starter-logging</artifactId>
                 </exclusion>
+                <!-- Spring JCL is unnecessary and conflicts with commons-logging when both are on classpath -->
+                <exclusion>
+                    <groupId>org.springframework</groupId>
+                    <artifactId>spring-jcl</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
 

dspace-swordv2/pom.xml

@@ -78,6 +78,11 @@
                     <groupId>org.springframework.boot</groupId>
                     <artifactId>spring-boot-starter-logging</artifactId>
                 </exclusion>
+                <!-- Spring JCL is unnecessary and conflicts with commons-logging when both are on classpath -->
+                <exclusion>
+                    <groupId>org.springframework</groupId>
+                    <artifactId>spring-jcl</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
 

pom.xml

@@ -1155,6 +1155,13 @@
                 <groupId>org.springframework</groupId>
                 <artifactId>spring-orm</artifactId>
                 <version>${spring.version}</version>
+                <exclusions>
+                    <!-- Spring JCL is unnecessary and conflicts with commons-logging when both are on classpath -->
+                    <exclusion>
+                        <groupId>org.springframework</groupId>
+                        <artifactId>spring-jcl</artifactId>
+                    </exclusion>
+                </exclusions>
             </dependency>
             <dependency>
                 <groupId>org.swordapp</groupId>