#81/bug/fix cache problem

.github/workflows/cache.yaml

@@ -0,0 +1,69 @@
+name: cache
+on:
+  pull_request:
+    types:
+      - closed
+
+env:
+  DOCKER_FILE_DIRECTORY: environments
+  DOCKER_COMPOSE_DIRECTORY: environments/ci
+  COMPOSE_DOCKER_CLI_BUILD: 1
+  DOCKER_BUILDKIT: 1
+  USE_CACHE: true
+
+jobs:
+  create-cache:
+    if: github.event.pull_request.merged == true
+    name: ${{ matrix.os }} / ${{ matrix.python-version }}
+    runs-on: ${{ matrix.image }}
+    timeout-minutes: 10
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu]
+        python-version: ["3.8", "3.9"]
+        include:
+          - os: ubuntu
+            image: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Check version info
+        run: pwd && docker-compose --version && docker --version
+
+      # Provide a builder instance with docker buildx.
+      - name: Set up Docker Buildx
+        if: env.USE_CACHE == 'true'
+        uses: docker/setup-buildx-action@v3
+
+      # Specify a cache directory for build cache, and reuse past build cache as long as there are no changes to the environment files.
+      - name: Cache Docker layers
+        if: env.USE_CACHE == 'true'
+        id: build-cache
+        uses: actions/cache@v4
+        with:
+          path: /tmp/.buildx-cache/${{ matrix.python-version }}
+          key: buildx-${{ hashFiles(format('{0}/Dockerfile', env.DOCKER_FILE_DIRECTORY), format('{0}/docker-compose.yaml', env.DOCKER_COMPOSE_DIRECTORY), 'poetry.lock') }}
+
+      # If build cache is not found, build the Docker image to create a new cache.
+      - name: Build docker image with cache using buildx
+        if: steps.build-cache.outputs.cache-hit != 'true' && env.USE_CACHE == 'true'
+        uses: docker/bake-action@v4
+        with:
+          files: docker-compose.yaml
+          workdir: ${{ env.DOCKER_COMPOSE_DIRECTORY }}
+          load: true
+          set: |
+            core.args.PYTHON_VERSION=${{ matrix.python-version }}
+            *.cache-from=type=local,src=/tmp/.buildx-cache/${{ matrix.python-version }}
+            *.cache-to=type=local,dest=/tmp/.buildx-cache-new/${{ matrix.python-version }}
+
+      # Temp fix
+      # https://github.com/docker/build-push-action/issues/252
+      # https://github.com/moby/buildkit/issues/1896
+      - name: Move cache
+        if: steps.build-cache.outputs.cache-hit != 'true' && env.USE_CACHE == 'true'
+        run: |
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache

.github/workflows/lint-and-test.yaml

@@ -1,4 +1,4 @@
-name: tests
+name: lint-and-test
 on: [pull_request]
 
 env:
@@ -23,78 +23,49 @@ jobs:
             image: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Check version info
-        run: pwd && docker-compose --version && docker --version
+        run: pwd && docker compose --version && docker --version
 
+      # Provide a builder instance with docker buildx.
       - name: Set up Docker Buildx
-        if: ${{ env.USE_CACHE == 'true' }}
-        id: buildx
-        uses: docker/setup-buildx-action@v2
-        with:
-          driver-opts: network=host
+        if: env.USE_CACHE == 'true'
+        uses: docker/setup-buildx-action@v3
 
+      # Specify a cache directory for build cache, and reuse past build cache as long as there are no changes to the environment files.
       - name: Cache Docker layers
-        if: ${{ env.USE_CACHE == 'true' }}
-        uses: actions/cache@v3
+        if: env.USE_CACHE == 'true'
+        id: build-cache
+        uses: actions/cache@v4
         with:
           path: /tmp/.buildx-cache/${{ matrix.python-version }}
-          key: buildx-${{ github.ref }}-${{ github.sha }}
-          restore-keys: |
-            buildx-${{ github.ref }}
-            buildx-
+          key: buildx-${{ hashFiles(format('{0}/Dockerfile', env.DOCKER_FILE_DIRECTORY), format('{0}/docker-compose.yaml', env.DOCKER_COMPOSE_DIRECTORY), 'poetry.lock') }}
 
-      - name: Cache Docker registry
-        if: ${{ env.USE_CACHE == 'true' }}
-        uses: actions/cache@v3
+      # If build cache is found, use it to build the Docker image.
+      # The build is carried out using the `bake` command from buildx, referencing the `docker-compose.yaml` for CI.
+      - name: Build docker image with cache using buildx
+        if: steps.build-cache.outputs.cache-hit == 'true' && env.USE_CACHE == 'true'
+        uses: docker/bake-action@v4
         with:
-          path: /tmp/docker-registry/${{ matrix.python-version }}
-          key: docker-registry-${{ github.ref }}-${{ github.sha }}
-          restore-keys: |
-            docker-registry-${{ github.ref }}
-            docker-registry-
-
-      - name: Boot-up local Docker registry
-        if: ${{ env.USE_CACHE == 'true' }}
-        run: docker run -d -p 5000:5000 --restart=always --name registry -v /tmp/docker-registry:/var/lib/registry registry:2
-
-      - name: Wait for Docker registry
-        if: ${{ env.USE_CACHE == 'true' }}
-        run: npx wait-on --httpTimeout 30000 tcp:5000
-
-      - name: Generate Docker image tag
-        run: |
-            SHA=${{ github.sha }}
-            TAG=$(TZ=UTC-9 date '+%Y%m')-${SHA:0:7}
-            echo "DOCKER_IMAGE_TAG_CI=$TAG" >> $GITHUB_ENV
-            echo TAG $TAG
-            echo "::set-output name=docker_image_tag_ci::$TAG"
-
-      - name: Build docker image with cache
-        if: ${{ env.USE_CACHE == 'true' }}
-        run: |
-          docker buildx bake \
-          --builder="${{ steps.buildx.outputs.name }}" \
-          --set="core.args.PYTHON_VERSION=${{ matrix.python-version }}" \
-          --set="*.cache-from=type=local,src=/tmp/.buildx-cache/${{ matrix.python-version }}" \
-          --set="*.cache-to=type=local,dest=/tmp/.buildx-cache-new/${{ matrix.python-version }}" \
-          --push \
-          -f docker-compose.yaml
-        working-directory: ${{ env.DOCKER_COMPOSE_DIRECTORY }}
-
-      - name: Build docker image
-        if: ${{ env.USE_CACHE != 'true' }}
-        run: docker-compose build --parallel --build-arg PYTHON_VERSION=${{ matrix.python-version }} core
-        working-directory: ${{ env.DOCKER_COMPOSE_DIRECTORY }}
-
-      - name: Pull Docker image
-        if: ${{ env.USE_CACHE == 'true' }}
-        run: docker-compose pull
+          files: docker-compose.yaml
+          workdir: ${{ env.DOCKER_COMPOSE_DIRECTORY }}
+          load: true
+          set: |
+            core.args.PYTHON_VERSION=${{ matrix.python-version }}
+            *.cache-from=type=local,src=/tmp/.buildx-cache/${{ matrix.python-version }}
+
+      # If build cache is not found, build the Docker image without cache.
+      # It also works with docker/bake-action, but the `bake` command creates `/tmp/.buildx-cache/${{ matrix.python-version }}` as specified in cache-from.
+      # Since it is a cache directory tracked by actions/cache, if the directory exists at the end of the workflow, unnecessary cache is created in the merge branch.
+      # We used the regular build command to avoid this problem.
+      - name: Build docker image without cache using normal build
+        if: steps.build-cache.outputs.cache-hit != 'true' || env.USE_CACHE != 'true'
+        run: docker compose build --parallel --build-arg PYTHON_VERSION=${{ matrix.python-version }} core
         working-directory: ${{ env.DOCKER_COMPOSE_DIRECTORY }}
 
       - name: Create and start docker container
-        run: docker-compose up --no-build -d
+        run: docker compose up --no-build -d
         working-directory: ${{ env.DOCKER_COMPOSE_DIRECTORY }}
 
       # pytest-cov export coverage data to a file
@@ -106,22 +77,13 @@ jobs:
       # In the built stage of Docker image, .venv dir is moved from working directory to prevent
       # overwrite by volume operation of Docker. Here, .venv is moved back to working directory.
       - name: Move .venv directory
-        run: docker-compose exec -T core mv ../.venv .
+        run: docker compose exec -T core mv ../.venv .
         working-directory: ${{ env.DOCKER_COMPOSE_DIRECTORY }}
 
       - name: Run lint
-        run: docker-compose exec -T core make lint
+        run: docker compose exec -T core make lint
         working-directory: ${{ env.DOCKER_COMPOSE_DIRECTORY }}
 
       - name: Run test code
-        run: docker-compose exec -T core make test
-        working-directory: ${{ env.DOCKER_COMPOSE_DIRECTORY }}
-
-        # Temp fix
-        # https://github.com/docker/build-push-action/issues/252
-        # https://github.com/moby/buildkit/issues/1896
-      - name: Move cache
-        if: ${{ env.USE_CACHE == 'true' }}
-        run: |
-          rm -rf /tmp/.buildx-cache
-          mv /tmp/.buildx-cache-new /tmp/.buildx-cache
+        run: docker compose exec -T core make test
+        working-directory: ${{ env.DOCKER_COMPOSE_DIRECTORY }}

environments/ci/docker-compose.yaml

@@ -2,7 +2,7 @@ version: "3.8"
 
 services:
   core:
-    image: localhost:5000/core:${DOCKER_IMAGE_TAG_CI}
+    image: ci
     build:
       args:
         - BASE_IMAGE=ubuntu:20.04

environments/cpu/docker-compose.yaml

@@ -18,4 +18,4 @@ services:
     volumes:
         - ../../:/home/challenger/${PROJECT_NAME_ENV}
     ports:
-        - 9700:8000
+        - 8000:8000