Conductor (#322)

.github/workflows/run-improc-tests.yml

@@ -21,50 +21,83 @@ jobs:
         uses: jwalton/gh-docker-logs@v2
 
       - name: checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           submodules: recursive
 
       - name: log into github container registry
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: setup docker buildx
-        uses: docker/setup-buildx-action@v2
-        with:
-          driver: docker-container
-
-      - name: bake
-        uses: docker/[email protected]
-        with:
-          workdir: tests
-          load: true
-          files: docker-compose.yaml
-          set: |
-            seechange_postgres.tags=ghcr.io/${{ github.repository_owner }}/seechange-postgres
-            seechange_postgres.cache-from=type=gha,scope=cached-seechange-postgres
-            seechange_postgres.cache-to=type=gha,scope=cached-seechange-postgres,mode=max
-            setuptables.tags=ghcr.io/${{ github.repository_owner }}/runtests
-            setuptables.cache-from=type=gha,scope=cached-seechange
-            setuptables.cache-to=type=gha,scope=cached-seechange,mode=max
-            runtests.tags=ghcr.io/${{ github.repository_owner }}/runtests
-            runtests.cache-from=type=gha,scope=cached-seechange
-            runtests.cache-to=type=gha,scope=cached-seechange,mode=max
-            shell.tags=ghcr.io/${{ github.repository_owner }}/runtests
-            shell.cache-from=type=gha,scope=cached-seechange
-            shell.cache-to=type=gha,scope=cached-seechange,mode=max
-
-      - name: run test
+      - name: cleanup
         run: |
           # try to save HDD space on the runner by removing some unneeded stuff
           # ref: https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
           sudo rm -rf /usr/share/dotnet
           sudo rm -rf /opt/ghc
           sudo rm -rf "/usr/local/share/boost"
           sudo rm -rf "$AGENT_TOOLSDIRECTORY"
- 
+
+      # A NOTE ABOUT DOCKER IMAGES
+      # Because they don't change with every pull request, we've stopped
+      # building them in the github actions.  In pratice, they were
+      # getting rebuilt all the time, which was slowing things down.
+      #
+      # Now, these actions depend on all the docker images having been
+      # pre-built and stored on the github container archive.  Look at
+      # tests/docker-compose.yaml; there, you can see what the various
+      # image names are expected to be.
+      #
+      # For building and pushing these docker images, see "Running tests
+      # on github actions" in the "Testing" section of the code documentation.
+      #
+      # If we ever want to go back to building the docker images in
+      # all of the workflow files, the code is below.  However, you
+      # should make sure that all the things under "set" in the "bake"
+      # section are up to date with whats in tests/docker-compose.yaml
+
+      # - name: setup docker buildx
+      #   uses: docker/setup-buildx-action@v3
+      #   with:
+      #     driver: docker-container
+
+      # - name: bake
+      #   uses: docker/bake-action@v5
+      #   with:
+      #     workdir: tests
+      #     load: true
+      #     files: docker-compose.yaml
+      #     set: |
+      #       archive.tags=ghcr.io/${{ github.repository_owner }}/archive
+      #       archive.cache-from=type=gha,scope=cached-archive
+      #       archive.cache-to=type=gha,scope=cached-archive,mode=max
+      #       postgres.tags=ghcr.io/${{ github.repository_owner }}/postgres
+      #       postgres.cache-from=type=gha,scope=cached-postgres
+      #       postgres.cache-to=type=gha,scope=cached-postgres,mode=max
+      #       setuptables.tags=ghcr.io/${{ github.repository_owner }}/runtests
+      #       setuptables.cache-from=type=gha,scope=cached-seechange
+      #       setuptables.cache-to=type=gha,scope=cached-seechange,mode=max
+      #       conductor.tags=ghcr.io/${{ github.repository_owner }}/conductor
+      #       conductor.cache-from=type=gha,scipe=cached-conductor
+      #       conductor.cache-to=type=gha,scope=cached-conductor,mode=max
+      #       webap.tags=ghcr.io/${{ github.repository_owner }}/seechange-webap
+      #       webap.cache-from=type=gha,scipe=cached-seechange-webap
+      #       webap.cache-to=type=gha,scope=cached-seechange-webap,mode=max
+      #       runtests.tags=ghcr.io/${{ github.repository_owner }}/runtests
+      #       runtests.cache-from=type=gha,scope=cached-seechange
+      #       runtests.cache-to=type=gha,scope=cached-seechange,mode=max
+      #       shell.tags=ghcr.io/${{ github.repository_owner }}/runtests
+      #       shell.cache-from=type=gha,scope=cached-seechange
+      #       shell.cache-to=type=gha,scope=cached-seechange,mode=max
+
+      - name: pull images
+        run: |
+          docker compose pull archive postgres conductor mailhog webap runtests
+
+      - name: run test
+        run: |
           shopt -s nullglob
-          TEST_SUBFOLDER=tests/improc docker compose run runtests
+          TEST_SUBFOLDER=tests/improc docker compose run -e SKIP_BIG_MEMORY=1 runtests

.github/workflows/run-model-tests-1.yml

@@ -21,43 +21,18 @@ jobs:
         uses: jwalton/gh-docker-logs@v2
 
       - name: checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           submodules: recursive
 
       - name: log into github container registry
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: setup docker buildx
-        uses: docker/setup-buildx-action@v2
-        with:
-          driver: docker-container
-
-      - name: bake
-        uses: docker/[email protected]
-        with:
-          workdir: tests
-          load: true
-          files: docker-compose.yaml
-          set: |
-            seechange_postgres.tags=ghcr.io/${{ github.repository_owner }}/seechange-postgres
-            seechange_postgres.cache-from=type=gha,scope=cached-seechange-postgres
-            seechange_postgres.cache-to=type=gha,scope=cached-seechange-postgres,mode=max
-            setuptables.tags=ghcr.io/${{ github.repository_owner }}/runtests
-            setuptables.cache-from=type=gha,scope=cached-seechange
-            setuptables.cache-to=type=gha,scope=cached-seechange,mode=max
-            runtests.tags=ghcr.io/${{ github.repository_owner }}/runtests
-            runtests.cache-from=type=gha,scope=cached-seechange
-            runtests.cache-to=type=gha,scope=cached-seechange,mode=max
-            shell.tags=ghcr.io/${{ github.repository_owner }}/runtests
-            shell.cache-from=type=gha,scope=cached-seechange
-            shell.cache-to=type=gha,scope=cached-seechange,mode=max
-
-      - name: run test
+      - name: cleanup
         run: |
           # try to save HDD space on the runner by removing some unneeded stuff
           # ref: https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
@@ -66,5 +41,15 @@ jobs:
           sudo rm -rf "/usr/local/share/boost"
           sudo rm -rf "$AGENT_TOOLSDIRECTORY"
 
+      # IF BUILDING DOCKER IMAGES IN EACH STEP
+      # Make sure the code doing this in run-improc-tests.yml is right.
+      # Uncomment it there, and copy it here.  Remove the "pull images" step.
+
+      - name: pull images
+        run: |
+          docker compose pull archive postgres conductor mailhog webap runtests
+
+      - name: run test
+        run: |
           shopt -s nullglob
-          TEST_SUBFOLDER=$(ls tests/models/test_{a..l}*.py) docker compose run runtests
+          TEST_SUBFOLDER=$(ls tests/models/test_{a..l}*.py) docker compose run -e SKIP_BIG_MEMORY=1 runtests

.github/workflows/run-model-tests-2.yml

@@ -21,43 +21,18 @@ jobs:
         uses: jwalton/gh-docker-logs@v2
 
       - name: checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           submodules: recursive
 
       - name: log into github container registry
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: setup docker buildx
-        uses: docker/setup-buildx-action@v2
-        with:
-          driver: docker-container
-
-      - name: bake
-        uses: docker/[email protected]
-        with:
-          workdir: tests
-          load: true
-          files: docker-compose.yaml
-          set: |
-            seechange_postgres.tags=ghcr.io/${{ github.repository_owner }}/seechange-postgres
-            seechange_postgres.cache-from=type=gha,scope=cached-seechange-postgres
-            seechange_postgres.cache-to=type=gha,scope=cached-seechange-postgres,mode=max
-            setuptables.tags=ghcr.io/${{ github.repository_owner }}/runtests
-            setuptables.cache-from=type=gha,scope=cached-seechange
-            setuptables.cache-to=type=gha,scope=cached-seechange,mode=max
-            runtests.tags=ghcr.io/${{ github.repository_owner }}/runtests
-            runtests.cache-from=type=gha,scope=cached-seechange
-            runtests.cache-to=type=gha,scope=cached-seechange,mode=max
-            shell.tags=ghcr.io/${{ github.repository_owner }}/runtests
-            shell.cache-from=type=gha,scope=cached-seechange
-            shell.cache-to=type=gha,scope=cached-seechange,mode=max
-
-      - name: run test
+      - name: cleanup
         run: |
           # try to save HDD space on the runner by removing some unneeded stuff
           # ref: https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
@@ -66,5 +41,15 @@ jobs:
           sudo rm -rf "/usr/local/share/boost"
           sudo rm -rf "$AGENT_TOOLSDIRECTORY"
 
+      # IF BUILDING DOCKER IMAGES IN EACH STEP
+      # Make sure the code doing this in run-improc-tests.yml is right.
+      # Uncomment it there, and copy it here.  Remove the "pull images" step.
+
+      - name: pull images
+        run: |
+          docker compose pull archive postgres conductor mailhog webap runtests
+
+      - name: run test
+        run: |
           shopt -s nullglob
-          TEST_SUBFOLDER=$(ls tests/models/test_{m..z}*.py) docker compose run runtests
+          TEST_SUBFOLDER=$(ls tests/models/test_{m..z}*.py) docker compose run -e SKIP_BIG_MEMORY=1 runtests

.github/workflows/run-pipeline-tests-1.yml

@@ -21,43 +21,18 @@ jobs:
         uses: jwalton/gh-docker-logs@v2
 
       - name: checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           submodules: recursive
 
       - name: log into github container registry
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: setup docker buildx
-        uses: docker/setup-buildx-action@v2
-        with:
-          driver: docker-container
-
-      - name: bake
-        uses: docker/[email protected]
-        with:
-          workdir: tests
-          load: true
-          files: docker-compose.yaml
-          set: |
-            seechange_postgres.tags=ghcr.io/${{ github.repository_owner }}/seechange-postgres
-            seechange_postgres.cache-from=type=gha,scope=cached-seechange-postgres
-            seechange_postgres.cache-to=type=gha,scope=cached-seechange-postgres,mode=max
-            setuptables.tags=ghcr.io/${{ github.repository_owner }}/runtests
-            setuptables.cache-from=type=gha,scope=cached-seechange
-            setuptables.cache-to=type=gha,scope=cached-seechange,mode=max
-            runtests.tags=ghcr.io/${{ github.repository_owner }}/runtests
-            runtests.cache-from=type=gha,scope=cached-seechange
-            runtests.cache-to=type=gha,scope=cached-seechange,mode=max
-            shell.tags=ghcr.io/${{ github.repository_owner }}/runtests
-            shell.cache-from=type=gha,scope=cached-seechange
-            shell.cache-to=type=gha,scope=cached-seechange,mode=max
-
-      - name: run test
+      - name: cleanup
         run: |
           # try to save HDD space on the runner by removing some uneeded stuff
           # ref: https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
@@ -66,5 +41,15 @@ jobs:
           sudo rm -rf "/usr/local/share/boost"
           sudo rm -rf "$AGENT_TOOLSDIRECTORY"
 
+      # IF BUILDING DOCKER IMAGES IN EACH STEP
+      # Make sure the code doing this in run-improc-tests.yml is right.
+      # Uncomment it there, and copy it here.  Remove the "pull images" step.
+
+      - name: pull images
+        run: |
+          docker compose pull archive postgres conductor mailhog webap runtests
+
+      - name: run test
+        run: |
           shopt -s nullglob
-          TEST_SUBFOLDER=$(ls tests/pipeline/test_{a..o}*.py) docker compose run runtests
+          TEST_SUBFOLDER=$(ls tests/pipeline/test_{a..o}*.py) docker compose run -e SKIP_BIG_MEMORY=1 runtests

.github/workflows/run-pipeline-tests-2.yml

@@ -21,43 +21,18 @@ jobs:
         uses: jwalton/gh-docker-logs@v2
 
       - name: checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           submodules: recursive
 
       - name: log into github container registry
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: setup docker buildx
-        uses: docker/setup-buildx-action@v2
-        with:
-          driver: docker-container
-
-      - name: bake
-        uses: docker/[email protected]
-        with:
-          workdir: tests
-          load: true
-          files: docker-compose.yaml
-          set: |
-            seechange_postgres.tags=ghcr.io/${{ github.repository_owner }}/seechange-postgres
-            seechange_postgres.cache-from=type=gha,scope=cached-seechange-postgres
-            seechange_postgres.cache-to=type=gha,scope=cached-seechange-postgres,mode=max
-            setuptables.tags=ghcr.io/${{ github.repository_owner }}/runtests
-            setuptables.cache-from=type=gha,scope=cached-seechange
-            setuptables.cache-to=type=gha,scope=cached-seechange,mode=max
-            runtests.tags=ghcr.io/${{ github.repository_owner }}/runtests
-            runtests.cache-from=type=gha,scope=cached-seechange
-            runtests.cache-to=type=gha,scope=cached-seechange,mode=max
-            shell.tags=ghcr.io/${{ github.repository_owner }}/runtests
-            shell.cache-from=type=gha,scope=cached-seechange
-            shell.cache-to=type=gha,scope=cached-seechange,mode=max
-
-      - name: run test
+      - name: cleanup
         run: |
           # try to save HDD space on the runner by removing some unneeded stuff
           # ref: https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
@@ -66,6 +41,15 @@ jobs:
           sudo rm -rf "/usr/local/share/boost"
           sudo rm -rf "$AGENT_TOOLSDIRECTORY"
 
+      # IF BUILDING DOCKER IMAGES IN EACH STEP
+      # Make sure the code doing this in run-improc-tests.yml is right.
+      # Uncomment it there, and copy it here.  Remove the "pull images" step.
+
+      - name: pull images
+        run: |
+          docker compose pull archive postgres conductor mailhog webap runtests
+
+      - name: run test
+        run: |
           shopt -s nullglob
-          TEST_SUBFOLDER=$(ls tests/pipeline/test_{p..z}*.py) docker compose run runtests
-          
+          TEST_SUBFOLDER=$(ls tests/pipeline/test_{p..z}*.py) docker compose run -e SKIP_BIG_MEMORY=1 runtests