builder refactor

chanzuckerberg · Aug 27, 2024 · 9e0ae7e · 9e0ae7e
1 parent 7cc6b24
commit 9e0ae7e
Showing 1 changed file with 172 additions and 16 deletions.
diff --git a/.github/workflows/argus-docker-build.yaml b/.github/workflows/argus-docker-build.yaml
@@ -8,7 +8,7 @@ on:
         required: true
         type: string
       images:
-        description: 'JSON array of images to build (required keys: dockerfile, context, name, platform)'
+        description: 'JSON object specifying the images to build'
         required: true
         type: string
       path_filters:
@@ -24,7 +24,7 @@ on:
         required: false
         type: string
         default: ${{ github.ref }}
-      branches:
+      branches_include:
         description: 'Branch names to run this job on, supports wildcards, comma delimited'
         required: false
         type: string
@@ -34,11 +34,6 @@ on:
         required: false
         type: string
         default: ''
-      working_directory:
-        description: 'The Argus project root (parent directory that contains the .infra/ directory)'
-        required: false
-        type: string
-        default: '.'
 
 jobs:
   prep:
@@ -49,32 +44,109 @@ jobs:
       image_tag: ${{ steps.build_prep.outputs.image_tag }}
       should_build: ${{ steps.build_prep.outputs.should_build }}
       images: ${{ steps.parse_images.outputs.images }}
+      path_filter_base: ${{ steps.build_prep.outputs.base }}
     permissions:
       id-token: write
       contents: read
     steps:
-      - uses: chanzuckerberg/github-actions/.github/actions/argus-builder/build-prep@c210eb8dba1aeb3ccf02186ad89c3c85f6fc4da7
+      - uses: chanzuckerberg/github-actions/.github/actions/argus-builder/build-prep@bfe3e9bba81a3a1d20c017b0bfd2b0361d8f97bc
         id: build_prep
         with:
           path_filters: ${{ inputs.path_filters }}
           path_filters_base: ${{ inputs.path_filters_base }}
-          branches: ${{ inputs.branches }}
+          branches: ${{ inputs.branches_include }}
           branches_ignore: ${{ inputs.branches_ignore }}
+
+      - uses: actions/checkout@v4
+      - uses: chanzuckerberg/github-actions/.github/actions/validate-json-schema@7cc6b249575fe530d88f345599524c939821aeda
+        name: Validate images input
+        with:
+          schema: |
+            {
+              "type": "object",
+              "additionalProperties": {
+                type: "object",
+                "properties": {
+                  "context": { "type": "string" },
+                  "dockerfile": { "type": "string" },
+                  "platform": { "type": "string" },
+                  "build_args": {
+                    "type": "array",
+                    "items": { "type": "string" }
+                  },
+                  "secret_files": {
+                    "type": "array",
+                    "items": { "type": "string" }
+                  },
+                  "argus_root": { "type": "string" },
+                  "path_filters": {
+                    "type": "array",
+                    "items": {
+                      "oneOf": [
+                        { "type": "string" },
+                        { "type": "array", "items": { "type": "string" } }
+                      ]
+                    }
+                  },
+                  "branches_include": {
+                    "type": "array",
+                    "items": { "type": "string" }
+                  },
+                  "branches_ignore": {
+                    "type": "array",
+                    "items": { "type": "string" }
+                  },
+                },
+                "required": ["context", "dockerfile"]
+              }
+            }
+          data: ${{ inputs.images }}
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+      - run: npm install yaml
       - name: Parse inputs
         id: parse_images
         uses: actions/github-script@v7
         with:
           script: |
+            const YAML = require('yaml');
+
             const images = JSON.parse(`${{ inputs.images }}`);
-            images.forEach(image => {
+            const processedImages = Object.entries(images).map(([name, image]) => {
+              image.name = name;
+
               const buildArgs = image.build_args || [];
               image.build_args = buildArgs.join("\n");
 
               const secretFiles = image.secret_files || [];
               image.secret_files = secretFiles.join("\n");
+
+              const pathFilters = image.path_filters || ['**/*'];
+              // convert to an object with keys
+              const processed = pathFilters.reduce((acc, filter, index) => {
+                acc[index] = Array.isArray(filter) ? filter : [filter];
+                return acc;
+              }, {});
+              core.info(`Processed path filters: ${JSON.stringify(processed, null, 2)}`);
+              const doc = new YAML.Document();
+              doc.contents = processed;
+              image.path_filters = doc.toString();
+
+              const branchesInclude = image.branches_include || ['*'];
+              image.branches_include = branchesInclude.join(",");
+
+              const branchesIgnore = image.branches_ignore || [];
+              image.branches_ignore = branchesIgnore.join(",");
+
+              const argusRoot = image.argus_root || '.';
+              image.argus_root = argusRoot;
+
+              return image;
             });
-            core.info(`Images to build: ${JSON.stringify(images, null, 2)}`);
-            core.setOutput('images', images);
+            core.info(`Images to build: ${JSON.stringify(processedImages, null, 2)}`);
+            core.setOutput('images', processedImages);
 
   build-docker:
     name: Build Docker Image
@@ -92,7 +164,42 @@ jobs:
       matrix:
         image: ${{ fromJson(needs.prep.outputs.images) }}
     steps:
-      - uses: chanzuckerberg/github-actions/.github/actions/argus-builder/docker-build@0bf13ddcc9cd1b7bcd69e080d472104569f53f28
+      - uses: actions/checkout@v4
+      - name: Log image config
+        id: log_image
+        uses: actions/github-script@v7
+        with:
+          script: |
+            core.info(`Image to build: ${{ toJson(matrix.image) }}`);
+      - name: Check branch
+        id: check_branch
+        uses: chanzuckerberg/github-actions/.github/actions/check-branch-match@98e955dbb01d403ed58c0c04d509eed8aa629f98
+        with:
+          branches_include: ${{ matrix.image.branches_include }}
+          branches_ignore: ${{ matrix.image.branches_ignore }}
+      - name: Check for matching file changes
+        uses: dorny/paths-filter@v3
+        id: path_filter
+        with:
+          # predicate-quantifier will cause a warning but it is a valid input (https://github.com/dorny/paths-filter/pull/226)
+          predicate-quantifier: 'every'
+          filters: ${{ matrix.image.path_filters }}
+          base: ${{ needs.prep.outputs.path_filter_base }}
+          list-files: json
+      - name: Check if we should build
+        id: should_build
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const matchedBranches = ${{ steps.check_branch.outputs.match }};
+            console.log('matched branches?', matchedBranches);
+            const matchedFiles = ${{ steps.path_filter.outputs.changes }}.length > 0;
+            console.log('matched files?', matchedFiles);
+            const shouldBuild = matchedBranches && matchedFiles;
+            console.log('should build?', shouldBuild);
+            return shouldBuild;
+      - uses: chanzuckerberg/github-actions/.github/actions/argus-builder/docker-build@e5c37d11ce543f5072765c754fe483e63f477513
+        if: steps.should_build.outputs.result == 'true'
         with:
           image_name: ${{ matrix.image.name }}
           dockerfile: ${{ matrix.image.dockerfile }}
@@ -103,7 +210,16 @@ jobs:
           image_tag: ${{ needs.prep.outputs.image_tag }}
           github_app_id: ${{ secrets.CZI_GITHUB_HELPER_APP_ID }}
           github_private_key: ${{ secrets.CZI_GITHUB_HELPER_PK }}
-          working_directory: ${{ inputs.working_directory }}
+      - run: echo "BUILD_ARTIFACT_FILENAME=built-${{ matrix.image.name }}-${{ needs.prep.outputs.image_tag }}" >> $GITHUB_OUTPUT
+        if: steps.should_build.outputs.result == 'true'
+        id: set_artifact_name
+      - run: echo true > ${{ steps.set_artifact_name.outputs.BUILD_ARTIFACT_FILENAME }}
+        if: steps.should_build.outputs.result == 'true'
+      - uses: actions/upload-artifact@v4
+        if: steps.should_build.outputs.result == 'true'
+        with:
+          name: ${{ steps.set_artifact_name.outputs.BUILD_ARTIFACT_FILENAME }}
+          path: ${{ steps.set_artifact_name.outputs.BUILD_ARTIFACT_FILENAME }}
 
   update-manifests:
     name: Update ArgoCD manifests
@@ -124,10 +240,50 @@ jobs:
               core.info('All builds passed, continuing with manifest update...');
             }
 
-      - uses: chanzuckerberg/github-actions/.github/actions/argus-builder/manifest-update@c210eb8dba1aeb3ccf02186ad89c3c85f6fc4da7
+      - uses: actions/download-artifact@v4
+        id: download
+        with:
+          path: build-results
+          pattern: built-*
+
+      - name: Determine manifests to update
+        uses: actions/github-script@v7
+        id: determine_manifests
+        with:
+          result-encoding: string
+          script: |
+            const fs = require('fs');
+
+            const images = ${{ needs.prep.outputs.images }};
+            const filenameRegex = new RegExp(`^built-(\\S*)-(sha-\\S*)$`);
+
+            if (!fs.existsSync(`${{ steps.download.outputs.download-path }}`)) {
+              core.info('No images were built, skipping manifest update');
+              return '';
+            }
+
+            const argusRootDirs = [];
+            fs.readdirSync(`${{ steps.download.outputs.download-path }}`).forEach(file => {
+              console.log('checking file:', file);
+              const match = file.match(filenameRegex);
+              if (!match || match.length < 2) {
+                return;
+              }
+              const imageName = match[1];
+              const image = images.find(img => img.name === imageName);
+              if (!image) {
+                throw new Error(`Image with name=${imageName} was built but cannot be found in ${JSON.stringify(images)}`);
+              }
+              argusRootDirs.push(image.argus_root)
+            });
+            console.log('Argus root dirs:', argusRootDirs);
+            return argusRootDirs.join(',');
+
+      - uses: chanzuckerberg/github-actions/.github/actions/argus-builder/manifest-update@1d4de79138f4999e3c4b06a6762c37ace5edc18c
+        if: steps.determine_manifests.outputs.result != ''
         with:
           envs: ${{ inputs.envs }}
           image_tag: ${{ needs.prep.outputs.image_tag }}
-          working_directory: ${{ inputs.working_directory }}
+          argus_project_dirs: ${{ steps.determine_manifests.outputs.result }}
           github_app_id: ${{ secrets.CZI_GITHUB_HELPER_APP_ID }}
           github_private_key: ${{ secrets.CZI_GITHUB_HELPER_PK }}