Build helm index

.github/pull_request_template.md

@@ -0,0 +1,7 @@
+
+
+## Checklist
+- [ ] The PR conforms to DataHub's [Contributing Guideline](https://github.com/datahub-project/datahub/blob/master/docs/CONTRIBUTING.md) (particularly [Commit Message Format](https://github.com/datahub-project/datahub/blob/master/docs/CONTRIBUTING.md#commit-message-format))
+- [ ] Links to related issues (if applicable)
+- [ ] Tests for the changes have been added/updated (if applicable)
+- [ ] Docs related to the changes have been added/updated (if applicable)

.github/scripts/bump-chart-versions.sh

@@ -0,0 +1,19 @@
+#! /bin/sh
+
+export TAG=$1
+
+updateVersion() {
+    TAG=$1 yq -i e '.version |= env(TAG) ' $2;
+    echo "Version is updated to $1 in $2"
+}
+
+# updating subcharts 
+for file in charts/datahub-executor-worker/*/*/Chart.yaml; do 
+   updateVersion $TAG $file
+done
+
+# updating datahub chart
+updateVersion $TAG charts/datahub-executor-worker/Chart.yaml
+
+
+ yq -i e ".dependencies[].version |= env(TAG)" charts/datahub-executor-worker/Chart.yaml

.github/scripts/check_alternatives.py

@@ -0,0 +1,22 @@
+import os
+import re
+import fnmatch
+
+def find_files(directory, pattern):
+    matches = []
+    for root, _, filenames in os.walk(directory):
+        for filename in fnmatch.filter(filenames, pattern):
+            matches.append(os.path.join(root, filename))
+    return matches
+
+directory_to_search = 'charts/datahub-executor-worker/subcharts'
+pattern = '*deployment-alternatives*'
+
+matched_files = find_files(directory_to_search, pattern)
+for file in matched_files:
+    with open(file) as f:
+        lines = f.readlines()
+        for line in lines:
+            if re.compile("{{\s+.Values").search(line):
+                raise Exception(f"Found problem in {file}: {line}. Please use  $.Values. instead of .Values.")
+

.github/scripts/check_versions.py

@@ -0,0 +1,15 @@
+import yaml
+
+DATAHUB_CHART_PATH = 'charts/datahub-executor-worker/Chart.yaml'
+with open(DATAHUB_CHART_PATH) as datahub_chart_yaml:
+    datahub_chart_obj = yaml.safe_load(datahub_chart_yaml)
+    dependencies = datahub_chart_obj['dependencies']
+    for dependency in dependencies:
+        dependency_version_main_chart = dependency['version']
+        repository = dependency['repository'].replace("file://./", "")
+        subchart_path = DATAHUB_CHART_PATH.replace("Chart.yaml", "") + repository + '/Chart.yaml'
+        with open(subchart_path) as subchart_yaml:
+            subchart_obj = yaml.safe_load(subchart_yaml)
+            subchart_version = subchart_obj['version']
+            if dependency_version_main_chart != subchart_version:
+                raise Exception(f"Version mismatch in {subchart_path}. Main chart: {dependency_version_main_chart}, Subchart: {subchart_version}")

.github/workflows/lint-test.yaml

@@ -0,0 +1,48 @@
+name: Lint and Test Charts
+
+on:
+  push:
+    branches:
+      - main
+    paths-ignore:
+      - '**.md'
+      - 'LICENSE'
+      - 'NOTICE'
+  pull_request:
+    paths-ignore:
+      - '**.md'
+      - 'LICENSE'
+      - 'NOTICE'
+
+jobs:
+  lint-test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Set up Helm
+        uses: azure/setup-helm@v1
+        with:
+          version: v3.4.0
+
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+
+      - name: Set up chart-testing
+        uses: helm/[email protected]
+
+      - name: Run chart-testing (list-changed)
+        id: list-changed
+        run: |
+          changed=$(ct list-changed --target-branch ${{ github.event.pull_request.base.ref }})
+
+      - name: Run chart-testing (lint)
+        run: ct lint --target-branch ${{ github.event.pull_request.base.ref }}
+
+      - name: Run deployment alternatives Check
+        run: |
+          python3 .github/scripts/check_alternatives.py

.github/workflows/pr-delete.yaml

@@ -0,0 +1,56 @@
+name: Delete PR Helm Charts
+
+on:
+  workflow_dispatch:
+    inputs:
+      branch:
+        description: 'branch name to delete'
+        required: true
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      pages: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          ref: gh-pages
+          fetch-depth: 0
+
+      - name: Configure Git
+        run: |
+          git config user.name "$GITHUB_ACTOR"
+          git config user.email "[email protected]"
+
+      - name: Extract branch name
+        shell: bash
+        run: |
+          echo "branch=$(echo ${{ github.event.inputs.branch }})" >> $GITHUB_OUTPUT
+        id: extract_branch
+
+      - name: Download YQ
+        uses: chrisdickinson/[email protected]
+        with:
+          yq-version: v4.28.2
+
+      - name: Debug step
+        run: |
+          echo "Branch: ${{ steps.extract_branch.outputs.branch }}"
+          echo "Chart version: 0.0.0-${{ steps.extract_branch.outputs.branch }}"
+          cat pr/index.yaml | yq '.entries.datahub-executor-worker[] | select(.version | test("${{ steps.extract_branch.outputs.branch }}"))'
+          find pr/ -name '*${{ steps.extract_branch.outputs.branch }}*'
+
+      - name: Delete PR Helm charts
+        run: |
+          cat pr/index.yaml | yq 'del(.entries.datahub-executor-worker[] | select(.version | test("${{ steps.extract_branch.outputs.branch }}")))' > pr/index.yaml
+          git status
+          git ls-files --deleted | xargs git add --all
+          # Ensure that apiVersion exists
+          yq -i '.apiVersion = "v1"' pr/index.yaml
+          git add pr/index.yaml
+          git status
+          git commit -m "Deleted all Helm charts from branch: ${{ steps.extract_branch.outputs.branch }}"
+          git push

.github/workflows/pr-release.yaml

@@ -0,0 +1,65 @@
+name: Publish PR Helm Charts
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+    paths-ignore:
+      - '**.md'
+      - 'LICENSE'
+      - 'NOTICE'
+  pull_request:
+    paths-ignore:
+      - '**.md'
+      - 'LICENSE'
+      - 'NOTICE'
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      pages: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Fetch history
+        run: git fetch --prune --unshallow
+
+      - name: Configure Git
+        run: |
+          git config user.name "$GITHUB_ACTOR"
+          git config user.email "[email protected]"
+
+      - name: Extract branch name
+        shell: bash
+        run: |
+          echo "branch=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_OUTPUT
+          echo "commit=$(echo ${{ github.sha }} | cut -c -7 )" >> $GITHUB_OUTPUT
+        id: extract_branch
+
+      - name: Debug step
+        run: |
+          echo "Branch: ${{ steps.extract_branch.outputs.branch }}"
+          echo "Commit: ${{ steps.extract_branch.outputs.commit }}"
+          echo "Sha: ${{ github.sha }}"
+          echo "Latest Chart version: 0.0.0-${{ steps.extract_branch.outputs.branch }}"
+          echo "Specific Chart version: 0.0.0-${{ steps.extract_branch.outputs.branch }}-${{ steps.extract_branch.outputs.commit }}"
+
+      - name: Publish Helm charts for specific commit
+        uses: stefanprodan/helm-gh-pages@master
+        with:
+          charts_url: https://executor-helm.acryl.io
+          target_dir: pr
+          chart_version: 0.0.0-${{ steps.extract_branch.outputs.branch }}-${{ steps.extract_branch.outputs.commit }}
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Publish Latest PR Helm charts
+        uses: stefanprodan/helm-gh-pages@master
+        with:
+          charts_url: https://executor-helm.acryl.io
+          target_dir: pr
+          chart_version: 0.0.0-${{ steps.extract_branch.outputs.branch }}
+          token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/release.yaml

@@ -0,0 +1,35 @@
+name: Release Charts
+
+on:
+  push:
+    branches:
+      - main
+    paths-ignore:
+      - '**.md'
+      - 'LICENSE'
+      - 'NOTICE'
+  workflow_dispatch:
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      pages: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Fetch history
+        run: git fetch --prune --unshallow
+
+      - name: Configure Git
+        run: |
+          git config user.name "$GITHUB_ACTOR"
+          git config user.email "[email protected]"
+
+      - name: Publish Helm charts
+        uses: stefanprodan/helm-gh-pages@master
+        with:
+          charts_url: https://executor-helm.acryl.io
+          token: ${{ secrets.GITHUB_TOKEN }}

charts/datahub-executor-worker/Chart.yaml

@@ -3,4 +3,4 @@ name: datahub-executor-worker
 description: A Helm chart for datahub-executor-worker
 type: application
 version: 0.0.5
-appVersion: v0.0.1
+appVersion: 0.0.1