This repository has been archived by the owner on May 25, 2024. It is now read-only.
2.16.0 - [CLOSE: #179, CLOSE: #180] #52
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI/CD - Streamlit Image | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| logLevel: | |
| description: 'Log level' | |
| required: true | |
| default: 'warning' | |
| type: choice | |
| options: | |
| - info | |
| push: | |
| paths-ignore: | |
| - '.github/**' | |
| - 'api/**' | |
| - 'etl/**' | |
| - 'cli/**' | |
| - 'dbt_prod/**' | |
| - 'prefect/**' | |
| - 'soda/**' | |
| - 'terraform/**' | |
| - 'tests/**' | |
| - 'CHANGELOG.md' | |
| - '.dockerignore' | |
| - '.gitignore' | |
| - '.prefectignore' | |
| - '.pre-commit-config.yaml' | |
| - '.streamlit' | |
| - 'pyproject.toml' | |
| - 'README.md' | |
| branches: | |
| - "main" | |
| env: | |
| GAR_LOCATION: us-central1 | |
| VERSION_NUMBER: '2.16.0' | |
| REGISTRY_IMAGE: digitalghostdev/premier-league | |
| jobs: | |
| snyk: | |
| runs-on: ubuntu-22.04 | |
| permissions: | |
| actions: read | |
| contents: read | |
| security-events: write | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Run Snyk | |
| uses: snyk/actions/python-3.10@master | |
| continue-on-error: true | |
| env: | |
| SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} | |
| with: | |
| args: --sarif-file-output=snyk.sarif --skip-unresolved=true | |
| - name: Upload Result to GitHub Code Scanning | |
| uses: github/codeql-action/upload-sarif@v2 | |
| with: | |
| sarif_file: snyk.sarif | |
| pytest: | |
| permissions: | |
| contents: 'read' | |
| id-token: 'write' | |
| runs-on: ubuntu-22.04 | |
| needs: [snyk] | |
| if: | | |
| always() && | |
| (needs.snyk.result == 'success') | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: 'actions/[email protected]' | |
| with: | |
| python-version: 3.12.0 | |
| - name: Install Dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements.txt | |
| pip install pytest==7.4.3 | |
| pip install pytest-cov==4.1.0 | |
| - name: Google Auth | |
| id: auth | |
| uses: 'google-github-actions/auth@v2' | |
| with: | |
| token_format: 'access_token' | |
| workload_identity_provider: '${{ secrets.WIF_PROVIDER }}' | |
| service_account: '${{ secrets.WIF_SERVICE_ACCOUNT_DATABASE }}' | |
| project_id: '${{ secrets.PROJECT_ID }}' | |
| - name: Run Tests | |
| run: | | |
| pytest --cov=streamlit_app tests/ -v | |
| build-streamlit-image: | |
| runs-on: ubuntu-22.04 | |
| needs: [pytest] | |
| if: | | |
| always() && | |
| (needs.pytest.result == 'success') | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up Docker Buildx | |
| uses: 'docker/[email protected]' | |
| - name: Prepare Docker Build Context | |
| run: | | |
| mkdir docker-context | |
| cp ./.dockerignore docker-context | |
| cp ./Dockerfile docker-context | |
| cp -r ./components docker-context/components | |
| cp ./streamlit_app.py docker-context | |
| cp ./requirements.txt docker-context | |
| - name: Build and Export | |
| uses: 'docker/[email protected]' | |
| with: | |
| context: ./docker-context | |
| tags: streamlit:${{ env.VERSION_NUMBER }} | |
| outputs: type=docker,dest=/tmp/streamlit.tar | |
| - name: Upload Artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: streamlit | |
| path: /tmp/streamlit.tar | |
| push-artifact-registry: | |
| permissions: | |
| contents: 'read' | |
| id-token: 'write' | |
| runs-on: ubuntu-22.04 | |
| needs: [build-streamlit-image] | |
| if: | | |
| always() && | |
| (needs.build-streamlit-image.result == 'success') | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up Docker Buildx | |
| uses: 'docker/[email protected]' | |
| - name: Download Artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: streamlit | |
| path: /tmp | |
| - name: Google Auth | |
| id: auth | |
| uses: 'google-github-actions/auth@v2' | |
| with: | |
| token_format: 'access_token' | |
| workload_identity_provider: '${{ secrets.WIF_PROVIDER }}' | |
| service_account: '${{ secrets.WIF_SERVICE_ACCOUNT_ARTIFACT_REGISTRY }}' | |
| project_id: '${{ secrets.PROJECT_ID }}' | |
| - name: Docker Auth | |
| id: docker-auth | |
| uses: 'docker/login-action@v3' | |
| with: | |
| username: 'oauth2accesstoken' | |
| password: '${{ steps.auth.outputs.access_token }}' | |
| registry: '${{ env.GAR_LOCATION }}-docker.pkg.dev' | |
| - name: Load Image | |
| run: | | |
| docker load --input /tmp/streamlit.tar | |
| docker image ls -a | |
| - name: Tag Image | |
| run: | | |
| docker tag \ | |
| streamlit:${{ env.VERSION_NUMBER }} \ | |
| "${{ env.GAR_LOCATION }}-docker.pkg.dev/${{ secrets.PROJECT_ID }}/${{ secrets.REGISTRY_REPO }}/streamlit:${{ env.VERSION_NUMBER }}" | |
| - name: Push Image | |
| run: | | |
| docker push \ | |
| "${{ env.GAR_LOCATION }}-docker.pkg.dev/${{ secrets.PROJECT_ID }}/${{ secrets.REGISTRY_REPO }}/streamlit:${{ env.VERSION_NUMBER }}" | |
| deploy-streamlit-image: | |
| permissions: | |
| contents: 'read' | |
| id-token: 'write' | |
| runs-on: ubuntu-22.04 | |
| needs: [push-artifact-registry] | |
| if: | | |
| always() && | |
| (needs.push-artifact-registry.result == 'success') | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Google Auth | |
| id: auth | |
| uses: 'google-github-actions/auth@v2' | |
| with: | |
| token_format: 'access_token' | |
| workload_identity_provider: '${{ secrets.WIF_PROVIDER }}' | |
| service_account: '${{ secrets.WIF_SERVICE_ACCOUNT_CLOUD_RUN }}' | |
| project_id: '${{ secrets.PROJECT_ID }}' | |
| - name: Deploy Image | |
| id: 'deploy' | |
| uses: 'google-github-actions/[email protected]' | |
| with: | |
| service: streamlit | |
| image: "${{ env.GAR_LOCATION }}-docker.pkg.dev/${{ secrets.PROJECT_ID }}/${{ secrets.REGISTRY_REPO }}/streamlit:${{ env.VERSION_NUMBER }}" | |
| flags: "--service-account=${{ secrets.WIF_SERVICE_ACCOUNT_CLOUD_RUN }} --max-instances=3" | |
| no_traffic: false | |
| set-latest-revision: | |
| permissions: | |
| contents: 'read' | |
| id-token: 'write' | |
| runs-on: ubuntu-22.04 | |
| needs: [deploy-streamlit-image] | |
| if: | | |
| always() && | |
| (needs.deploy-streamlit-image.result == 'success') | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Google Auth | |
| id: auth | |
| uses: 'google-github-actions/auth@v2' | |
| with: | |
| token_format: 'access_token' | |
| workload_identity_provider: '${{ secrets.WIF_PROVIDER }}' | |
| service_account: '${{ secrets.WIF_SERVICE_ACCOUNT_CLOUD_RUN }}' | |
| project_id: '${{ secrets.PROJECT_ID }}' | |
| - name: 'Set up Cloud SDK' | |
| uses: 'google-github-actions/setup-gcloud@v2' | |
| - name: 'Set Latest Revision' | |
| run: | | |
| gcloud run services update-traffic streamlit \ | |
| --to-latest \ | |
| --region=${{ env.GAR_LOCATION }} \ | |
| architecture-build: | |
| runs-on: ubuntu-22.04 | |
| needs: [pytest] | |
| if: | | |
| always() && | |
| (needs.pytest.result == 'success') | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| platform: [linux/amd64, linux/arm64] | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Docker Meta | |
| id: meta | |
| uses: 'docker/[email protected]' | |
| with: | |
| images: ${{ env.REGISTRY_IMAGE }} | |
| - name: Set up QEMU | |
| uses: 'docker/setup-qemu-action@v3' | |
| - name: Set up Docker Buildx | |
| uses: 'docker/[email protected]' | |
| - name: Login to Docker Hub | |
| uses: 'docker/login-action@v3' | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Build and Push by Digest | |
| id: build | |
| uses: 'docker/[email protected]' | |
| with: | |
| context: . | |
| platforms: ${{ matrix.platform }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=true | |
| - name: Export Digest | |
| run: | | |
| mkdir -p /tmp/digests | |
| digest="${{ steps.build.outputs.digest }}" | |
| touch "/tmp/digests/${digest#sha256:}" | |
| - name: Upload Digest for AMD64 | |
| if: matrix.platform == 'linux/amd64' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: digests-amd64 | |
| path: /tmp/digests/* | |
| if-no-files-found: error | |
| retention-days: 1 | |
| - name: Upload Digest for ARM64 | |
| if: matrix.platform == 'linux/arm64' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: digests-arm64 | |
| path: /tmp/digests/* | |
| if-no-files-found: error | |
| retention-days: 1 | |
| create-manifest-and-push: | |
| runs-on: ubuntu-22.04 | |
| needs: | |
| - architecture-build | |
| steps: | |
| - name: Download Digests | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: digests-* | |
| path: /tmp/digests | |
| merge-multiple: true | |
| - name: Set up Docker Buildx | |
| uses: 'docker/[email protected]' | |
| - name: Docker meta | |
| id: meta | |
| uses: 'docker/[email protected]' | |
| with: | |
| images: ${{ env.REGISTRY_IMAGE }} | |
| tags: ${{ env.VERSION_NUMBER }} | |
| - name: Login to Docker Hub | |
| uses: 'docker/login-action@v3' | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Create Manifest List and Push | |
| working-directory: /tmp/digests | |
| run: | | |
| docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ | |
| $(printf '${{ env.REGISTRY_IMAGE }}@sha256:%s ' *) | |
| - name: Inspect image | |
| run: | | |
| docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:${{ steps.meta.outputs.version }} | |
| syft: | |
| permissions: | |
| contents: 'read' | |
| id-token: 'write' | |
| runs-on: ubuntu-22.04 | |
| needs: [build-streamlit-image] | |
| if: | | |
| always() && | |
| (needs.build-streamlit-image.result == 'success') | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up Docker Buildx | |
| uses: 'docker/[email protected]' | |
| - name: Download Artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: streamlit | |
| path: /tmp | |
| - name: Load Image | |
| run: | | |
| docker load --input /tmp/streamlit.tar | |
| docker image ls -a | |
| - name: Create and Upload SBOM | |
| uses: anchore/sbom-action@v0 | |
| with: | |
| image: streamlit:${{ env.VERSION_NUMBER }} | |
| artifact-name: streamlit-sbom-${{ env.VERSION_NUMBER }}.spdx.json | |
| upload-artifact: true | |
| grype: | |
| permissions: | |
| actions: read | |
| contents: read | |
| security-events: write | |
| runs-on: ubuntu-22.04 | |
| needs: [syft] | |
| if: | | |
| always() && | |
| (needs.syft.result == 'success') | |
| steps: | |
| - name: Download SBOM | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: streamlit-sbom-${{ env.VERSION_NUMBER }}.spdx.json | |
| - name: Scan SBOM | |
| uses: anchore/scan-action@v3 | |
| id: scan | |
| with: | |
| sbom: streamlit-sbom-${{ env.VERSION_NUMBER }}.spdx.json | |
| fail-build: false | |
| output-format: sarif | |
| severity-cutoff: critical | |
| - name: Upload SARIF Report | |
| uses: github/codeql-action/upload-sarif@v2 | |
| with: | |
| sarif_file: ${{ steps.scan.outputs.sarif }} | |