updated config #38
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Based on: https://learn.microsoft.com/en-us/azure/databricks/dev-tools/bundles/ci-cd | |
| name: "ci-cd" | |
| # Ensure that only a single job or workflow using the same concurrency group runs at a time. | |
| concurrency: 1 | |
| # Trigger this workflow upon commit to main branch | |
| on: | |
| push: | |
| branches: | |
| - main | |
| jobs: | |
| build: | |
| name: "Build and Run Unit Testing" | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: trstringer/[email protected] | |
| with: | |
| secret: ${{ secrets.GITHUB_TOKEN }} | |
| approvers: mwojtyczka | |
| minimum-approvals: 1 | |
| issue-title: "Deploying to PROD from QA" | |
| issue-body: "Please approve or deny the deployment to PROD." | |
| additional-approved-words: '' | |
| additional-denied-words: '' | |
| # Check out this repo, so that this workflow can access it. | |
| - uses: actions/checkout@v3 | |
| # Download the Databricks CLI. | |
| # See https://github.com/databricks/setup-cli | |
| - uses: databricks/setup-cli@main | |
| # Install poetry: https://python-poetry.org/docs/#installing-with-the-official-installer | |
| - run: curl -sSL https://install.python-poetry.org | python3 - | |
| # Install test dependencies | |
| - run: poetry install | |
| # Build the bundle and run unit tests | |
| - run: databricks bundle build | |
| - run: | | |
| source $(poetry env info --path)/bin/activate | |
| pytest tests/unit --cov | |
| run-it: | |
| name: "Run Integration Tests" | |
| runs-on: ubuntu-latest | |
| needs: | |
| - build | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: databricks/setup-cli@main | |
| - run: curl -sSL https://install.python-poetry.org | python3 - | |
| - run: poetry install | |
| - run: | |
| source $(poetry env info --path)/bin/activate | |
| pytest tests/integration --cov | |
| env: | |
| DATABRICKS_HOST: ${{ secrets.DATABRICKS_TEST_HOST }} | |
| DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TEST_TOKEN }} | |
| DATABRICKS_CLUSTER_ID: ${{ secrets.DATABRICKS_TEST_CLUSTER_ID }} | |
| deploy-qa: | |
| name: "Deploy Jobs in QA" | |
| runs-on: ubuntu-latest | |
| needs: | |
| - run-it | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: databricks/setup-cli@main | |
| - run: curl -sSL https://install.python-poetry.org | python3 - | |
| - run: databricks bundle deploy --target qa | |
| working-directory: . | |
| env: | |
| DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TEST_TOKEN }} | |
| run-e2e: | |
| name: "Run End to End Tests" | |
| runs-on: ubuntu-latest | |
| needs: | |
| - deploy-qa | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: databricks/setup-cli@main | |
| - run: curl -sSL https://install.python-poetry.org | python3 - | |
| - run: poetry install | |
| - run: | | |
| source $(poetry env info --path)/bin/activate | |
| pytest tests/e2e --cov | |
| env: | |
| DATABRICKS_HOST: ${{ secrets.DATABRICKS_TEST_HOST }} | |
| DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TEST_TOKEN }} | |
| DATABRICKS_CLUSTER_ID: ${{ secrets.DATABRICKS_TEST_CLUSTER_ID }} | |
| deploy-and-run-prod: | |
| name: "Deploy and Start Jobs in Prod" | |
| runs-on: ubuntu-latest | |
| needs: | |
| - run-e2e | |
| steps: | |
| - uses: trstringer/[email protected] | |
| with: | |
| issue-title: "Deploying to PROD from QA" | |
| issue-body: "Please approve or deny the deployment to PROD." | |
| additional-approved-words: '' | |
| additional-denied-words: '' | |
| - uses: actions/checkout@v3 | |
| - uses: databricks/setup-cli@main | |
| - run: curl -sSL https://install.python-poetry.org | python3 - | |
| - run: | | |
| databricks bundle deploy --target prod | |
| # specify the job node here as defined in the job.yml (not the job name!) | |
| databricks bundle run marcin_project_job --refresh-all --target prod | |
| working-directory: . | |
| env: | |
| DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_PROD_TOKEN }} |