updated configs #25
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Based on: https://learn.microsoft.com/en-us/azure/databricks/dev-tools/bundles/ci-cd | |
| name: "ci-cd" | |
| # Ensure that only a single job or workflow using the same concurrency group runs at a time. | |
| concurrency: 1 | |
| # Trigger this workflow upon commit to main branch | |
| on: | |
| push: | |
| branches: | |
| - main | |
| jobs: | |
| build: | |
| name: "Build and Run Unit Testing" | |
| runs-on: ubuntu-latest | |
| steps: | |
| # Check out this repo, so that this workflow can access it. | |
| - uses: actions/checkout@v3 | |
| # Download the Databricks CLI. | |
| # See https://github.com/databricks/setup-cli | |
| - uses: databricks/setup-cli@main | |
| # Install poetry: https://python-poetry.org/docs/#installing-with-the-official-installer | |
| - run: curl -sSL https://install.python-poetry.org | python3 - | |
| # Install test dependencies | |
| - run: poetry install | |
| - run: databricks bundle build | |
| - run: | | |
| source $(poetry env info --path)/bin/activate | |
| pytest tests/unit --cov | |
| run-it: | |
| name: "Run Integration Tests" | |
| runs-on: ubuntu-latest | |
| needs: | |
| - build | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: databricks/setup-cli@main | |
| - run: curl -sSL https://install.python-poetry.org | python3 - | |
| - run: poetry install | |
| - run: | | |
| echo "[DEFAULT]" >> ~/.databrickscfg | |
| echo "host = ${{ secrets.DATABRICKS_TEST_HOST }}" >> ~/.databrickscfg | |
| echo "token = ${{ secrets.DATABRICKS_TEST_TOKEN }}" >> ~/.databrickscfg | |
| echo "cluster_id = ${{ secrets.DATABRICKS_TEST_CLUSTER_ID }}" >> ~/.databrickscfg | |
| - run: | | |
| source $(poetry env info --path)/bin/activate | |
| pytest tests/integration --cov | |
| deploy-qa: | |
| name: "Deploy Jobs in QA" | |
| runs-on: ubuntu-latest | |
| needs: | |
| - run-it | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: databricks/setup-cli@main | |
| - run: curl -sSL https://install.python-poetry.org | python3 - | |
| - run: databricks bundle deploy --target qa | |
| working-directory: . | |
| env: | |
| DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TEST_TOKEN }} | |
| run-e2e: | |
| name: "Run End to End Tests" | |
| runs-on: ubuntu-latest | |
| needs: | |
| - deploy-qa | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: databricks/setup-cli@main | |
| - run: curl -sSL https://install.python-poetry.org | python3 - | |
| - run: poetry install | |
| - run: | | |
| echo "[DEFAULT]" >> ~/.databrickscfg | |
| echo "host = ${{ secrets.DATABRICKS_TEST_HOST }}" >> ~/.databrickscfg | |
| echo "token = ${{ secrets.DATABRICKS_TEST_TOKEN }}" >> ~/.databrickscfg | |
| - run: | | |
| source $(poetry env info --path)/bin/activate | |
| pytest tests/e2e --cov | |
| deploy-and-run-prod: | |
| name: "Deploy and Start Jobs in Prod" | |
| runs-on: ubuntu-latest | |
| needs: | |
| - run-e2e | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: databricks/setup-cli@main | |
| - run: curl -sSL https://install.python-poetry.org | python3 - | |
| - run: | | |
| databricks bundle deploy --target prod | |
| # specify the job node here as defined in the job.yml (not the job name!) | |
| databricks bundle run marcin_project_job --refresh-all --target prod | |
| working-directory: . | |
| env: | |
| DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_PROD_TOKEN }} |