-
Notifications
You must be signed in to change notification settings - Fork 0
126 lines (106 loc) · 3.81 KB
/
ci-cd.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
# Based on: https://learn.microsoft.com/en-us/azure/databricks/dev-tools/bundles/ci-cd
name: "ci-cd"
# Ensure that only a single job or workflow using the same concurrency group runs at a time.
concurrency: 1
# Trigger this workflow upon commit to main branch
on:
push:
branches:
- main
jobs:
build:
name: "Build and Run Unit Testing"
runs-on: ubuntu-latest
steps:
- uses: trstringer/[email protected]
with:
secret: ${{ secrets.GITHUB_TOKEN }}
approvers: mwojtyczka
minimum-approvals: 1
issue-title: "Deploying to PROD from QA"
issue-body: "Please approve or deny the deployment to PROD."
additional-approved-words: ''
additional-denied-words: ''
# Check out this repo, so that this workflow can access it.
- uses: actions/checkout@v3
# Download the Databricks CLI.
# See https://github.com/databricks/setup-cli
- uses: databricks/setup-cli@main
# Install poetry: https://python-poetry.org/docs/#installing-with-the-official-installer
- run: curl -sSL https://install.python-poetry.org | python3 -
# Install test dependencies
- run: poetry install
# Build the bundle and run unit tests
- run: databricks bundle build
- run: |
source $(poetry env info --path)/bin/activate
pytest tests/unit --cov
run-it:
name: "Run Integration Tests"
runs-on: ubuntu-latest
needs:
- build
steps:
- uses: actions/checkout@v3
- uses: databricks/setup-cli@main
- run: curl -sSL https://install.python-poetry.org | python3 -
- run: poetry install
- run:
source $(poetry env info --path)/bin/activate
pytest tests/integration --cov
env:
DATABRICKS_HOST: ${{ secrets.DATABRICKS_TEST_HOST }}
DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TEST_TOKEN }}
DATABRICKS_CLUSTER_ID: ${{ secrets.DATABRICKS_TEST_CLUSTER_ID }}
deploy-qa:
name: "Deploy Jobs in QA"
runs-on: ubuntu-latest
needs:
- run-it
steps:
- uses: actions/checkout@v3
- uses: databricks/setup-cli@main
- run: curl -sSL https://install.python-poetry.org | python3 -
- run: databricks bundle deploy --target qa
working-directory: .
env:
DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TEST_TOKEN }}
run-e2e:
name: "Run End to End Tests"
runs-on: ubuntu-latest
needs:
- deploy-qa
steps:
- uses: actions/checkout@v3
- uses: databricks/setup-cli@main
- run: curl -sSL https://install.python-poetry.org | python3 -
- run: poetry install
- run: |
source $(poetry env info --path)/bin/activate
pytest tests/e2e --cov
env:
DATABRICKS_HOST: ${{ secrets.DATABRICKS_TEST_HOST }}
DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TEST_TOKEN }}
DATABRICKS_CLUSTER_ID: ${{ secrets.DATABRICKS_TEST_CLUSTER_ID }}
deploy-and-run-prod:
name: "Deploy and Start Jobs in Prod"
runs-on: ubuntu-latest
needs:
- run-e2e
steps:
- uses: trstringer/[email protected]
with:
issue-title: "Deploying to PROD from QA"
issue-body: "Please approve or deny the deployment to PROD."
additional-approved-words: ''
additional-denied-words: ''
- uses: actions/checkout@v3
- uses: databricks/setup-cli@main
- run: curl -sSL https://install.python-poetry.org | python3 -
- run: |
databricks bundle deploy --target prod
# specify the job node here as defined in the job.yml (not the job name!)
databricks bundle run marcin_project_job --refresh-all --target prod
working-directory: .
env:
DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_PROD_TOKEN }}