-
Notifications
You must be signed in to change notification settings - Fork 4k
218 lines (193 loc) · 8.18 KB
/
nightly_build.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
name: Nightly Build
on:
workflow_dispatch:
schedule:
# Run job at 6:30 UTC, 10.30pm PST, or 11.30pm PDT
- cron: "30 6 * * *"
env:
POETRY_VERSION: "1.8.3"
PYTHON_VERSION: "3.12"
jobs:
create-nightly-tag:
if: github.repository == 'langflow-ai/langflow'
runs-on: ubuntu-latest
defaults:
run:
shell: bash
permissions:
# Required to create tag
contents: write
outputs:
main_tag: ${{ steps.generate_main_tag.outputs.main_tag }}
base_tag: ${{ steps.set_base_tag.outputs.base_tag }}
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
persist-credentials: true
- name: Set up Python ${{ env.PYTHON_VERSION }} + Poetry ${{ env.POETRY_VERSION }}
uses: "./.github/actions/poetry_caching"
with:
python-version: ${{ env.PYTHON_VERSION }}
poetry-version: ${{ env.POETRY_VERSION }}
cache-key: ${{ runner.os }}-poetry-${{ env.POETRY_VERSION }}-${{ hashFiles('**/poetry.lock') }}
- name: Install dependencies
run: |
poetry env use ${{ env.PYTHON_VERSION }}
poetry install
- name: Generate main nightly tag
id: generate_main_tag
run: |
# NOTE: This outputs the tag with the `v` prefix.
MAIN_TAG="$(poetry run python ./scripts/ci/pypi_nightly_tag.py main)"
echo "main_tag=$MAIN_TAG" >> $GITHUB_OUTPUT
echo "main_tag=$MAIN_TAG"
- name: Check if main tag already exists
id: check_main_tag
run: |
git fetch --tags
if git rev-parse -q --verify "refs/tags/${{ steps.generate_main_tag.outputs.main_tag }}" >/dev/null; then
echo "main_tag_exists=true" >> $GITHUB_OUTPUT
else
echo "main_tag_exists=false" >> $GITHUB_OUTPUT
fi
- name: Generate base nightly tag
id: generate_base_tag
if: ${{ steps.check_main_tag.outputs.main_tag_exists == 'false' }}
run: |
# NOTE: This outputs the tag with the `v` prefix.
BASE_TAG="$(poetry run python ./scripts/ci/pypi_nightly_tag.py base)"
echo "base_tag=$BASE_TAG" >> $GITHUB_OUTPUT
echo "base_tag=$BASE_TAG"
- name: Commit tag
id: commit_tag
if: ${{ steps.check_main_tag.outputs.main_tag_exists == 'false' }}
run: |
# If the main tag does not exist in GH, we create the base tag from the existing codebase.
git config --global user.email "[email protected]"
git config --global user.name "Langflow Bot"
# WARNING: These scripts must be run in this order.
# Poetry will use a different cached virtual environment once the main pyproject.toml
# project-name is updated, which does not have dependencies installed.
BASE_TAG="${{ steps.generate_base_tag.outputs.base_tag }}"
echo "Updating base project version to $BASE_TAG"
poetry run python ./scripts/ci/update_pyproject_name.py langflow-base-nightly base
poetry run python ./scripts/ci/update_pyproject_version.py $BASE_TAG base
# This updates the dependency of langflow-base to langflow-base-nightly in {project_root}/pyproject.toml
poetry run python ./scripts/ci/update_lf_base_dependency.py $BASE_TAG
# Use the main tag created earlier
MAIN_TAG="${{ steps.generate_main_tag.outputs.main_tag }}"
echo "Updating main project version to $MAIN_TAG"
poetry run python ./scripts/ci/update_pyproject_version.py $MAIN_TAG main
poetry run python ./scripts/ci/update_pyproject_name.py langflow-nightly main
git add pyproject.toml src/backend/base/pyproject.toml
git commit -m "Update version and project name"
echo "Tagging main with $MAIN_TAG"
if ! git tag -a $MAIN_TAG -m "Langflow nightly $MAIN_TAG"; then
echo "Tag creation failed. Exiting the workflow."
exit 1
fi
echo "Pushing main tag $MAIN_TAG"
if ! git push origin $MAIN_TAG; then
echo "Tag push failed. Check if the tag already exists. Exiting the workflow."
exit 1
fi
# TODO: notify on failure
- name: Checkout main nightly tag
uses: actions/checkout@v4
if: ${{ steps.check_main_tag.outputs.main_tag_exists == 'true' }}
with:
ref: ${{ steps.generate_main_tag.outputs.main_tag }}
- name: Retrieve Base Tag
id: retrieve_base_tag
if: ${{ steps.check_main_tag.outputs.main_tag_exists == 'true' }}
working-directory: src/backend/base
run: |
# If the main tag already exists, we need to retrieve the base version from the main tag codebase.
version=$(poetry version --short)
echo "base_tag=v$version" >> $GITHUB_OUTPUT
echo "base_tag=v$version"
- name: Set Base Tag
id: set_base_tag
run: |
if [ "${{ steps.retrieve_base_tag.conclusion }}" != "skipped" ] && [ "${{ steps.retrieve_base_tag.outputs.base_tag }}" ]; then
BASE_TAG="${{ steps.retrieve_base_tag.outputs.base_tag }}"
echo "base_tag=$BASE_TAG" >> $GITHUB_OUTPUT
echo "base_tag=$BASE_TAG"
elif [ "${{ steps.commit_tag.conclusion }}" != "skipped" ] && [ "${{ steps.generate_base_tag.outputs.base_tag }}" ]; then
BASE_TAG="${{ steps.generate_base_tag.outputs.base_tag }}"
echo "base_tag=$BASE_TAG" >> $GITHUB_OUTPUT
echo "base_tag=$BASE_TAG"
else
echo "No base tag found. Exiting the workflow."
exit 1
fi
frontend-tests:
if: github.repository == 'langflow-ai/langflow'
name: Run Frontend Tests
needs: create-nightly-tag
uses: ./.github/workflows/typescript_test.yml
with:
tests_folder: "tests"
secrets:
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
STORE_API_KEY: ${{ secrets.STORE_API_KEY }}
backend-unit-tests:
if: github.repository == 'langflow-ai/langflow'
name: Run Backend Unit Tests
needs: create-nightly-tag
uses: ./.github/workflows/python_test.yml
with:
python-versions: '["3.10", "3.11", "3.12"]'
# Not making nightly builds dependent on integration test success
# due to inherent flakiness of 3rd party integrations
# Revisit when https://github.com/langflow-ai/langflow/pull/3607 is merged.
# backend-integration-tests:
# name: Run Backend Integration Tests
# needs: create-nightly-tag
# uses: ./.github/workflows/integration_tests.yml
# with:
# python-versions: '["3.10", "3.11", "3.12"]'
# ref: ${{ needs.create-nightly-tag.outputs.tag }}
release-nightly-build:
if: github.repository == 'langflow-ai/langflow'
name: Run Nightly Langflow Build
needs: [frontend-tests, backend-unit-tests, create-nightly-tag]
uses: ./.github/workflows/release_nightly.yml
with:
build_docker_base: true
build_docker_main: true
nightly_tag_main: ${{ needs.create-nightly-tag.outputs.main_tag }}
nightly_tag_base: ${{ needs.create-nightly-tag.outputs.base_tag }}
secrets: inherit
# slack-notification:
# name: Send Slack Notification
# needs: run-nightly-build
# runs-on: ubuntu-latest
# steps:
# - name: Send success notification to Slack
# if: success()
# uses: slackapi/[email protected]
# with:
# payload: |
# {
# "channel": "#langflow-nightly-builds",
# "username": "GitHub Actions",
# "text": "Nightly Build Successful :white_check_mark:",
# "icon_emoji": ":rocket:"
# }
# env:
# SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
# - name: Send failure notification to Slack
# if: failure()
# uses: slackapi/[email protected]
# with:
# payload: |
# {
# "channel": "#langflow-nightly-builds",
# "username": "GitHub Actions",
# "text": "Nightly Build Failed :x:",
# "icon_emoji": ":warning:"
# }
# env:
# SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}