Build images for Update .gitignore https://github.com/potiuk/airflow/pull/290 #528

Workflow file for this run

.github/workflows/build-images.yml at e4c2c0f

	# Licensed to the Apache Software Foundation (ASF) under one
	# or more contributor license agreements. See the NOTICE file
	# distributed with this work for additional information
	# regarding copyright ownership. The ASF licenses this file
	# to you under the Apache License, Version 2.0 (the
	# "License"); you may not use this file except in compliance
	# with the License. You may obtain a copy of the License at
	#
	# http://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing,
	# software distributed under the License is distributed on an
	# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
	# KIND, either express or implied. See the License for the
	# specific language governing permissions and limitations
	# under the License.
	#
	---
	name: "Build Images"
	run-name: >
	Build images for ${{ github.event.pull_request.title }} ${{ github.event.pull_request._links.html.href }}
	on: # yamllint disable-line rule:truthy
	pull_request_target:
	permissions:
	# all other permissions are set to none
	contents: read
	pull-requests: read
	packages: read
	env:
	ANSWER: "yes"
	DB_RESET: "true"
	VERBOSE: "true"
	GITHUB_REPOSITORY: ${{ github.repository }}
	GITHUB_USERNAME: ${{ github.actor }}
	# You can override CONSTRAINTS_GITHUB_REPOSITORY by setting secret in your repo but by default the
	# Airflow one is going to be used
	CONSTRAINTS_GITHUB_REPOSITORY: >-
	${{ secrets.CONSTRAINTS_GITHUB_REPOSITORY != '' &&
	secrets.CONSTRAINTS_GITHUB_REPOSITORY \|\| 'apache/airflow' }}
	# This token is WRITE one - pull_request_target type of events always have the WRITE token
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
	IMAGE_TAG: "${{ github.event.pull_request.head.sha \|\| github.sha }}"
	USE_SUDO: "true"
	INCLUDE_SUCCESS_OUTPUTS: "true"
	# Version of Python used for reproducibility of the packages built
	# Python 3.8 tarfile produces different tarballs than Python 3.9+ tarfile that's why we are forcing
	# Python 3.9 for all release preparation commands to make sure that the tarballs are reproducible
	# TODO: remove me when we switch to Python 3.9 as minimal version
	REPRODUCIBLE_PYTHON_VERSION: "3.9"

	concurrency:
	group: build-${{ github.event.pull_request.number \|\| github.ref }}
	cancel-in-progress: true

	jobs:
	build-info:
	timeout-minutes: 10
	name: "Build Info"
	runs-on: 'ubuntu-22.04'
	env:
	TARGET_BRANCH: ${{ github.event.pull_request.base.ref }}
	outputs:
	python-versions: "${{ steps.selective-checks.python-versions }}"
	upgrade-to-newer-dependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }}
	all-python-versions-list-as-string: >-
	${{ steps.selective-checks.outputs.all-python-versions-list-as-string }}
	default-python-version: ${{ steps.selective-checks.outputs.default-python-version }}
	run-tests: ${{ steps.selective-checks.outputs.run-tests }}
	run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }}
	ci-image-build: ${{ steps.selective-checks.outputs.ci-image-build }}
	prod-image-build: ${{ steps.selective-checks.outputs.prod-image-build }}
	cache-directive: ${{ steps.selective-checks.outputs.cache-directive }}
	default-branch: ${{ steps.selective-checks.outputs.default-branch }}
	default-constraints-branch: ${{ steps.selective-checks.outputs.default-constraints-branch }}
	runs-on: ${{steps.selective-checks.outputs.runs-on}}
	is-self-hosted-runner: ${{ steps.selective-checks.outputs.is-self-hosted-runner }}
	is-committer-build: ${{ steps.selective-checks.outputs.is-committer-build }}
	is-airflow-runner: ${{ steps.selective-checks.outputs.is-airflow-runner }}
	is-amd-runner: ${{ steps.selective-checks.outputs.is-amd-runner }}
	is-arm-runner: ${{ steps.selective-checks.outputs.is-arm-runner }}
	is-vm-runner: ${{ steps.selective-checks.outputs.is-vm-runner }}
	is-k8s-runner: ${{ steps.selective-checks.outputs.is-k8s-runner }}
	chicken-egg-providers: ${{ steps.selective-checks.outputs.chicken-egg-providers }}
	target-commit-sha: "${{steps.discover-pr-merge-commit.outputs.target-commit-sha \|\|
	github.event.pull_request.head.sha \|\|
	github.sha
	}}"
	if: github.repository == 'apache/airflow'
	steps:
	- name: Discover PR merge commit
	id: discover-pr-merge-commit
	run: \|
	TARGET_COMMIT_SHA="$(gh api '${{ github.event.pull_request.url }}' --jq .merge_commit_sha)"
	echo "TARGET_COMMIT_SHA=$TARGET_COMMIT_SHA" >> ${GITHUB_ENV}
	echo "target-commit-sha=${TARGET_COMMIT_SHA}" >> ${GITHUB_OUTPUT}
	if: github.event_name == 'pull_request_target'
	# The labels in the event aren't updated when re-triggering the job, So lets hit the API to get
	# up-to-date values
	- name: Get latest PR labels
	id: get-latest-pr-labels
	run: \|
	echo -n "pull-request-labels=" >> ${GITHUB_OUTPUT}
	gh api graphql --paginate -F node_id=${{github.event.pull_request.node_id}} -f query='
	query($node_id: ID!, $endCursor: String) {
	node(id:$node_id) {
	... on PullRequest {
	labels(first: 100, after: $endCursor) {
	nodes { name }
	pageInfo { hasNextPage endCursor }
	}
	}
	}
	}' --jq '.data.node.labels.nodes[]' \| jq --slurp -c '[.[].name]' >> ${GITHUB_OUTPUT}
	if: github.event_name == 'pull_request_target'
	- name: Cleanup repo
	run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
	- uses: actions/checkout@v4
	with:
	ref: ${{ env.TARGET_COMMIT_SHA }}
	persist-credentials: false
	fetch-depth: 2
	####################################################################################################
	# WE ONLY DO THAT CHECKOUT ABOVE TO RETRIEVE THE TARGET COMMIT AND IT'S PARENT. DO NOT RUN ANY CODE
	# RIGHT AFTER THAT AS WE ARE GOING TO RESTORE THE TARGET BRANCH CODE IN THE NEXT STEP.
	####################################################################################################
	- name: Checkout target branch to use ci/scripts and breeze from there.
	uses: actions/checkout@v4
	with:
	ref: ${{ github.base_ref }}
	persist-credentials: false
	####################################################################################################
	# HERE EVERYTHING IS PERFECTLY SAFE TO RUN. AT THIS POINT WE HAVE THE TARGET BRANCH CHECKED OUT
	# AND WE CAN RUN ANY CODE FROM IT. WE CAN RUN BREEZE COMMANDS, WE CAN RUN SCRIPTS, WE CAN RUN
	# COMPOSITE ACTIONS. WE CAN RUN ANYTHING THAT IS IN THE TARGET BRANCH AND THERE IS NO RISK THAT
	# CODE WILL BE RUN FROM THE PR.
	####################################################################################################
	- name: "Setup python"
	uses: actions/setup-python@v5
	with:
	python-version: 3.8
	- name: "Install Breeze"
	uses: ./.github/actions/breeze
	####################################################################################################
	# WE RUN SELECTIVE CHECKS HERE USING THE TARGET COMMIT AND ITS PARENT TO BE ABLE TO COMPARE THEM
	# AND SEE WHAT HAS CHANGED IN THE PR. THE CODE IS STILL RUN FROM THE TARGET BRANCH, SO IT IS SAFE
	# TO RUN IT, WE ONLY PASS TARGET_COMMIT_SHA SO THAT SELECTIVE CHECKS CAN SEE WHAT'S COMING IN THE PR
	####################################################################################################
	- name: Selective checks
	id: selective-checks
	env:
	PR_LABELS: "${{ steps.get-latest-pr-labels.outputs.pull-request-labels }}"
	COMMIT_REF: "${{ env.TARGET_COMMIT_SHA }}"
	VERBOSE: "false"
	AIRFLOW_SOURCES_ROOT: "${{ github.workspace }}"
	run: breeze ci selective-check 2>> ${GITHUB_OUTPUT}
	- name: env
	run: printenv
	env:
	PR_LABELS: ${{ steps.get-latest-pr-labels.outputs.pull-request-labels }}
	GITHUB_CONTEXT: ${{ toJson(github) }}

	build-ci-images:
	strategy:
	fail-fast: true
	matrix:
	python-version: ${{fromJson(needs.build-info.outputs.python-versions)}}
	permissions:
	contents: read
	packages: write
	timeout-minutes: 80
	name: Build CI image ${{ matrix.python-version }}
	runs-on: ["ubuntu-22.04"]
	needs: [build-info]
	if: \|
	needs.build-info.outputs.ci-image-build == 'true' &&
	github.event.pull_request.head.repo.full_name != 'apache/airflow'
	env:
	DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
	DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }}
	RUNS_ON: "${{ needs.build-info.outputs.runs-on }}"
	BACKEND: sqlite
	VERSION_SUFFIX_FOR_PYPI: "dev0"
	steps:
	- name: Cleanup repo
	run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
	- uses: actions/checkout@v4
	with:
	ref: ${{ needs.build-info.outputs.target-commit-sha }}
	persist-credentials: false
	####################################################################################################
	# BE VERY CAREFUL HERE! THIS LINE AND THE END OF THE WARNING. HERE WE CHECK OUT THE TARGET
	# COMMIT AND ITS PARENT TO BE ABLE TO COMPARE THEM BUT ALSO TO BE ABLE TO BUILD THE IMAGE FROM
	# THE INCOMING PR, RATHER THAN FROM TARGET BRANCH. THIS IS A SECURITY RISK, BECAUSE THE PR
	# CAN CONTAIN ANY CODE AND WE EXECUTE IT HERE. THEREFORE, WE NEED TO BE VERY CAREFUL WHAT WE
	# DO HERE. WE SHOULD NOT EXECUTE ANY CODE THAT COMES FROM THE PR. WE SHOULD NOT RUN ANY BREEZE
	# COMMAND NOR SCRIPTS NOR COMPOSITE ACTIONS. WE SHOULD ONLY RUN CODE THAT IS EMBEDDED DIRECTLY IN
	# THIS WORKFLOW - BECAUSE THIS IS THE ONLY CODE THAT WE CAN TRUST.
	####################################################################################################
	- name: Checkout target branch to 'target-airflow' folder to use ci/scripts and breeze from there.
	uses: actions/checkout@v4
	with:
	path: "target-airflow"
	ref: ${{ github.base_ref }}
	persist-credentials: false
	if: needs.build-info.outputs.is-committer-build != 'true'
	- name: >
	Replace "scripts/ci", "dev" and ".github/actions" with the target branch
	so that the those directories are not coming from the PR
	shell: bash
	run: \|
	echo
	echo -e "\033[33m Replace scripts, dev, actions with target branch for non-committer builds!\033[0m"
	echo
	rm -rfv "scripts/ci"
	mv -v "target-airflow/scripts/ci" "scripts"
	rm -rfv "dev"
	mv -v "target-airflow/dev" "."
	rm -rfv ".github/actions"
	mv -v "target-airflow/.github/actions" ".github"
	if: needs.build-info.outputs.is-committer-build != 'true'
	####################################################################################################
	# HERE IT'S A BIT SAFER. THE `dev`, `scripts/ci` AND `.github/actions` ARE NOW COMING FROM THE
	# BASE_REF - WHICH IS THE TARGET BRANCH OF THE PR. WE CAN TRUST THAT THOSE SCRIPTS ARE SAVE TO RUN.
	# ALL THE REST OF THE CODE COMES FROM THE PR, AND FOR EXAMPLE THE CODE IN THE `Dockerfile.ci` CAN
	# BE RUN SAFELY AS PART OF DOCKER BUILD. BECAUSE IT RUNS INSIDE THE DOCKER CONTAINER AND IT IS
	# ISOLATED FROM THE RUNNER.
	####################################################################################################
	- name: Build CI Image ${{ matrix.python-version }}:${{env.IMAGE_TAG}}
	uses: ./.github/actions/build-ci-images
	with:
	python-version: ${{ matrix.python-version }}
	env:
	UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
	DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
	PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}}
	DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }}
	BUILD_TIMEOUT_MINUTES: 70

	build-prod-images:
	strategy:
	fail-fast: true
	matrix:
	python-version: ${{fromJson(needs.build-info.outputs.python-versions)}}
	permissions:
	contents: read
	packages: write
	timeout-minutes: 80
	name: Build PROD image ${{ matrix.python-version }}
	runs-on: ["ubuntu-22.04"]
	needs: [build-info, build-ci-images]
	if: \|
	needs.build-info.outputs.prod-image-build == 'true' &&
	github.event.pull_request.head.repo.full_name != 'apache/airflow'
	env:
	DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
	DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }}
	RUNS_ON: "${{ needs.build-info.outputs.runs-on }}"
	BACKEND: sqlite
	VERSION_SUFFIX_FOR_PYPI: "dev0"
	INCLUDE_NOT_READY_PROVIDERS: "true"
	steps:
	- name: Cleanup repo
	run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
	- uses: actions/checkout@v4
	with:
	ref: ${{ needs.build-info.outputs.target-commit-sha }}
	persist-credentials: false
	####################################################################################################
	# BE VERY CAREFUL HERE! THIS LINE AND THE END OF THE WARNING. HERE WE CHECK OUT THE TARGET
	# COMMIT AND ITS PARENT TO BE ABLE TO COMPARE THEM BUT ALSO TO BE ABLE TO BUILD THE IMAGE FROM
	# THE INCOMING PR, RATHER THAN FROM TARGET BRANCH. THIS IS A SECURITY RISK, BECAUSE THE PR
	# CAN CONTAIN ANY CODE AND WE EXECUTE IT HERE. THEREFORE, WE NEED TO BE VERY CAREFUL WHAT WE
	# DO HERE. WE SHOULD NOT EXECUTE ANY CODE THAT COMES FROM THE PR. WE SHOULD NOT RUN ANY BREEZE
	# COMMAND NOR SCRIPTS NOR COMPOSITE ACTIONS. WE SHOULD ONLY RUN CODE THAT IS EMBEDDED DIRECTLY IN
	# THIS WORKFLOW - BECAUSE THIS IS THE ONLY CODE THAT WE CAN TRUST.
	####################################################################################################
	- name: Checkout target branch to 'target-airflow' folder to use ci/scripts and breeze from there.
	uses: actions/checkout@v4
	with:
	path: "target-airflow"
	ref: ${{ github.base_ref }}
	persist-credentials: false
	if: needs.build-info.outputs.is-committer-build != 'true'
	- name: >
	Replace "scripts/ci", "dev" and ".github/actions" with the target branch
	so that the those directories are not coming from the PR
	shell: bash
	run: \|
	echo
	echo -e "\033[33m Replace scripts, dev, actions with target branch for non-committer builds!\033[0m"
	echo
	rm -rfv "scripts/ci"
	mv -v "target-airflow/scripts/ci" "scripts"
	rm -rfv "dev"
	mv -v "target-airflow/dev" "."
	rm -rfv ".github/actions"
	mv -v "target-airflow/.github/actions" ".github"
	if: needs.build-info.outputs.is-committer-build != 'true'
	####################################################################################################
	# HERE IT'S A BIT SAFER. THE `dev`, `scripts/ci` AND `.github/actions` ARE NOW COMING FROM THE
	# BASE_REF - WHICH IS THE TARGET BRANCH OF THE PR. WE CAN TRUST THAT THOSE SCRIPTS ARE SAVE TO RUN.
	# ALL THE REST OF THE CODE COMES FROM THE PR, AND FOR EXAMPLE THE CODE IN THE `Dockerfile.ci` CAN
	# BE RUN SAFELY AS PART OF DOCKER BUILD. BECAUSE IT RUNS INSIDE THE DOCKER CONTAINER AND IT IS
	# ISOLATED FROM THE RUNNER.
	####################################################################################################
	- name: "Install Breeze"
	uses: ./.github/actions/breeze
	with:
	python-version: ${{ env.REPRODUCIBLE_PYTHON_VERSION }}
	- name: Build PROD Image ${{ matrix.python-version }}:${{env.IMAGE_TAG}}
	uses: ./.github/actions/build-prod-images
	with:
	build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }}
	chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }}
	python-version: ${{ matrix.python-version }}
	env:
	UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
	DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
	PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}}
	DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }}

	build-ci-images-arm:
	timeout-minutes: 50
	name: "Build ARM CI images ${{needs.build-info.outputs.all-python-versions-list-as-string}}"
	runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
	needs: [build-info, build-prod-images]
	# We can change the job to run on ASF ARM runners and do not start our instance once we enable ASF runners
	if: \|
	needs.build-info.outputs.ci-image-build == 'true' &&
	needs.build-info.outputs.upgrade-to-newer-dependencies != 'false' &&
	github.event.pull_request.head.repo.full_name != 'apache/airflow' &&
	needs.build-info.outputs.is-self-hosted-runner == 'true' &&
	needs.build-info.outputs.is-airflow-runner == 'true'
	env:
	DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
	DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }}
	RUNS_ON: "${{ needs.build-info.outputs.runs-on }}"
	BACKEND: sqlite
	outputs: ${{toJSON(needs.build-info.outputs) }}
	VERSION_SUFFIX_FOR_PYPI: "dev0"
	steps:
	- name: Cleanup repo
	run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
	- uses: actions/checkout@v4
	with:
	ref: ${{ needs.build-info.outputs.target-commit-sha }}
	persist-credentials: false
	####################################################################################################
	# BE VERY CAREFUL HERE! THIS LINE AND THE END OF THE WARNING. HERE WE CHECK OUT THE TARGET
	# COMMIT AND ITS PARENT TO BE ABLE TO COMPARE THEM BUT ALSO TO BE ABLE TO BUILD THE IMAGE FROM
	# THE INCOMING PR, RATHER THAN FROM TARGET BRANCH. THIS IS A SECURITY RISK, BECAUSE THE PR
	# CAN CONTAIN ANY CODE AND WE EXECUTE IT HERE. THEREFORE, WE NEED TO BE VERY CAREFUL WHAT WE
	# DO HERE. WE SHOULD NOT EXECUTE ANY CODE THAT COMES FROM THE PR. WE SHOULD NOT RUN ANY BREEZE
	# COMMAND NOR SCRIPTS NOR COMPOSITE ACTIONS. WE SHOULD ONLY RUN CODE THAT IS EMBEDDED DIRECTLY IN
	# THIS WORKFLOW - BECAUSE THIS IS THE ONLY CODE THAT WE CAN TRUST.
	####################################################################################################
	- name: Checkout target branch to 'target-airflow' folder to use ci/scripts and breeze from there.
	uses: actions/checkout@v4
	with:
	path: "target-airflow"
	ref: ${{ github.base_ref }}
	persist-credentials: false
	if: needs.build-info.outputs.is-committer-build != 'true'
	- name: >
	Replace "scripts/ci", "dev" and ".github/actions" with the target branch
	so that the those directories are not coming from the PR
	shell: bash
	run: \|
	echo
	echo -e "\033[33m Replace scripts, dev, actions with target branch for non-committer builds!\033[0m"
	echo
	rm -rfv "scripts/ci"
	mv -v "target-airflow/scripts/ci" "scripts"
	rm -rfv "dev"
	mv -v "target-airflow/dev" "."
	rm -rfv ".github/actions"
	mv -v "target-airflow/.github/actions" ".github"
	if: needs.build-info.outputs.is-committer-build != 'true'
	####################################################################################################
	# HERE IT'S A BIT SAFER. THE `dev`, `scripts/ci` AND `.github/actions` ARE NOW COMING FROM THE
	# BASE_REF - WHICH IS THE TARGET BRANCH OF THE PR. WE CAN TRUST THAT THOSE SCRIPTS ARE SAVE TO RUN.
	# ALL THE REST OF THE CODE COMES FROM THE PR, AND FOR EXAMPLE THE CODE IN THE `Dockerfile.ci` CAN
	# BE RUN SAFELY AS PART OF DOCKER BUILD. BECAUSE IT RUNS INSIDE THE DOCKER CONTAINER AND IT IS
	# ISOLATED FROM THE RUNNER.
	####################################################################################################
	- name: "Start ARM instance"
	run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh
	- name: "Install Breeze"
	uses: ./.github/actions/breeze
	- name: >
	Build ARM CI images ${{ env.IMAGE_TAG }}
	${{needs.build-info.outputs.all-python-versions-list-as-string}}
	run: >
	breeze ci-image build --run-in-parallel --builder airflow_cache --platform "linux/arm64"
	env:
	UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
	DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
	PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}}
	- name: "Stop ARM instance"
	run: ./scripts/ci/images/ci_stop_arm_instance.sh
	if: always()

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Build images for Update .gitignore https://github.com/potiuk/airflow/pull/290 #528

Workflow file

Build images for Update .gitignore https://github.com/potiuk/airflow/pull/290 #528

Jobs

Run details

Workflow file for this run