diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 88f834050b61..1385d405a20f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -167,6 +167,12 @@ repos: \.cfg$|\.conf$|\.ini$|\.ldif$|\.properties$|\.readthedocs$|\.service$|\.tf$|Dockerfile.*$ - repo: local hooks: + - id: check-min-python-version + name: Check minimum Python version + entry: ./scripts/ci/pre_commit/check_min_python_version.py + language: python + additional_dependencies: ['rich>=12.4.4'] + require_serial: true - id: update-common-sql-api-stubs name: Check and update common.sql API stubs entry: ./scripts/ci/pre_commit/update_common_sql_api_stubs.py diff --git a/contributing-docs/08_static_code_checks.rst b/contributing-docs/08_static_code_checks.rst index b122b3486756..211ee7c9c34d 100644 --- a/contributing-docs/08_static_code_checks.rst +++ b/contributing-docs/08_static_code_checks.rst @@ -196,6 +196,8 @@ require Breeze Docker image to be built locally. +-----------------------------------------------------------+--------------------------------------------------------------+---------+ | check-merge-conflict | Check that merge conflicts are not being committed | | +-----------------------------------------------------------+--------------------------------------------------------------+---------+ +| check-min-python-version | Check minimum Python version | | ++-----------------------------------------------------------+--------------------------------------------------------------+---------+ | check-newsfragments-are-valid | Check newsfragments are valid | | +-----------------------------------------------------------+--------------------------------------------------------------+---------+ | check-no-airflow-deprecation-in-providers | Do not use DeprecationWarning in providers | | diff --git a/dev/breeze/doc/images/output-commands.svg b/dev/breeze/doc/images/output-commands.svg index 5888d1fc862e..08d3dc2a13ee 100644 --- a/dev/breeze/doc/images/output-commands.svg +++ b/dev/breeze/doc/images/output-commands.svg @@ -298,53 +298,53 @@ Usage:breeze[OPTIONSCOMMAND [ARGS]... ╭─ Execution mode ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ ---python-pPython major/minor version used in Airflow image for images. +--python-pPython major/minor version used in Airflow image for images. (>3.8< | 3.9 | 3.10 | 3.11 | 3.12)                           [default: 3.8]                                               ---integrationIntegration(s) to enable when running (can be more than one).                        +--integrationIntegration(s) to enable when running (can be more than one).                        (all | all-testable | cassandra | celery | drill | kafka | kerberos | mongo | mssql  | openlineage | otel | pinot | qdrant | redis | statsd | trino | ydb)                ---standalone-dag-processorRun standalone dag processor for start-airflow. ---database-isolationRun airflow in database isolation mode. +--standalone-dag-processorRun standalone dag processor for start-airflow. +--database-isolationRun airflow in database isolation mode. ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Docker Compose selection and cleanup ───────────────────────────────────────────────────────────────────────────────╮ ---project-nameName of the docker-compose project to bring down. The `docker-compose` is for legacy breeze        -project name and you can use `breeze down --project-name docker-compose` to stop all containers    +--project-nameName of the docker-compose project to bring down. The `docker-compose` is for legacy breeze        +project name and you can use `breeze down --project-name docker-compose` to stop all containers    belonging to it.                                                                                   (breeze | pre-commit | docker-compose)                                                             [default: breeze]                                                                                  ---docker-hostOptional - docker host to use when running docker commands. When set, the `--builder` option is    +--docker-hostOptional - docker host to use when running docker commands. When set, the `--builder` option is    ignored when building images.                                                                      (TEXT)                                                                                             ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Database ───────────────────────────────────────────────────────────────────────────────────────────────────────────╮ ---backend-bDatabase backend to use. If 'none' is chosen, Breeze will start with an invalid database     +--backend-bDatabase backend to use. If 'none' is chosen, Breeze will start with an invalid database     configuration, meaning there will be no database available, and any attempts to connect to   the Airflow database will fail.                                                              (>sqlite< | mysql | postgres | none)                                                         [default: sqlite]                                                                            ---postgres-version-PVersion of Postgres used.(>12< | 13 | 14 | 15 | 16)[default: 12] ---mysql-version-MVersion of MySQL used.(>8.0< | 8.4)[default: 8.0] ---db-reset-dReset DB when entering the container. +--postgres-version-PVersion of Postgres used.(>12< | 13 | 14 | 15 | 16)[default: 12] +--mysql-version-MVersion of MySQL used.(>8.0< | 8.4)[default: 8.0] +--db-reset-dReset DB when entering the container. ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Build CI image (before entering shell) ─────────────────────────────────────────────────────────────────────────────╮ ---github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow] ---builderBuildx builder used to perform `docker buildx build` commands.(TEXT) +--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow] +--builderBuildx builder used to perform `docker buildx build` commands.(TEXT) [default: autodetect]                                          ---use-uv/--no-use-uvUse uv instead of pip as packaging tool to build the image.[default: use-uv] ---uv-http-timeoutTimeout for requests that UV makes (only used in case of UV builds).(INTEGER RANGE) +--use-uv/--no-use-uvUse uv instead of pip as packaging tool to build the image.[default: use-uv] +--uv-http-timeoutTimeout for requests that UV makes (only used in case of UV builds).(INTEGER RANGE) [default: 300; x>=1]                                                 ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Other options ──────────────────────────────────────────────────────────────────────────────────────────────────────╮ ---forward-credentials-fForward local credentials to container when running. ---max-timeMaximum time that the command should take - if it takes longer, the command will fail. +--forward-credentials-fForward local credentials to container when running. +--max-timeMaximum time that the command should take - if it takes longer, the command will fail. (INTEGER RANGE)                                                                        ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ ---answer-aForce answer to questions.(y | n | q | yes | no | quit) ---dry-run-DIf dry-run is set, commands are only printed, not executed. ---verbose-vPrint verbose information about performed steps. ---help-hShow this message and exit. +--answer-aForce answer to questions.(y | n | q | yes | no | quit) +--dry-run-DIf dry-run is set, commands are only printed, not executed. +--verbose-vPrint verbose information about performed steps. +--help-hShow this message and exit. ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Developer commands ─────────────────────────────────────────────────────────────────────────────────────────────────╮ start-airflow          Enter breeze environment and starts all Airflow components in the tmux session. Compile     diff --git a/dev/breeze/doc/images/output_static-checks.svg b/dev/breeze/doc/images/output_static-checks.svg index de796de4e07b..d8f715c4b856 100644 --- a/dev/breeze/doc/images/output_static-checks.svg +++ b/dev/breeze/doc/images/output_static-checks.svg @@ -1,4 +1,4 @@ - + Run static checks. ╭─ Pre-commit flags ───────────────────────────────────────────────────────────────────────────────────────────────────╮ ---type-tType(s) of the static checks to run.                                              +--type-tType(s) of the static checks to run.                                              (all | bandit | blacken-docs | check-aiobotocore-optional |                       check-airflow-k8s-not-used | check-airflow-provider-compatibility |               check-airflow-providers-bug-report-template | check-apache-license-rat |          @@ -340,68 +343,69 @@ check-incorrect-use-of-LoggingMixin | check-init-decorator-arguments |            check-integrations-list-consistent | check-lazy-logging |                         check-links-to-example-dags-do-not-use-hardcoded-versions | check-merge-conflict  -| check-newsfragments-are-valid | check-no-airflow-deprecation-in-providers |     -check-no-providers-in-core-examples | check-only-new-session-with-provide-session -| check-persist-credentials-disabled-in-github-workflows |                        -check-pre-commit-information-consistent | check-provide-create-sessions-imports | -check-provider-docs-valid | check-provider-yaml-valid |                           -check-providers-init-file-missing | check-providers-subpackages-init-file-exist | -check-pydevd-left-in-code | check-revision-heads-map |                            -check-safe-filter-usage-in-html | check-sql-dependency-common-data-structure |    -check-start-date-not-used-in-defaults | check-system-tests-present |              -check-system-tests-tocs | check-taskinstance-tis-attrs |                          -check-template-context-variable-in-sync | check-tests-in-the-right-folders |      -check-tests-unittest-testcase | check-urlparse-usage-in-code |                    -check-usage-of-re2-over-re | check-xml | codespell | compile-www-assets |         -compile-www-assets-dev | create-missing-init-py-files-tests | debug-statements |  -detect-private-key | doctoc | end-of-file-fixer | fix-encoding-pragma | flynt |   -generate-airflow-diagrams | generate-pypi-readme | identity | insert-license |    -kubeconform | lint-chart-schema | lint-css | lint-dockerfile | lint-helm-chart |  -lint-json-schema | lint-markdown | lint-openapi | mixed-line-ending |             -mypy-airflow | mypy-dev | mypy-docs | mypy-providers | pretty-format-json |       -pylint | python-no-log-warn | replace-bad-characters | rst-backticks | ruff |     -ruff-format | shellcheck | trailing-whitespace | ts-compile-format-lint-www |     -update-black-version | update-breeze-cmd-output |                                 -update-breeze-readme-config-hash | update-build-dependencies |                    -update-chart-dependencies | update-common-sql-api-stubs | update-er-diagram |     -update-extras | update-in-the-wild-to-be-sorted |                                 -update-inlined-dockerfile-scripts | update-installed-providers-to-be-sorted |     -update-installers | update-local-yml-file | update-migration-references |         -update-openapi-spec-tags-to-be-sorted | update-providers-dependencies |           -update-reproducible-source-date-epoch | update-spelling-wordlist-to-be-sorted |   -update-supported-versions | update-vendored-in-k8s-json-schema | update-version | -validate-operators-init | yamllint)                                               ---show-diff-on-failure-sShow diff for files modified by the checks. ---initialize-environmentInitialize environment before running checks. ---max-initialization-attemptsMaximum number of attempts to initialize environment before giving up. -(INTEGER RANGE)                                                        -[default: 3; 1<=x<=10]                                                 -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Selecting files to run the checks on ───────────────────────────────────────────────────────────────────────────────╮ ---file-fList of files to run the checks on.(PATH) ---all-files-aRun checks on all files. ---commit-ref-rRun checks for this commit reference only (can be any git commit-ish reference). Mutually     -exclusive with --last-commit.                                                                 -(TEXT)                                                                                        ---last-commit-cRun checks for all files in last commit. Mutually exclusive with --commit-ref. ---only-my-changes-mRun checks for commits belonging to my PR only: for all commits between merge base to `main`  -branch and HEAD of your branch.                                                               -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Building image before running checks ───────────────────────────────────────────────────────────────────────────────╮ ---skip-image-upgrade-checkSkip checking if the CI image is up to date. ---force-buildForce image build no matter if it is determined as needed. ---image-tagTag of the image which is used to run the image (implies --mount-sources=skip). -(TEXT)                                                                          -[default: latest]                                                               ---github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow] ---builderBuildx builder used to perform `docker buildx build` commands.(TEXT) -[default: autodetect]                                          -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ ---dry-run-DIf dry-run is set, commands are only printed, not executed. ---verbose-vPrint verbose information about performed steps. ---help-hShow this message and exit. -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +| check-min-python-version | check-newsfragments-are-valid |                      +check-no-airflow-deprecation-in-providers | check-no-providers-in-core-examples | +check-only-new-session-with-provide-session |                                     +check-persist-credentials-disabled-in-github-workflows |                          +check-pre-commit-information-consistent | check-provide-create-sessions-imports | +check-provider-docs-valid | check-provider-yaml-valid |                           +check-providers-init-file-missing | check-providers-subpackages-init-file-exist | +check-pydevd-left-in-code | check-revision-heads-map |                            +check-safe-filter-usage-in-html | check-sql-dependency-common-data-structure |    +check-start-date-not-used-in-defaults | check-system-tests-present |              +check-system-tests-tocs | check-taskinstance-tis-attrs |                          +check-template-context-variable-in-sync | check-tests-in-the-right-folders |      +check-tests-unittest-testcase | check-urlparse-usage-in-code |                    +check-usage-of-re2-over-re | check-xml | codespell | compile-www-assets |         +compile-www-assets-dev | create-missing-init-py-files-tests | debug-statements |  +detect-private-key | doctoc | end-of-file-fixer | fix-encoding-pragma | flynt |   +generate-airflow-diagrams | generate-pypi-readme | identity | insert-license |    +kubeconform | lint-chart-schema | lint-css | lint-dockerfile | lint-helm-chart |  +lint-json-schema | lint-markdown | lint-openapi | mixed-line-ending |             +mypy-airflow | mypy-dev | mypy-docs | mypy-providers | pretty-format-json |       +pylint | python-no-log-warn | replace-bad-characters | rst-backticks | ruff |     +ruff-format | shellcheck | trailing-whitespace | ts-compile-format-lint-www |     +update-black-version | update-breeze-cmd-output |                                 +update-breeze-readme-config-hash | update-build-dependencies |                    +update-chart-dependencies | update-common-sql-api-stubs | update-er-diagram |     +update-extras | update-in-the-wild-to-be-sorted |                                 +update-inlined-dockerfile-scripts | update-installed-providers-to-be-sorted |     +update-installers | update-local-yml-file | update-migration-references |         +update-openapi-spec-tags-to-be-sorted | update-providers-dependencies |           +update-reproducible-source-date-epoch | update-spelling-wordlist-to-be-sorted |   +update-supported-versions | update-vendored-in-k8s-json-schema | update-version | +validate-operators-init | yamllint)                                               +--show-diff-on-failure-sShow diff for files modified by the checks. +--initialize-environmentInitialize environment before running checks. +--max-initialization-attemptsMaximum number of attempts to initialize environment before giving up. +(INTEGER RANGE)                                                        +[default: 3; 1<=x<=10]                                                 +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Selecting files to run the checks on ───────────────────────────────────────────────────────────────────────────────╮ +--file-fList of files to run the checks on.(PATH) +--all-files-aRun checks on all files. +--commit-ref-rRun checks for this commit reference only (can be any git commit-ish reference). Mutually     +exclusive with --last-commit.                                                                 +(TEXT)                                                                                        +--last-commit-cRun checks for all files in last commit. Mutually exclusive with --commit-ref. +--only-my-changes-mRun checks for commits belonging to my PR only: for all commits between merge base to `main`  +branch and HEAD of your branch.                                                               +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Building image before running checks ───────────────────────────────────────────────────────────────────────────────╮ +--skip-image-upgrade-checkSkip checking if the CI image is up to date. +--force-buildForce image build no matter if it is determined as needed. +--image-tagTag of the image which is used to run the image (implies --mount-sources=skip). +(TEXT)                                                                          +[default: latest]                                                               +--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow] +--builderBuildx builder used to perform `docker buildx build` commands.(TEXT) +[default: autodetect]                                          +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +--dry-run-DIf dry-run is set, commands are only printed, not executed. +--verbose-vPrint verbose information about performed steps. +--help-hShow this message and exit. +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/dev/breeze/doc/images/output_static-checks.txt b/dev/breeze/doc/images/output_static-checks.txt index 53bb9e5b46ef..d109d39417a1 100644 --- a/dev/breeze/doc/images/output_static-checks.txt +++ b/dev/breeze/doc/images/output_static-checks.txt @@ -1 +1 @@ -97228aa2e9cdff69f86a9c0dfee5f83e +a109ca358651bbfcf3a36d007fd0646a diff --git a/dev/breeze/src/airflow_breeze/pre_commit_ids.py b/dev/breeze/src/airflow_breeze/pre_commit_ids.py index 70200d1ec6a7..b0ceca5c4b2b 100644 --- a/dev/breeze/src/airflow_breeze/pre_commit_ids.py +++ b/dev/breeze/src/airflow_breeze/pre_commit_ids.py @@ -63,6 +63,7 @@ "check-lazy-logging", "check-links-to-example-dags-do-not-use-hardcoded-versions", "check-merge-conflict", + "check-min-python-version", "check-newsfragments-are-valid", "check-no-airflow-deprecation-in-providers", "check-no-providers-in-core-examples", diff --git a/scripts/ci/pre_commit/check_min_python_version.py b/scripts/ci/pre_commit/check_min_python_version.py new file mode 100755 index 000000000000..ac175eac50e0 --- /dev/null +++ b/scripts/ci/pre_commit/check_min_python_version.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.resolve())) + +from common_precommit_utils import console + +# update this version when we switch to a newer version of Python +required_version = tuple(map(int, "3.8".split("."))) +required_version_str = f"{required_version[0]}.{required_version[1]}" +global_version = tuple( + map( + int, + subprocess.run( + [ + "python3", + "-c", + 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")', + ], + capture_output=True, + text=True, + check=True, + ) + .stdout.strip() + .split("."), + ) +) + + +if global_version < required_version: + console.print(f"[red]Python {required_version_str} or higher is required to install pre-commit.\n") + console.print(f"[green]Current version is {global_version}\n") + console.print( + "[bright_yellow]Please follow those steps:[/]\n\n" + f" * make sure that `python3 --version` is at least {required_version_str}\n" + f" * run `pre-commit clean`\n" + f" * run `pre-commit install --install-hooks`\n\n" + ) + console.print( + "There are various ways you can set `python3` to point to a newer version of Python.\n\n" + f"For example run `pyenv global {required_version_str}` if you use pyenv, or\n" + f"you can use venv with python {required_version_str} when you use pre-commit, or\n" + "you can use `update-alternatives` if you use Ubuntu, or\n" + "you can set `PATH` to point to the newer version of Python.\n\n" + ) + sys.exit(1) +else: + console.print(f"Python version is sufficient: {required_version_str}")