diff --git a/.github/workflows/check-providers.yml b/.github/workflows/check-providers.yml index 3faf19b61f53..bd659cc75ea1 100644 --- a/.github/workflows/check-providers.yml +++ b/.github/workflows/check-providers.yml @@ -40,7 +40,7 @@ on: # yamllint disable-line rule:truthy description: "Whether to upgrade to newer dependencies" required: true type: string - affected-providers-list-as-string: + selected-providers-list-as-string: description: "List of affected providers as string" required: false type: string @@ -163,7 +163,7 @@ jobs: run: > breeze release-management prepare-provider-packages --include-not-ready-providers --version-suffix-for-pypi dev0 --package-format sdist - ${{ inputs.affected-providers-list-as-string }} + ${{ inputs.selected-providers-list-as-string }} - name: "Prepare airflow package: sdist" run: > breeze release-management prepare-airflow-package @@ -187,7 +187,7 @@ jobs: --providers-constraints-location /files/constraints-${{env.PYTHON_MAJOR_MINOR_VERSION}}/constraints-source-providers-${{env.PYTHON_MAJOR_MINOR_VERSION}}.txt --run-in-parallel - if: inputs.affected-providers-list-as-string == '' + if: inputs.selected-providers-list-as-string == '' - name: "Install affected provider packages and airflow via sdist files" run: > breeze release-management install-provider-packages @@ -198,7 +198,7 @@ jobs: --providers-constraints-location /files/constraints-${{env.PYTHON_MAJOR_MINOR_VERSION}}/constraints-source-providers-${{env.PYTHON_MAJOR_MINOR_VERSION}}.txt --run-in-parallel - if: inputs.affected-providers-list-as-string != '' + if: inputs.selected-providers-list-as-string != '' providers-compatibility-checks: timeout-minutes: 80 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9f61e356b3b2..ac14a81634d3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,8 +58,8 @@ jobs: image-tag: ${{ github.event.pull_request.head.sha || github.sha }} docker-cache: ${{ steps.selective-checks.outputs.docker-cache }} disable-airflow-repo-cache: ${{ steps.selective-checks.outputs.disable-airflow-repo-cache }} - affected-providers-list-as-string: >- - ${{ steps.selective-checks.outputs.affected-providers-list-as-string }} + selected-providers-list-as-string: >- + ${{ steps.selective-checks.outputs.selected-providers-list-as-string }} upgrade-to-newer-dependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }} python-versions: ${{ steps.selective-checks.outputs.python-versions }} python-versions-list-as-string: ${{ steps.selective-checks.outputs.python-versions-list-as-string }} @@ -79,12 +79,12 @@ jobs: default-kind-version: ${{ steps.selective-checks.outputs.default-kind-version }} force-pip: ${{ steps.selective-checks.outputs.force-pip }} full-tests-needed: ${{ steps.selective-checks.outputs.full-tests-needed }} - parallel-test-types-list-as-string: >- - ${{ steps.selective-checks.outputs.parallel-test-types-list-as-string }} + core-test-types-list-as-string: >- + ${{ steps.selective-checks.outputs.core-test-types-list-as-string }} providers-test-types-list-as-string: >- ${{ steps.selective-checks.outputs.providers-test-types-list-as-string }} - separate-test-types-list-as-string: >- - ${{ steps.selective-checks.outputs.separate-test-types-list-as-string }} + individual-providers-test-types-list-as-string: >- + ${{ steps.selective-checks.outputs.individual-providers-test-types-list-as-string }} include-success-outputs: ${{ steps.selective-checks.outputs.include-success-outputs }} postgres-exclude: ${{ steps.selective-checks.outputs.postgres-exclude }} mysql-exclude: ${{ steps.selective-checks.outputs.mysql-exclude }} @@ -336,7 +336,7 @@ jobs: canary-run: ${{ needs.build-info.outputs.canary-run }} default-python-version: ${{ needs.build-info.outputs.default-python-version }} upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} - affected-providers-list-as-string: ${{ needs.build-info.outputs.affected-providers-list-as-string }} + selected-providers-list-as-string: ${{ needs.build-info.outputs.selected-providers-list-as-string }} providers-compatibility-checks: ${{ needs.build-info.outputs.providers-compatibility-checks }} skip-provider-tests: ${{ needs.build-info.outputs.skip-provider-tests }} python-versions: ${{ needs.build-info.outputs.python-versions }} @@ -379,7 +379,7 @@ jobs: backend-versions: ${{ needs.build-info.outputs.postgres-versions }} excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} excludes: ${{ needs.build-info.outputs.postgres-exclude }} - parallel-test-types-list-as-string: ${{ needs.build-info.outputs.parallel-test-types-list-as-string }} + core-test-types-list-as-string: ${{ needs.build-info.outputs.core-test-types-list-as-string }} include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} run-migration-tests: "true" run-coverage: ${{ needs.build-info.outputs.run-coverage }} @@ -404,7 +404,7 @@ jobs: backend-versions: ${{ needs.build-info.outputs.mysql-versions }} excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} excludes: ${{ needs.build-info.outputs.mysql-exclude }} - parallel-test-types-list-as-string: ${{ needs.build-info.outputs.parallel-test-types-list-as-string }} + core-test-types-list-as-string: ${{ needs.build-info.outputs.core-test-types-list-as-string }} include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} run-coverage: ${{ needs.build-info.outputs.run-coverage }} run-migration-tests: "true" @@ -431,7 +431,7 @@ jobs: backend-versions: "['']" excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} excludes: ${{ needs.build-info.outputs.sqlite-exclude }} - parallel-test-types-list-as-string: ${{ needs.build-info.outputs.parallel-test-types-list-as-string }} + core-test-types-list-as-string: ${{ needs.build-info.outputs.core-test-types-list-as-string }} include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} run-coverage: ${{ needs.build-info.outputs.run-coverage }} run-migration-tests: "true" @@ -458,7 +458,7 @@ jobs: backend-versions: "['']" excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} excludes: ${{ needs.build-info.outputs.sqlite-exclude }} - parallel-test-types-list-as-string: ${{ needs.build-info.outputs.parallel-test-types-list-as-string }} + core-test-types-list-as-string: ${{ needs.build-info.outputs.core-test-types-list-as-string }} include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} run-coverage: ${{ needs.build-info.outputs.run-coverage }} debug-resources: ${{ needs.build-info.outputs.debug-resources }} @@ -480,7 +480,7 @@ jobs: with: runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} image-tag: ${{ needs.build-info.outputs.image-tag }} - parallel-test-types-list-as-string: ${{ needs.build-info.outputs.parallel-test-types-list-as-string }} + core-test-types-list-as-string: ${{ needs.build-info.outputs.core-test-types-list-as-string }} run-coverage: ${{ needs.build-info.outputs.run-coverage }} default-python-version: ${{ needs.build-info.outputs.default-python-version }} python-versions: ${{ needs.build-info.outputs.python-versions }} @@ -531,7 +531,8 @@ jobs: backend-versions: "['${{ needs.build-info.outputs.default-postgres-version }}']" excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} excludes: "[]" - parallel-test-types-list-as-string: ${{ needs.build-info.outputs.separate-test-types-list-as-string }} + # yamllint disable rule:line-length + core-test-types-list-as-string: ${{ needs.build-info.outputs.individual-providers-test-types-list-as-string }} include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} run-coverage: ${{ needs.build-info.outputs.run-coverage }} debug-resources: ${{ needs.build-info.outputs.debug-resources }} diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml index eb3e1a90707f..9e80abfbde69 100644 --- a/.github/workflows/run-unit-tests.yml +++ b/.github/workflows/run-unit-tests.yml @@ -28,6 +28,10 @@ on: # yamllint disable-line rule:truthy description: "The backend to run the tests on" required: true type: string + test-group: + description: "The group of the test to run: ('Core', 'Providers')" + required: true + type: string test-scope: description: "The scope of the test to run: ('DB', 'Non-DB', 'All', 'ARM collection')" required: true @@ -61,8 +65,8 @@ on: # yamllint disable-line rule:truthy description: "Excluded combos (stringified JSON array of python-version/backend-version dicts)" required: true type: string - parallel-test-types-list-as-string: - description: "The list of parallel test types to run separated by spaces" + test-types-list-as-string: + description: "The list of test types to run separated by spaces" required: true type: string run-migration-tests: @@ -119,7 +123,7 @@ jobs: name: "\ ${{ inputs.test-scope }}:\ ${{ inputs.test-name }}${{ inputs.test-name-separator }}${{ matrix.backend-version }}:\ - ${{matrix.python-version}}: ${{ inputs.parallel-test-types-list-as-string }}" + ${{matrix.python-version}}: ${{ inputs.core-test-types-list-as-string }}" runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }} strategy: fail-fast: false @@ -147,7 +151,7 @@ jobs: # yamllint disable rule:line-length JOB_ID: "${{ inputs.test-scope }}-${{ inputs.test-name }}-${{inputs.backend}}-${{ matrix.backend-version }}-${{ matrix.python-version }}" MOUNT_SOURCES: "skip" - PARALLEL_TEST_TYPES: "${{ inputs.parallel-test-types-list-as-string }}" + PARALLEL_TEST_TYPES: "${{ inputs.core-test-types-list-as-string }}" PYTHON_MAJOR_MINOR_VERSION: "${{ matrix.python-version }}" UPGRADE_BOTO: "${{ inputs.upgrade-boto }}" AIRFLOW_MONITOR_DELAY_TIME_IN_SECONDS: "${{inputs.monitor-delay-time-in-seconds}}" @@ -166,32 +170,13 @@ jobs: uses: ./.github/actions/prepare_breeze_and_image - name: > Migration Tests: - ${{ matrix.python-version }}:${{ inputs.parallel-test-types-list-as-string }} + ${{ matrix.python-version }}:${{ inputs.core-test-types-list-as-string }} uses: ./.github/actions/migration_tests if: inputs.run-migration-tests == 'true' - name: > - ${{ inputs.test-scope }} Tests ${{ inputs.test-name }} ${{ matrix.backend-version }} - Py${{ matrix.python-version }}:${{ inputs.parallel-test-types-list-as-string}} - run: | - if [[ "${{ inputs.test-scope }}" == "DB" ]]; then - breeze testing db-tests \ - --parallel-test-types "${{ inputs.parallel-test-types-list-as-string }}" - elif [[ "${{ inputs.test-scope }}" == "Non-DB" ]]; then - breeze testing non-db-tests \ - --parallel-test-types "${{ inputs.parallel-test-types-list-as-string }}" - elif [[ "${{ inputs.test-scope }}" == "All" ]]; then - breeze testing tests --run-in-parallel \ - --parallel-test-types "${{ inputs.parallel-test-types-list-as-string }}" - elif [[ "${{ inputs.test-scope }}" == "Quarantined" ]]; then - breeze testing tests --test-type "All-Quarantined" || true - elif [[ "${{ inputs.test-scope }}" == "ARM collection" ]]; then - breeze testing tests --collect-only --remove-arm-packages - elif [[ "${{ inputs.test-scope }}" == "System" ]]; then - breeze testing tests tests/system/example_empty.py --system core - else - echo "Unknown test scope: ${{ inputs.test-scope }}" - exit 1 - fi + ${{ inputs.test-group}}:${{ inputs.test-scope }} Tests ${{ inputs.test-name }} ${{ matrix.backend-version }} + Py${{ matrix.python-version }}:${{ inputs.test-types-list-as-string}} + run: ./scripts/ci/testing/run_unit_tests.sh - name: "Post Tests success" uses: ./.github/actions/post_tests_success with: diff --git a/.github/workflows/special-tests.yml b/.github/workflows/special-tests.yml index 78b4d928f7a9..b71ac76b6792 100644 --- a/.github/workflows/special-tests.yml +++ b/.github/workflows/special-tests.yml @@ -28,7 +28,7 @@ on: # yamllint disable-line rule:truthy description: "Tag to set for the image" required: true type: string - parallel-test-types-list-as-string: + core-test-types-list-as-string: description: "The list of parallel test types to run separated by spaces" required: true type: string @@ -83,7 +83,7 @@ jobs: backend-versions: "['${{ inputs.default-postgres-version }}']" excluded-providers-as-string: ${{ inputs.excluded-providers-as-string }} excludes: "[]" - parallel-test-types-list-as-string: ${{ inputs.parallel-test-types-list-as-string }} + core-test-types-list-as-string: ${{ inputs.core-test-types-list-as-string }} include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} run-coverage: ${{ inputs.run-coverage }} debug-resources: ${{ inputs.debug-resources }} @@ -106,7 +106,7 @@ jobs: backend-versions: "['${{ inputs.default-postgres-version }}']" excluded-providers-as-string: ${{ inputs.excluded-providers-as-string }} excludes: "[]" - parallel-test-types-list-as-string: ${{ inputs.parallel-test-types-list-as-string }} + core-test-types-list-as-string: ${{ inputs.core-test-types-list-as-string }} include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} run-coverage: ${{ inputs.run-coverage }} debug-resources: ${{ inputs.debug-resources }} @@ -129,7 +129,7 @@ jobs: backend-versions: "['${{ inputs.default-postgres-version }}']" excluded-providers-as-string: ${{ inputs.excluded-providers-as-string }} excludes: "[]" - parallel-test-types-list-as-string: ${{ inputs.parallel-test-types-list-as-string }} + core-test-types-list-as-string: ${{ inputs.core-test-types-list-as-string }} include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} run-coverage: ${{ inputs.run-coverage }} debug-resources: ${{ inputs.debug-resources }} @@ -152,7 +152,7 @@ jobs: backend-versions: "['${{ inputs.default-postgres-version }}']" excluded-providers-as-string: ${{ inputs.excluded-providers-as-string }} excludes: "[]" - parallel-test-types-list-as-string: ${{ inputs.parallel-test-types-list-as-string }} + core-test-types-list-as-string: ${{ inputs.core-test-types-list-as-string }} include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} run-coverage: ${{ inputs.run-coverage }} debug-resources: ${{ inputs.debug-resources }} @@ -174,7 +174,7 @@ jobs: backend-versions: "['${{ inputs.default-postgres-version }}']" excluded-providers-as-string: ${{ inputs.excluded-providers-as-string }} excludes: "[]" - parallel-test-types-list-as-string: ${{ inputs.parallel-test-types-list-as-string }} + core-test-types-list-as-string: ${{ inputs.core-test-types-list-as-string }} include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} run-coverage: ${{ inputs.run-coverage }} debug-resources: ${{ inputs.debug-resources }} @@ -196,7 +196,7 @@ jobs: backend-versions: "['${{ inputs.default-postgres-version }}']" excluded-providers-as-string: ${{ inputs.excluded-providers-as-string }} excludes: "[]" - parallel-test-types-list-as-string: ${{ inputs.parallel-test-types-list-as-string }} + core-test-types-list-as-string: ${{ inputs.core-test-types-list-as-string }} include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} run-coverage: ${{ inputs.run-coverage }} debug-resources: ${{ inputs.debug-resources }} @@ -218,7 +218,7 @@ jobs: backend-versions: "['${{ inputs.default-postgres-version }}']" excluded-providers-as-string: ${{ inputs.excluded-providers-as-string }} excludes: "[]" - parallel-test-types-list-as-string: ${{ inputs.parallel-test-types-list-as-string }} + core-test-types-list-as-string: ${{ inputs.core-test-types-list-as-string }} include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} run-coverage: ${{ inputs.run-coverage }} debug-resources: ${{ inputs.debug-resources }} diff --git a/Dockerfile.ci b/Dockerfile.ci index 943270aec693..8c6c1792fce4 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1068,12 +1068,6 @@ function check_run_tests() { python "${IN_CONTAINER_DIR}/remove_arm_packages.py" fi - if [[ ${TEST_TYPE} == "PlainAsserts" ]]; then - # Plain asserts should be converted to env variable to make sure they are taken into account - # otherwise they will not be effective during test collection when plain assert is breaking collection - export PYTEST_PLAIN_ASSERTS="true" - fi - if [[ ${DATABASE_ISOLATION=} == "true" ]]; then echo "${COLOR_BLUE}Starting internal API server:${COLOR_RESET}" # We need to start the internal API server before running tests diff --git a/contributing-docs/testing/unit_tests.rst b/contributing-docs/testing/unit_tests.rst index 88ca07c6e0e6..6b7b7a611aaf 100644 --- a/contributing-docs/testing/unit_tests.rst +++ b/contributing-docs/testing/unit_tests.rst @@ -104,18 +104,8 @@ Those test types are defined: ``virtualenv_operator`` and ``external_python_operator`` test markers that the tests are marked with. * ``WWW`` - Tests for the Airflow webserver (www folder) * ``Providers`` - Tests for all Providers of Airflow (providers folder) -* ``PlainAsserts`` - tests that require disabling ``assert-rewrite`` feature of Pytest (usually because - a buggy/complex implementation of an imported library) (``plain_asserts`` marker) * ``Other`` - all other tests remaining after the above tests are selected -There are also Virtualenv/ExternalPython operator test types that are excluded from ``Operators`` test type -and run as separate test types. Those are : - -* ``PythonVenv`` - tests for PythonVirtualenvOperator - selected directly as TestPythonVirtualenvOperator -* ``BranchPythonVenv`` - tests for BranchPythonVirtualenvOperator - selected directly as TestBranchPythonVirtualenvOperator -* ``ExternalPython`` - tests for ExternalPythonOperator - selected directly as TestExternalPythonOperator -* ``BranchExternalPython`` - tests for BranchExternalPythonOperator - selected directly as TestBranchExternalPythonOperator - We have also tests that run "all" tests (so they do not look at the folder, but at the ``pytest`` markers the tests are marked with to run with some filters applied. diff --git a/dev/breeze/doc/05_test_commands.rst b/dev/breeze/doc/05_test_commands.rst index e210017088ac..42e29d91186d 100644 --- a/dev/breeze/doc/05_test_commands.rst +++ b/dev/breeze/doc/05_test_commands.rst @@ -75,34 +75,28 @@ This applies to all kind of tests - all our tests can be run using pytest. Running unit tests with ``breeze testing`` commands ................................................... -An option you have is that you can also run tests via built-in ``breeze testing tests`` command - which -is a "swiss-army-knife" of unit testing with Breeze. This command has a lot of parameters and is very -flexible thus might be a bit overwhelming. +An option you have is that you can also run tests via built-in ``breeze testing *tests*`` commands - which +is a "swiss-army-knife" of unit testing with Breeze. You can run all groups of tests with that Airflow +supports with one of the commands below. -In most cases if you want to run tess you want to use dedicated ``breeze testing db-tests`` -or ``breeze testing non-db-tests`` commands that automatically run groups of tests that allow you to choose -subset of tests to run (with ``--parallel-test-types`` flag) +Using ``breeze testing core-tests`` command +........................................... -Using ``breeze testing tests`` command -...................................... +The ``breeze testing core-tests`` command is that you can run for all or specify sub-set of the tests +for Core. -The ``breeze testing tests`` command is that you can easily specify sub-set of the tests -- including -selecting specific Providers tests to run. - -For example this will only run provider tests for airbyte and http providers: +For example this will run all core tests : .. code-block:: bash - breeze testing tests --test-type "Providers[airbyte,http]" - -You can also exclude tests for some providers from being run when whole "Providers" test type is run. + breeze testing core-tests -For example this will run tests for all providers except amazon and google provider tests: +For example this will only run "Other" tests : .. code-block:: bash - breeze testing tests --test-type "Providers[-amazon,google]" + breeze testing core-tests --test-type "Other" You can also run parallel tests with ``--run-in-parallel`` flag - by default it will run all tests types in parallel, but you can specify the test type that you want to run with space separated list of test @@ -114,25 +108,17 @@ For example this will run API and WWW tests in parallel: breeze testing tests --parallel-test-types "API WWW" --run-in-parallel -There are few special types of tests that you can run: - -* ``All`` - all tests are run in single pytest run. -* ``All-Postgres`` - runs all tests that require Postgres database -* ``All-MySQL`` - runs all tests that require MySQL database -* ``All-Quarantine`` - runs all tests that are in quarantine (marked with ``@pytest.mark.quarantined`` - decorator) - Here is the detailed set of options for the ``breeze testing tests`` command. -.. image:: ./images/output_testing_tests.svg - :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/images/output_testing_tests.svg +.. image:: ./images/output_testing_core-tests.svg + :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/images/output_testing_core-tests.svg :width: 100% - :alt: Breeze testing tests + :alt: Breeze testing core-tests -Using ``breeze testing db-tests`` command -......................................... +Using ``breeze testing core-db-tests`` command +.............................................. -The ``breeze testing db-tests`` command is simplified version of the ``breeze testing tests`` command +The ``breeze testing core-db-tests`` command is simplified version of the ``breeze testing core-tests`` command that only allows you to run tests that are not bound to a database - in parallel utilising all your CPUS. The DB-bound tests are the ones that require a database to be started and configured separately for each test type run and they are run in parallel containers/parallel docker compose projects to @@ -147,33 +133,33 @@ Run all DB tests: .. code-block:: bash - breeze testing db-tests + breeze testing core-db-tests Only run DB tests from "API CLI WWW" test types: .. code-block:: bash - breeze testing db-tests --parallel-test-types "API CLI WWW" + breeze testing core-db-tests --parallel-test-types "API CLI WWW" Run all DB tests excluding those in CLI and WWW test types: .. code-block:: bash - breeze testing db-tests --excluded-parallel-test-types "CLI WWW" + breeze testing core-db-tests --excluded-parallel-test-types "CLI WWW" -Here is the detailed set of options for the ``breeze testing db-tests`` command. +Here is the detailed set of options for the ``breeze testing core-db-tests`` command. -.. image:: ./images/output_testing_db-tests.svg - :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/images/output_testing_db-tests.svg +.. image:: ./images/output_testing_core-db-tests.svg + :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/images/output_testing_core-db-tests.svg :width: 100% - :alt: Breeze testing db-tests + :alt: Breeze testing core-db-tests -Using ``breeze testing non-db-tests`` command -......................................... +Using ``breeze testing core-non-db-tests`` command +.................................................. -The ``breeze testing non-db-tests`` command is simplified version of the ``breeze testing tests`` command -that only allows you to run tests that are not bound to a database - in parallel utilising all your CPUS. +The ``breeze testing core-non-db-tests`` command is simplified version of the ``breeze testing core-tests`` +command that only allows you to run tests that are not bound to a database - in parallel utilising all your CPUS. The non-DB-bound tests are the ones that do not expect a database to be started and configured and we can utilise multiple CPUs your machine has via ``pytest-xdist`` plugin - thus allowing you to quickly run few groups of tests in parallel using single container rather than many of them as it is the case for @@ -187,34 +173,136 @@ Run all non-DB tests: .. code-block:: bash - breeze testing non-db-tests + breeze testing core-non-db-tests Only run non-DB tests from "API CLI WWW" test types: .. code-block:: bash - breeze testing non-db-tests --parallel-test-types "API CLI WWW" + breeze testing core-non-db-tests --parallel-test-types "API CLI WWW" Run all non-DB tests excluding those in CLI and WWW test types: .. code-block:: bash - breeze testing non-db-tests --excluded-parallel-test-types "CLI WWW" + breeze testing core-non-db-tests --excluded-parallel-test-types "CLI WWW" + +Here is the detailed set of options for the ``breeze testing core-non-db-tests`` command. + +.. image:: ./images/output_testing_core-non-db-tests.svg + :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/images/output_testing_core-non-db-tests.svg + :width: 100% + :alt: Breeze testing core-non-db-tests + +Using ``breeze testing providers-tests`` command +................................................ + +The ``breeze testing providers-tests`` command is that you can run for all or specify sub-set of the tests +for Providers. + +For example this will run all provider tests tests : + +.. code-block:: bash + + breeze testing providers-tests + +This will only run "amazon" and "google" provider tests : + +.. code-block:: bash + + breeze testing providers-tests --test-type "Providers[amazon,google]" + +You can also run "all but" provider tests - this will run all providers tests except amazon and google : + +.. code-block:: bash + + breeze testing providers-tests --test-type "Providers[-amazon,google]" + +You can also run parallel tests with ``--run-in-parallel`` flag - by default it will run all tests types +in parallel, but you can specify the test type that you want to run with space separated list of test +types passed to ``--parallel-test-types`` flag. + +For example this will run API and WWW tests in parallel: + +.. code-block:: bash + + breeze testing tests --parallel-test-types "Providers[amazon] Providers[google]" --run-in-parallel + +Here is the detailed set of options for the ``breeze testing providers-test`` command. + +.. image:: ./images/output_testing_providers-tests.svg + :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/images/output_testing_providers-tests.svg + :width: 100% + :alt: Breeze testing providers-tests + +Using ``breeze testing providers-db-tests`` command +................................................... + +The ``breeze testing providers-db-tests`` command is simplified version of the ``breeze testing providers-tests`` command +that only allows you to run tests that are not bound to a database - in parallel utilising all your CPUS. +The DB-bound tests are the ones that require a database to be started and configured separately for +each test type run and they are run in parallel containers/parallel docker compose projects to +utilise multiple CPUs your machine has - thus allowing you to quickly run few groups of tests in parallel. +This command is used in CI to run DB tests. + +By default this command will run complete set of test types we have, thus allowing you to see result +of all DB tests we have but you can choose a subset of test types to run by ``--parallel-test-types`` +flag or exclude some test types by specifying ``--excluded-parallel-test-types`` flag. + +Run all DB tests: + +.. code-block:: bash + + breeze testing providers-db-tests + +You can also specify subset of tests similar to the ``breeze testing providers-tests`` command. + +Here is the detailed set of options for the ``breeze testing providers-db-tests`` command. + +.. image:: ./images/output_testing_providers-db-tests.svg + :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/images/output_testing_providers-db-tests.svg + :width: 100% + :alt: Breeze testing providers-db-tests + + +Using ``breeze testing providers-non-db-tests`` command +....................................................... + +The ``breeze testing providers-non-db-tests`` command is simplified version of the ``breeze testing providers-tests`` +command that only allows you to run tests that are not bound to a database - in parallel utilising all your CPUS. +The non-DB-bound tests are the ones that do not expect a database to be started and configured and we can +utilise multiple CPUs your machine has via ``pytest-xdist`` plugin - thus allowing you to quickly +run few groups of tests in parallel using single container rather than many of them as it is the case for +DB-bound tests. This command is used in CI to run Non-DB tests. + +By default this command will run complete set of test types we have, thus allowing you to see result +of all DB tests we have but you can choose a subset of test types to run by ``--parallel-test-types`` +flag or exclude some test types by specifying ``--excluded-parallel-test-types`` flag. + +Run all non-DB tests: + +.. code-block:: bash + + breeze testing providers-non-db-tests + +Only run non-DB tests from "API CLI WWW" test types: + +You can also specify subset of tests similar to the ``breeze testing providers-tests`` command. -Here is the detailed set of options for the ``breeze testing non-db-tests`` command. +Here is the detailed set of options for the ``breeze testing providers-non-db-tests`` command. -.. image:: ./images/output_testing_non-db-tests.svg - :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/images/output_testing_non-db-tests.svg +.. image:: ./images/output_testing_providers-non-db-tests.svg + :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/images/output_testing_providers-non-db-tests.svg :width: 100% - :alt: Breeze testing non-db-tests + :alt: Breeze testing providers-non-db-tests Using ``breeze testing task-sdk-tests`` command -............................................ +............................................... -The ``breeze testing task-sdk-tests`` command is simplified version of the ``breeze testing tests`` command -that allows you to run tests for Task SDK without initializing database. The Task SDK should not need -database to be started so this acts as a good check to see if the Task SDK tests are working properly. +The ``breeze testing task-sdk-tests`` command is allows you to run tests for Task SDK without +initializing database. The Task SDK should not need database to be started so this acts as a +good check to see if the Task SDK tests are working properly. Run all Task SDK tests: @@ -230,26 +318,90 @@ Here is the detailed set of options for the ``breeze testing task-sdk-tests`` co :alt: Breeze testing task-sdk-tests -Running integration tests -......................... +Running integration core tests +............................... -You can also run integration tests via built-in ``breeze testing integration-tests`` command. Some of our -tests require additional integrations to be started in docker-compose. The integration tests command will -run the expected integration and tests that need that integration. +You can also run integration core tests via built-in ``breeze testing integration-core-tests`` command. +Some of our core tests require additional integrations to be started in docker-compose. +The integration tests command will run the expected integration and tests that need that integration. For example this will only run kerberos tests: .. code-block:: bash - breeze testing integration-tests --integration kerberos + breeze testing integration-core-tests --integration kerberos + +Here is the detailed set of options for the ``breeze testing integration-core-tests`` command. + +.. image:: ./images/output_testing_integration-core-tests.svg + :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/images/output_testing_integration-core-tests.svg + :width: 100% + :alt: Breeze testing integration-core-tests + +Running integration providers tests +................................... +You can also run integration core tests via built-in ``breeze testing integration-providers-tests`` command. +Some of our core tests require additional integrations to be started in docker-compose. +The integration tests command will run the expected integration and tests that need that integration. -Here is the detailed set of options for the ``breeze testing integration-tests`` command. +For example this will only run kerberos tests: -.. image:: ./images/output_testing_integration-tests.svg - :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/images/output_testing_integration_tests.svg +.. code-block:: bash + + breeze testing integration-providers-tests --integration kerberos + +Here is the detailed set of options for the ``breeze testing integration-providers-tests`` command. + +.. image:: ./images/output_testing_integration-providers-tests.svg + :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/images/output_testing_integration-providers-tests.svg + :width: 100% + :alt: Breeze testing integration-providers-tests + + +Running system core tests +......................... + +You can also run system core tests via built-in ``breeze testing system-core-tests`` command. +Some of our core system tests runs against external systems and we can run them providing that +credentials are configured to connect to those systems. Usually you should run only one or +set of related tests this way. + +For example this will only run example_external_task_child_deferrable tests: + +.. code-block:: bash + + breeze testing system-core-tests tests/system/core/example_external_task_child_deferrable.py + +Here is the detailed set of options for the ``breeze testing system-core-tests`` command. + +.. image:: ./images/output_testing_system-core-tests.svg + :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/images/output_testing_system-core-tests.svg :width: 100% - :alt: Breeze testing integration-tests + :alt: Breeze testing system-core-tests + + +Running system providers tests +.............................. + +You can also run system providers tests via built-in ``breeze testing system-providers-tests`` command. +Some of our providers system tests runs against external systems and we can run them providing that +credentials are configured to connect to those systems. Usually you should run only one or +set of related tests this way. + +For example this will only run providers/tests/system/amazon/aws/example_appflow.py tests: + +.. code-block:: bash + + breeze testing system-providers-tests providers/tests/system/amazon/aws/example_appflow.py + +Here is the detailed set of options for the ``breeze testing system-providers-tests`` command. + +.. image:: ./images/output_testing_system-providers-tests.svg + :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/images/output_testing_system-providers-tests.svg + :width: 100% + :alt: Breeze testing system-providers-tests + Running Helm unit tests diff --git a/dev/breeze/doc/ci/04_selective_checks.md b/dev/breeze/doc/ci/04_selective_checks.md index b077bd1a70a1..e5b86e895cfc 100644 --- a/dev/breeze/doc/ci/04_selective_checks.md +++ b/dev/breeze/doc/ci/04_selective_checks.md @@ -166,74 +166,76 @@ separated by spaces. This is to accommodate for the wau how outputs of this kind Github Actions to pass the list of parameters to a command to execute -| Output | Meaning of the output | Example value | List as string | -|----------------------------------------|------------------------------------------------------------------------------------------------------|-------------------------------------------|----------------| -| affected-providers-list-as-string | List of providers affected when they are selectively affected. | airbyte http | * | -| all-python-versions | List of all python versions there are available in the form of JSON array | ['3.9', '3.10'] | | -| all-python-versions-list-as-string | List of all python versions there are available in the form of space separated string | 3.9 3.10 | * | -| all-versions | If set to true, then all python, k8s, DB versions are used for tests. | false | | -| basic-checks-only | Whether to run all static checks ("false") or only basic set of static checks ("true") | false | | -| build_system_changed_in_pyproject_toml | When builds system dependencies changed in pyproject.toml changed in the PR. | false | | -| chicken-egg-providers | List of providers that should be considered as "chicken-egg" - expecting development Airflow version | | | -| ci-image-build | Whether CI image build is needed | true | | -| debug-resources | Whether resources usage should be printed during parallel job execution ("true"/ "false") | false | | -| default-branch | Which branch is default for the build ("main" for main branch, "v2-4-test" for 2.4 line etc.) | main | | -| default-constraints-branch | Which branch is default for the build ("constraints-main" for main branch, "constraints-2-4" etc.) | constraints-main | | -| default-helm-version | Which Helm version to use as default | v3.9.4 | | -| default-kind-version | Which Kind version to use as default | v0.16.0 | | -| default-kubernetes-version | Which Kubernetes version to use as default | v1.25.2 | | -| default-mysql-version | Which MySQL version to use as default | 5.7 | | -| default-postgres-version | Which Postgres version to use as default | 10 | | -| default-python-version | Which Python version to use as default | 3.9 | | -| docker-cache | Which cache should be used for images ("registry", "local" , "disabled") | registry | | -| docs-build | Whether to build documentation ("true"/"false") | true | | -| docs-list-as-string | What filter to apply to docs building - based on which documentation packages should be built | apache-airflow helm-chart google | | -| full-tests-needed | Whether this build runs complete set of tests or only subset (for faster PR builds) [1] | false | | -| generated-dependencies-changed | Whether generated dependencies have changed ("true"/"false") | false | | -| hatch-build-changed | When hatch build.py changed in the PR. | false | | -| helm-version | Which Helm version to use for tests | v3.9.4 | | -| is-airflow-runner | Whether runner used is an airflow or infrastructure runner (true if airflow/false if infrastructure) | false | | -| is-amd-runner | Whether runner used is an AMD one | true | | -| is-arm-runner | Whether runner used is an ARM one | false | | -| is-committer-build | Whether the build is triggered by a committer | false | | -| is-k8s-runner | Whether the build runs on our k8s infrastructure | false | | -| is-self-hosted-runner | Whether the runner is self-hosted | false | | -| is-vm-runner | Whether the runner uses VM to run | true | | -| kind-version | Which Kind version to use for tests | v0.16.0 | | -| kubernetes-combos-list-as-string | All combinations of Python version and Kubernetes version to use for tests as space-separated string | 3.9-v1.25.2 3.9-v1.26.4 | * | -| kubernetes-versions | All Kubernetes versions to use for tests as JSON array | ['v1.25.2'] | | -| kubernetes-versions-list-as-string | All Kubernetes versions to use for tests as space-separated string | v1.25.2 | * | -| mypy-checks | List of folders to be considered for mypy | [] | | -| mysql-exclude | Which versions of MySQL to exclude for tests as JSON array | [] | | -| mysql-versions | Which versions of MySQL to use for tests as JSON array | ['5.7'] | | -| needs-api-codegen | Whether "api-codegen" are needed to run ("true"/"false") | true | | -| needs-api-tests | Whether "api-tests" are needed to run ("true"/"false") | true | | -| needs-helm-tests | Whether Helm tests are needed to run ("true"/"false") | true | | -| needs-javascript-scans | Whether javascript CodeQL scans should be run ("true"/"false") | true | | -| needs-mypy | Whether mypy check is supposed to run in this build | true | | -| needs-python-scans | Whether Python CodeQL scans should be run ("true"/"false") | true | | -| parallel-test-types-list-as-string | Which test types should be run for unit tests | API Always Providers Providers\[-google\] | * | -| postgres-exclude | Which versions of Postgres to exclude for tests as JSON array | [] | | -| postgres-versions | Which versions of Postgres to use for tests as JSON array | ['10'] | | -| prod-image-build | Whether PROD image build is needed | true | | -| prod-image-build | Whether PROD image build is needed | true | | -| providers-compatibility-checks | List of dicts: (python_version, airflow_version, removed_providers) for compatibility checks | [] | | -| pyproject-toml-changed | When pyproject.toml changed in the PR. | false | | -| python-versions | List of python versions to use for that build | ['3.9'] | * | -| python-versions-list-as-string | Which versions of MySQL to use for tests as space-separated string | 3.9 | * | -| run-amazon-tests | Whether Amazon tests should be run ("true"/"false") | true | | -| run-kubernetes-tests | Whether Kubernetes tests should be run ("true"/"false") | true | | -| run-tests | Whether unit tests should be run ("true"/"false") | true | | -| run-ui-tests | Whether WWW tests should be run ("true"/"false") | true | | -| run-www-tests | Whether WWW tests should be run ("true"/"false") | true | | -| runs-on-as-json-default | List of labels assigned for runners for that build for default runs for that build (as string) | ["ubuntu-22.04"] | | -| runs-on-as-json-self-hosted | List of labels assigned for runners for that build for self hosted runners | ["self-hosted", "Linux", "X64"] | | -| runs-on-as-json-public | List of labels assigned for runners for that build for public runners | ["ubuntu-22.04"] | | -| skip-pre-commits | Which pre-commits should be skipped during the static-checks run | check-provider-yaml-valid,flynt,identity | | -| skip-provider-tests | When provider tests should be skipped (on non-main branch or when no provider changes detected) | true | | -| sqlite-exclude | Which versions of Sqlite to exclude for tests as JSON array | [] | | -| testable-integrations | List of integrations that are testable in the build as JSON array | ['mongo', 'kafka', 'mssql'] | | -| upgrade-to-newer-dependencies | Whether the image build should attempt to upgrade all dependencies (true/false or commit hash) | false | | +| Output | Meaning of the output | Example value | List as string | +|------------------------------------------------|------------------------------------------------------------------------------------------------------|-----------------------------------------|----------------| +| selected-providers-list-as-string | List of providers affected when they are selectively affected. | airbyte http | * | +| all-python-versions | List of all python versions there are available in the form of JSON array | \['3.9', '3.10'\] | | +| all-python-versions-list-as-string | List of all python versions there are available in the form of space separated string | 3.9 3.10 | * | +| all-versions | If set to true, then all python, k8s, DB versions are used for tests. | false | | +| basic-checks-only | Whether to run all static checks ("false") or only basic set of static checks ("true") | false | | +| build_system_changed_in_pyproject_toml | When builds system dependencies changed in pyproject.toml changed in the PR. | false | | +| chicken-egg-providers | List of providers that should be considered as "chicken-egg" - expecting development Airflow version | | | +| ci-image-build | Whether CI image build is needed | true | | +| core-test-types-list-as-string | Which test types should be run for unit tests for core | API Always Providers | * | +| debug-resources | Whether resources usage should be printed during parallel job execution ("true"/ "false") | false | | +| default-branch | Which branch is default for the build ("main" for main branch, "v2-4-test" for 2.4 line etc.) | main | | +| default-constraints-branch | Which branch is default for the build ("constraints-main" for main branch, "constraints-2-4" etc.) | constraints-main | | +| default-helm-version | Which Helm version to use as default | v3.9.4 | | +| default-kind-version | Which Kind version to use as default | v0.16.0 | | +| default-kubernetes-version | Which Kubernetes version to use as default | v1.25.2 | | +| default-mysql-version | Which MySQL version to use as default | 5.7 | | +| default-postgres-version | Which Postgres version to use as default | 10 | | +| default-python-version | Which Python version to use as default | 3.9 | | +| docker-cache | Which cache should be used for images ("registry", "local" , "disabled") | registry | | +| docs-build | Whether to build documentation ("true"/"false") | true | | +| docs-list-as-string | What filter to apply to docs building - based on which documentation packages should be built | apache-airflow helm-chart google | | +| full-tests-needed | Whether this build runs complete set of tests or only subset (for faster PR builds) \[1\] | false | | +| generated-dependencies-changed | Whether generated dependencies have changed ("true"/"false") | false | | +| hatch-build-changed | When hatch build.py changed in the PR. | false | | +| helm-version | Which Helm version to use for tests | v3.9.4 | | +| individual-providers-test-types-list-as-string | Which test types should be run for unit tests for providers (individually listed) | Providers[\amazon\] Providers\[google\] | * | +| is-airflow-runner | Whether runner used is an airflow or infrastructure runner (true if airflow/false if infrastructure) | false | | +| is-amd-runner | Whether runner used is an AMD one | true | | +| is-arm-runner | Whether runner used is an ARM one | false | | +| is-committer-build | Whether the build is triggered by a committer | false | | +| is-k8s-runner | Whether the build runs on our k8s infrastructure | false | | +| is-self-hosted-runner | Whether the runner is self-hosted | false | | +| is-vm-runner | Whether the runner uses VM to run | true | | +| kind-version | Which Kind version to use for tests | v0.16.0 | | +| kubernetes-combos-list-as-string | All combinations of Python version and Kubernetes version to use for tests as space-separated string | 3.9-v1.25.2 3.9-v1.26.4 | * | +| kubernetes-versions | All Kubernetes versions to use for tests as JSON array | \['v1.25.2'\] | | +| kubernetes-versions-list-as-string | All Kubernetes versions to use for tests as space-separated string | v1.25.2 | * | +| mypy-checks | List of folders to be considered for mypy | [] | | +| mysql-exclude | Which versions of MySQL to exclude for tests as JSON array | [] | | +| mysql-versions | Which versions of MySQL to use for tests as JSON array | \['5.7'\] | | +| needs-api-codegen | Whether "api-codegen" are needed to run ("true"/"false") | true | | +| needs-api-tests | Whether "api-tests" are needed to run ("true"/"false") | true | | +| needs-helm-tests | Whether Helm tests are needed to run ("true"/"false") | true | | +| needs-javascript-scans | Whether javascript CodeQL scans should be run ("true"/"false") | true | | +| needs-mypy | Whether mypy check is supposed to run in this build | true | | +| needs-python-scans | Whether Python CodeQL scans should be run ("true"/"false") | true | | +| postgres-exclude | Which versions of Postgres to exclude for tests as JSON array | [] | | +| postgres-versions | Which versions of Postgres to use for tests as JSON array | \['10'\] | | +| prod-image-build | Whether PROD image build is needed | true | | +| prod-image-build | Whether PROD image build is needed | true | | +| providers-compatibility-checks | List of dicts: (python_version, airflow_version, removed_providers) for compatibility checks | [] | | +| providers-test-types-list-as-string | Which test types should be run for unit tests for providers | Providers Providers\[-google\] | * | +| pyproject-toml-changed | When pyproject.toml changed in the PR. | false | | +| python-versions | List of python versions to use for that build | \['3.9'\] | * | +| python-versions-list-as-string | Which versions of MySQL to use for tests as space-separated string | 3.9 | * | +| run-amazon-tests | Whether Amazon tests should be run ("true"/"false") | true | | +| run-kubernetes-tests | Whether Kubernetes tests should be run ("true"/"false") | true | | +| run-tests | Whether unit tests should be run ("true"/"false") | true | | +| run-ui-tests | Whether WWW tests should be run ("true"/"false") | true | | +| run-www-tests | Whether WWW tests should be run ("true"/"false") | true | | +| runs-on-as-json-default | List of labels assigned for runners for that build for default runs for that build (as string) | \["ubuntu-22.04"\] | | +| runs-on-as-json-self-hosted | List of labels assigned for runners for that build for self hosted runners | \["self-hosted", "Linux", "X64"\] | | +| runs-on-as-json-public | List of labels assigned for runners for that build for public runners | \["ubuntu-22.04"\] | | +| skip-pre-commits | Which pre-commits should be skipped during the static-checks run | flynt,identity | | +| skip-provider-tests | When provider tests should be skipped (on non-main branch or when no provider changes detected) | true | | +| sqlite-exclude | Which versions of Sqlite to exclude for tests as JSON array | [] | | +| testable-integrations | List of integrations that are testable in the build as JSON array | \['mongo', 'kafka', 'mssql'\] | | +| upgrade-to-newer-dependencies | Whether the image build should attempt to upgrade all dependencies (true/false or commit hash) | false | | [1] Note for deciding if `full tests needed` mode is enabled and provider.yaml files. diff --git a/dev/breeze/doc/images/output_setup_check-all-params-in-groups.svg b/dev/breeze/doc/images/output_setup_check-all-params-in-groups.svg index 836aa93edd3c..187a9a407351 100644 --- a/dev/breeze/doc/images/output_setup_check-all-params-in-groups.svg +++ b/dev/breeze/doc/images/output_setup_check-all-params-in-groups.svg @@ -1,4 +1,4 @@ - +